diff --git a/.github/workflows/sync-cloud-run-env.yml b/.github/workflows/sync-cloud-run-env.yml index bab7664..a375c90 100644 --- a/.github/workflows/sync-cloud-run-env.yml +++ b/.github/workflows/sync-cloud-run-env.yml @@ -203,6 +203,7 @@ jobs: fi # Trusted control code stays on this main checkout. The application archive is chosen below. approved_candidate=0b939723c1db3ef59175535998b470cbcd4b8824 + approved_http_snapshot_candidate=d8314a61df697cae1dd03a78ddc5c2fc4179ec67 history_count=0 for history_value in \ "${ACCOUNT_HISTORY_RECORDING_ENABLED:-}" \ @@ -248,6 +249,9 @@ jobs: elif [ "${WORKFLOW_TARGET}" = "PAPER" ] && [ "${history_count}" -eq 4 ] \ && [ "${SOURCE_COMMIT}" = "${approved_candidate}" ]; then image_mode=history + elif [ "${WORKFLOW_TARGET}" = "PAPER" ] && [ "${history_count}" -eq 0 ] \ + && [ "${SOURCE_COMMIT}" = "${approved_http_snapshot_candidate}" ]; then + image_mode=snapshot else echo "Image source is not approved." >&2 exit 1 @@ -300,6 +304,7 @@ jobs: run: | set -euo pipefail approved_candidate=0b939723c1db3ef59175535998b470cbcd4b8824 + approved_http_snapshot_candidate=d8314a61df697cae1dd03a78ddc5c2fc4179ec67 history_count=0 for history_value in \ "${ACCOUNT_HISTORY_RECORDING_ENABLED:-}" \ @@ -316,6 +321,9 @@ jobs: elif [ "${WORKFLOW_TARGET}" = "PAPER" ] && [ "${history_count}" -eq 4 ] \ && [ "${SOURCE_COMMIT}" = "${approved_candidate}" ]; then image_mode=history + elif [ "${WORKFLOW_TARGET}" = "PAPER" ] && [ "${history_count}" -eq 0 ] \ + && [ "${SOURCE_COMMIT}" = "${approved_http_snapshot_candidate}" ]; then + image_mode=snapshot else echo "Image source is not approved." >&2 exit 1 @@ -323,6 +331,9 @@ jobs: if [ "${image_mode}" = "same" ]; then archive_ref=HEAD image_tag="${GITHUB_SHA}" + elif [ "${image_mode}" = "snapshot" ]; then + archive_ref="${approved_http_snapshot_candidate}" + image_tag="${approved_http_snapshot_candidate}" else archive_ref="${approved_candidate}" image_tag="${approved_candidate}" @@ -337,10 +348,10 @@ jobs: history_arg="" snapshot_arg="" plan="" - if [ "${archive_ref}" = "${approved_candidate}" ]; then - git fetch --depth 1 origin "${approved_candidate}" - if [ "$(git cat-file -t "${approved_candidate}")" != "commit" ] \ - || [ "$(git rev-parse "${approved_candidate}^{commit}")" != "${approved_candidate}" ]; then + if [ "${image_mode}" != "same" ]; then + git fetch --depth 1 origin "${archive_ref}" + if [ "$(git cat-file -t "${archive_ref}")" != "commit" ] \ + || [ "$(git rev-parse "${archive_ref}^{commit}")" != "${archive_ref}" ]; then echo "Approved candidate commit is not available in this repository." >&2 exit 1 fi diff --git a/scripts/verify_deployed_runtime_target_admission.py b/scripts/verify_deployed_runtime_target_admission.py index 7f0bd94..4a15cc4 100644 --- a/scripts/verify_deployed_runtime_target_admission.py +++ b/scripts/verify_deployed_runtime_target_admission.py @@ -33,8 +33,16 @@ class AdmissionError(ValueError): _SHA = re.compile(r"^[0-9a-f]{40}$") -# Reviewed PAPER history image. Workflow control stays on main; this constant is the only other archive input. +# Reviewed PAPER history image. Workflow control stays on main; only fixed reviewed candidates may be archived. APPROVED_PAPER_HISTORY_CANDIDATE = "0b939723c1db3ef59175535998b470cbcd4b8824" +# Reviewed PAPER HTTP snapshot image. Distinct from the natural-cycle history archive above. +APPROVED_PAPER_HTTP_SNAPSHOT_CANDIDATE = "d8314a61df697cae1dd03a78ddc5c2fc4179ec67" +# One already-staged PAPER revision that may hold history env while serving still runs an older image. +_PAPER_HTTP_STAGED_SOURCE_REVISION = "longbridge-quant-paper-service-r36423178119" +_PAPER_HTTP_STAGED_SOURCE_COMMIT = APPROVED_PAPER_HISTORY_CANDIDATE +_PAPER_HTTP_STAGED_SOURCE_IMAGE_DIGEST = ( + "sha256:9a3260ca8255873309b1c27cd2fd7403e6006a4e9088241e6f89a4e5e65f7a10" +) _HISTORY_KEYS = ( "ACCOUNT_HISTORY_RECORDING_ENABLED", "ACCOUNT_HISTORY_GCS_PREFIX", @@ -461,6 +469,163 @@ def verify_template_matches_serving( return template_configuration +def _template_container_image(service_json: Mapping[str, Any]) -> str: + template = service_json.get("spec", {}).get("template") + if not isinstance(template, Mapping): + raise AdmissionError("container environment is malformed") + containers = (template.get("spec") or {}).get("containers") if isinstance(template.get("spec"), Mapping) else None + if not isinstance(containers, list) or not containers or not isinstance(containers[0], Mapping): + raise AdmissionError("container environment is malformed") + image = str(containers[0].get("image") or "").strip() + if not image: + raise AdmissionError("container environment is malformed") + return image + + +def _revision_container_image(revision: Mapping[str, Any]) -> str: + containers = (revision.get("spec") or {}).get("containers") if isinstance(revision.get("spec"), Mapping) else None + if not isinstance(containers, list) or not containers or not isinstance(containers[0], Mapping): + raise AdmissionError("staged source revision is incomplete") + image = str(containers[0].get("image") or "").strip() + if not image: + raise AdmissionError("staged source revision is incomplete") + return image + + +def _image_digest(image: str) -> str: + _, separator, digest = image.partition("@") + if separator != "@" or not re.fullmatch(r"sha256:[0-9a-f]{64}", digest): + raise AdmissionError("staged source revision image digest is invalid") + return digest + + +def _retained_history_values( + configuration: Mapping[str, Any], *, project_id: str +) -> dict[str, str]: + """Keep the exact literal PAPER history quartet already on the template.""" + + literals = _literal_values(configuration) + values = {key: str(literals.get(key) or "") for key in _HISTORY_KEYS} + if any(not values[key] for key in _HISTORY_KEYS): + raise AdmissionError("retained history settings are incomplete") + try: + retained = history_update(values, workflow_target="PAPER", project_id=project_id) + except AdmissionError as exc: + raise AdmissionError("retained history settings are invalid") from exc + if retained is None: + raise AdmissionError("retained history settings are incomplete") + return retained + + +def _optional_retained_history_values( + configuration: Mapping[str, Any], *, project_id: str +) -> dict[str, str]: + present = {item["name"] for item in configuration["env"]} + if not present.intersection(_HISTORY_KEYS): + return {} + return _retained_history_values(configuration, project_id=project_id) + + +def _admit_http_snapshot_template_via_staged_source( + *, + service: str, + project: str, + region: str, + service_json: Mapping[str, Any], + serving_revisions: Sequence[Mapping[str, Any]], + run: Callable[[Sequence[str]], str], +) -> tuple[dict[str, Any], dict[str, str]]: + """Allow one known staged history revision to differ from serving only on history keys.""" + + if not serving_revisions: + raise AdmissionError(f"{service}: serving revisions are unavailable") + staged = _load_object( + run( + [ + "gcloud", + "run", + "revisions", + "describe", + _PAPER_HTTP_STAGED_SOURCE_REVISION, + f"--project={project}", + f"--region={region}", + "--format=json", + ] + ) + ) + try: + observed = observe_ready_revision(staged) + except ReconcileError as exc: + raise AdmissionError("staged source revision was not observed") from exc + staged_image = _revision_container_image(staged) + if ( + observed["revision"] != _PAPER_HTTP_STAGED_SOURCE_REVISION + or observed["commit"] != _PAPER_HTTP_STAGED_SOURCE_COMMIT + or _image_digest(staged_image) != _PAPER_HTTP_STAGED_SOURCE_IMAGE_DIGEST + or observed["image"] != staged_image + ): + raise AdmissionError("staged source revision does not match the approved source") + status = service_json.get("status") + if not isinstance(status, Mapping): + raise AdmissionError(f"{service}: template does not match the staged source revision") + template = service_json.get("spec", {}).get("template") + template_metadata = template.get("metadata") if isinstance(template, Mapping) else None + if ( + not isinstance(template_metadata, Mapping) + or str(template_metadata.get("name") or "").strip() != _PAPER_HTTP_STAGED_SOURCE_REVISION + ): + raise AdmissionError(f"{service}: template does not match the staged source revision") + if str(status.get("latestCreatedRevisionName") or "").strip() != _PAPER_HTTP_STAGED_SOURCE_REVISION: + raise AdmissionError(f"{service}: template does not match the staged source revision") + template_configuration = _configuration(service_json) + staged_configuration = _configuration(staged) + if template_configuration != staged_configuration: + raise AdmissionError(f"{service}: template does not match the staged source revision") + if _template_container_image(service_json) != staged_image: + raise AdmissionError(f"{service}: template does not match the staged source revision") + retained = _retained_history_values(template_configuration, project_id=project) + for revision in serving_revisions: + serving_configuration = _configuration(revision) + if _config_digest(template_configuration, skip=set(_HISTORY_KEYS)) != _config_digest( + serving_configuration, skip=set(_HISTORY_KEYS) + ): + raise AdmissionError(f"{service}: template does not match the serving revision") + return template_configuration, retained + + +def _resolve_paper_template_configuration( + *, + service: str, + project: str, + region: str, + service_json: Mapping[str, Any], + serving_revisions: Sequence[Mapping[str, Any]], + image_commit: str, + run: Callable[[Sequence[str]], str], +) -> tuple[dict[str, Any], dict[str, str]]: + if image_commit != APPROVED_PAPER_HTTP_SNAPSHOT_CANDIDATE: + return ( + verify_template_matches_serving( + service=service, service_json=service_json, serving_revisions=serving_revisions + ), + {}, + ) + try: + configuration = verify_template_matches_serving( + service=service, service_json=service_json, serving_revisions=serving_revisions + ) + except AdmissionError: + return _admit_http_snapshot_template_via_staged_source( + service=service, + project=project, + region=region, + service_json=service_json, + serving_revisions=serving_revisions, + run=run, + ) + return configuration, _optional_retained_history_values(configuration, project_id=project) + + def _require_candidate_release_binding( *, service: str, service_json: Mapping[str, Any], profile: str, ues_revision: str ) -> None: @@ -495,7 +660,7 @@ def prepare_image_only_staging( image_commit: str, run: Callable[[Sequence[str]], str], ) -> dict[str, Any]: - """Admit a fixed history candidate or the signed main image for PAPER staging.""" + """Admit a fixed PAPER candidate or the signed main image for image-only staging.""" history, snapshot_value = _validate_image_only_source( image_commit=image_commit, env=env, project=project @@ -526,9 +691,21 @@ def prepare_image_only_staging( ) ) ) - configuration = verify_template_matches_serving( - service=service, service_json=service_json, serving_revisions=revisions - ) + if image_commit == APPROVED_PAPER_HTTP_SNAPSHOT_CANDIDATE: + configuration, retained_history = _resolve_paper_template_configuration( + service=service, + project=project, + region=region, + service_json=service_json, + serving_revisions=revisions, + image_commit=image_commit, + run=run, + ) + else: + configuration = verify_template_matches_serving( + service=service, service_json=service_json, serving_revisions=revisions + ) + retained_history = {} source_revision = ues_revision_at(image_commit, run) _require_candidate_release_binding( service=service, @@ -553,6 +730,7 @@ def prepare_image_only_staging( return { "history_arg": history_arg, "history_values": history or {}, + "retained_history_values": retained_history, "snapshot_arg": snapshot_arg, "snapshot_value": snapshot_value, "image_commit": image_commit, @@ -572,6 +750,9 @@ def _validate_image_only_source( if image_commit == APPROVED_PAPER_HISTORY_CANDIDATE: if history is None or snapshot_value is not None: raise AdmissionError("image source is not approved") + elif image_commit == APPROVED_PAPER_HTTP_SNAPSHOT_CANDIDATE: + if str(env.get("WORKFLOW_TARGET") or "") != "PAPER" or history is not None: + raise AdmissionError("image source is not approved") elif _is_exact_main_image(image_commit, env): if history is not None: raise AdmissionError("image source is not approved") @@ -675,7 +856,8 @@ def confirm_image_only_readback( ): raise AdmissionError("staged revision does not match the approved image") history = plan.get("history_values") or {} - if not isinstance(history, Mapping): + retained_history = plan.get("retained_history_values") or {} + if not isinstance(history, Mapping) or not isinstance(retained_history, Mapping): raise AdmissionError("staged readback is incomplete") snapshot_value = plan.get("snapshot_value") if snapshot_value not in {None, "true", "false"}: @@ -690,6 +872,9 @@ def confirm_image_only_readback( for key, value in history.items(): if literals.get(str(key)) != value: raise AdmissionError("staged history settings do not match") + for key, value in retained_history.items(): + if literals.get(str(key)) != value: + raise AdmissionError("retained history settings do not match") if snapshot_value is not None and literals.get(_ACCOUNT_SNAPSHOT_ENV) != snapshot_value: raise AdmissionError("staged account snapshot setting does not match") diff --git a/tests/test_deployed_runtime_target_admission.py b/tests/test_deployed_runtime_target_admission.py index 154a75d..1f06a43 100644 --- a/tests/test_deployed_runtime_target_admission.py +++ b/tests/test_deployed_runtime_target_admission.py @@ -82,6 +82,12 @@ def test_verify_service_rejects_target_drift(target, profile, message): "WORKFLOW_TARGET": "PAPER", } MAIN_SHA = "a" * 40 +HTTP_SNAPSHOT_CANDIDATE = admission.APPROVED_PAPER_HTTP_SNAPSHOT_CANDIDATE +STAGED_SOURCE_REVISION = "longbridge-quant-paper-service-r36423178119" +STAGED_SOURCE_IMAGE = ( + "asia-east1-docker.pkg.dev/synthetic-project/images/longbridgeplatform/paper-service" + "@sha256:9a3260ca8255873309b1c27cd2fd7403e6006a4e9088241e6f89a4e5e65f7a10" +) def _main_env(*, snapshot_setting: str = "") -> dict[str, str]: @@ -492,3 +498,257 @@ def run(command): confirm(_service(), _revision("paper-service-r123", CANDIDATE, image, ready=False, extra_env=history_env)) with pytest.raises(admission.AdmissionError): confirm(_service(), _revision("paper-service-r123", CANDIDATE, image, extra_env=history_env), run_error=True) + + +def _http_snapshot_service(*, history: dict[str, str] | None = None) -> dict: + service = _service() + template = service["spec"]["template"] + template["metadata"] = {"name": STAGED_SOURCE_REVISION} + container = template["spec"]["containers"][0] + container["image"] = STAGED_SOURCE_IMAGE + if history is not None: + container["env"].extend({"name": key, "value": value} for key, value in history.items()) + service["status"]["latestCreatedRevisionName"] = STAGED_SOURCE_REVISION + return service + + +def _http_snapshot_run(service: dict, *, staged_mutation=None): + serving = _revision("serving-rev", SERVING, "serving-image") + staged = _revision( + STAGED_SOURCE_REVISION, + CANDIDATE, + STAGED_SOURCE_IMAGE, + extra_env=[{"name": key, "value": value} for key, value in HISTORY.items() if key != "WORKFLOW_TARGET"], + ) + if staged_mutation: + staged_mutation(staged) + + def run(command): + if command[:3] == ["gcloud", "run", "revisions"]: + name = command[4] + if name == STAGED_SOURCE_REVISION: + return json.dumps(staged) + assert name == "serving-rev" + return json.dumps(serving) + if command[:2] == ["git", "show"]: + sha, name = command[2].split(":", 1) + assert sha in {HTTP_SNAPSHOT_CANDIDATE, SERVING} + return _declaration(name, UES) + raise AssertionError(command) + + return run + + +def _prepare_http_snapshot(*, service=None, env=None, staged_mutation=None): + service = service or _http_snapshot_service(history={key: value for key, value in HISTORY.items() if key != "WORKFLOW_TARGET"}) + return admission.prepare_image_only_staging( + service="paper-service", + project="synthetic-project", + region="synthetic-region", + service_json=service, + env=env or {**_main_env(), "SOURCE_COMMIT": HTTP_SNAPSHOT_CANDIDATE}, + image_commit=HTTP_SNAPSHOT_CANDIDATE, + run=_http_snapshot_run(service, staged_mutation=staged_mutation), + ) + + +def test_http_snapshot_candidate_retains_exact_staged_history_and_digest(): + service = _http_snapshot_service(history={key: value for key, value in HISTORY.items() if key != "WORKFLOW_TARGET"}) + env = {**_main_env(snapshot_setting="true"), "SOURCE_COMMIT": HTTP_SNAPSHOT_CANDIDATE} + plan = _prepare_http_snapshot(service=service, env=env) + + assert plan["image_commit"] == HTTP_SNAPSHOT_CANDIDATE + assert plan["history_arg"] == "" + assert plan["history_values"] == {} + assert plan["retained_history_values"] == { + key: value for key, value in HISTORY.items() if key != "WORKFLOW_TARGET" + } + assert plan["snapshot_arg"] == "LONGBRIDGE_ACCOUNT_SNAPSHOT_ENABLED=true" + assert plan["template_digest"] == admission._config_digest( + admission._configuration(service), skip={admission._ACCOUNT_SNAPSHOT_ENV} + ) + + +def test_http_snapshot_uses_strict_serving_match_when_template_already_matches(): + service = _service() + calls = [] + + def run(command): + calls.append(list(command)) + if command[:3] == ["gcloud", "run", "revisions"]: + return json.dumps(_revision("serving-rev", SERVING, "serving-image")) + sha, name = command[2].split(":", 1) + assert sha in {HTTP_SNAPSHOT_CANDIDATE, SERVING} + return _declaration(name, UES) + + plan = admission.prepare_image_only_staging( + service="paper-service", project="synthetic-project", region="synthetic-region", + service_json=service, + env={**_main_env(), "SOURCE_COMMIT": HTTP_SNAPSHOT_CANDIDATE}, + image_commit=HTTP_SNAPSHOT_CANDIDATE, + run=run, + ) + assert plan["retained_history_values"] == {} + assert not any( + command[:4] == ["gcloud", "run", "revisions", "describe"] + and command[4] == STAGED_SOURCE_REVISION + for command in calls + ) + + +@pytest.mark.parametrize( + "env_overrides", + [ + {"WORKFLOW_TARGET": "HK"}, + {"WORKFLOW_TARGET": "SG"}, + {"SOURCE_COMMIT": CANDIDATE}, + {"SOURCE_COMMIT": "c" * 40}, + {key: value for key, value in HISTORY.items() if key != "WORKFLOW_TARGET"}, + ], +) +def test_http_snapshot_source_requires_exact_candidate_paper_and_no_history(env_overrides): + env = {**_main_env(), "SOURCE_COMMIT": HTTP_SNAPSHOT_CANDIDATE} + env.update(env_overrides) + calls = [] + with pytest.raises(admission.AdmissionError, match="not approved"): + admission.prepare_image_only_staging( + service="paper-service", project="synthetic-project", region="synthetic-region", + service_json={}, env=env, image_commit=env["SOURCE_COMMIT"], + run=lambda command: calls.append(command) or "", + ) + assert calls == [] + + +@pytest.mark.parametrize( + "mutation", + [ + lambda revision: revision["metadata"].update(name="wrong-revision"), + lambda revision: revision["metadata"]["labels"].update({"commit-sha": "f" * 40}), + lambda revision: revision["spec"]["containers"][0].update( + image="repo@sha256:" + "f" * 64 + ), + ], +) +def test_http_snapshot_rejects_wrong_staged_revision_commit_or_digest(mutation): + with pytest.raises(admission.AdmissionError): + _prepare_http_snapshot(staged_mutation=mutation) + + +def test_http_snapshot_accepts_ready_staged_revision_when_not_active(): + def mark_inactive(revision): + revision["status"]["conditions"].extend( + [ + {"type": "Active", "status": "False"}, + {"type": "Retired", "status": "True"}, + ] + ) + + assert _prepare_http_snapshot(staged_mutation=mark_inactive)["history_arg"] == "" + + +def test_http_snapshot_rejects_template_that_does_not_match_staged_source(): + wrong_revision = _http_snapshot_service( + history={key: value for key, value in HISTORY.items() if key != "WORKFLOW_TARGET"} + ) + wrong_revision["status"]["latestCreatedRevisionName"] = "other-revision" + with pytest.raises(admission.AdmissionError, match="template does not match"): + _prepare_http_snapshot(service=wrong_revision) + + wrong_image = _http_snapshot_service( + history={key: value for key, value in HISTORY.items() if key != "WORKFLOW_TARGET"} + ) + wrong_image["spec"]["template"]["spec"]["containers"][0]["image"] = "repo@sha256:" + "f" * 64 + with pytest.raises(admission.AdmissionError, match="template does not match"): + _prepare_http_snapshot(service=wrong_image) + + wrong_template_name = _http_snapshot_service( + history={key: value for key, value in HISTORY.items() if key != "WORKFLOW_TARGET"} + ) + wrong_template_name["spec"]["template"]["metadata"]["name"] = "other-revision" + with pytest.raises(admission.AdmissionError, match="template does not match"): + _prepare_http_snapshot(service=wrong_template_name) + + +def test_http_snapshot_rejects_history_prefix_that_differs_from_immutable_revision(): + service = _http_snapshot_service( + history={**{key: value for key, value in HISTORY.items() if key != "WORKFLOW_TARGET"}, + "ACCOUNT_HISTORY_GCS_PREFIX": "gs://paper-bucket/other"} + ) + with pytest.raises(admission.AdmissionError, match="template does not match the staged source"): + _prepare_http_snapshot(service=service) + + +@pytest.mark.parametrize("case", ["missing", "secret"]) +def test_http_snapshot_requires_four_literal_retained_history_values(case): + service = _http_snapshot_service( + history={key: value for key, value in HISTORY.items() if key != "WORKFLOW_TARGET"} + ) + if case == "missing": + service["spec"]["template"]["spec"]["containers"][0]["env"] = [ + item for item in service["spec"]["template"]["spec"]["containers"][0]["env"] + if item["name"] != "ACCOUNT_HISTORY_TARGET_ID" + ] + else: + entry = next( + item for item in service["spec"]["template"]["spec"]["containers"][0]["env"] + if item["name"] == "ACCOUNT_HISTORY_GCS_PREFIX" + ) + entry.pop("value") + entry["valueFrom"] = {"secretKeyRef": {"name": "history-config", "key": "prefix"}} + with pytest.raises(admission.AdmissionError): + _prepare_http_snapshot(service=service) + + +def test_http_snapshot_rejects_nonhistory_environment_and_service_identity_drift(): + service = _http_snapshot_service( + history={key: value for key, value in HISTORY.items() if key != "WORKFLOW_TARGET"} + ) + service["spec"]["template"]["spec"]["serviceAccountName"] = "other@example.invalid" + with pytest.raises(admission.AdmissionError): + _prepare_http_snapshot(service=service) + + service = _http_snapshot_service( + history={key: value for key, value in HISTORY.items() if key != "WORKFLOW_TARGET"} + ) + target = json.loads(service["spec"]["template"]["spec"]["containers"][0]["env"][0]["value"]) + target["service_name"] = "other-service" + service["spec"]["template"]["spec"]["containers"][0]["env"][0]["value"] = json.dumps(target) + with pytest.raises(admission.AdmissionError, match="does not match the deployed service"): + _prepare_http_snapshot(service=service) + + +def test_http_snapshot_readback_preserves_retained_history_values(): + service = _http_snapshot_service( + history={key: value for key, value in HISTORY.items() if key != "WORKFLOW_TARGET"} + ) + plan = _prepare_http_snapshot(service=service) + image = "repo@sha256:" + "b" * 64 + history_env = [ + {"name": key, "value": value} + for key, value in HISTORY.items() + if key != "WORKFLOW_TARGET" + ] + staged = _revision( + "paper-service-r123", HTTP_SNAPSHOT_CANDIDATE, image, + extra_env=history_env + [{"name": admission._ACCOUNT_SNAPSHOT_ENV, "value": "false"}], + ) + plan["snapshot_value"] = "false" + + def confirm(revision): + def run(command): + if command[:3] == ["gcloud", "run", "services"]: + return json.dumps(service) + return json.dumps(revision) + admission.confirm_image_only_readback( + service="paper-service", project="synthetic-project", region="synthetic-region", + plan=plan, expected_image=image, expected_commit=HTTP_SNAPSHOT_CANDIDATE, + expected_revision="paper-service-r123", run=run, + ) + + confirm(staged) + for key in admission._HISTORY_KEYS: + changed = json.loads(json.dumps(staged)) + entry = next(item for item in changed["spec"]["containers"][0]["env"] if item["name"] == key) + entry["value"] = "gs://paper-bucket/tampered" if key.endswith("GCS_PREFIX") else "false" + with pytest.raises(admission.AdmissionError): + confirm(changed) diff --git a/tests/test_sync_cloud_run_env_workflow.sh b/tests/test_sync_cloud_run_env_workflow.sh index 11e13fc..9feb691 100644 --- a/tests/test_sync_cloud_run_env_workflow.sh +++ b/tests/test_sync_cloud_run_env_workflow.sh @@ -446,6 +446,7 @@ assert "git checkout" not in job assert 'git archive "${SOURCE_COMMIT}"' not in job assert "git archive ${{" not in job assert "0b939723c1db3ef59175535998b470cbcd4b8824" in job +assert "d8314a61df697cae1dd03a78ddc5c2fc4179ec67" in job assert '[ "${GITHUB_REPOSITORY:-}" != "QuantStrategyLab/LongBridgePlatform" ]' in job assert '[ "${SOURCE_COMMIT}" = "${GITHUB_SHA}" ] || [ "${SOURCE_COMMIT}" = "${approved_candidate}" ]' not in job for forbidden in ("sync_plan", "scheduler", "cleanup", "retire", "update-traffic"): @@ -472,6 +473,18 @@ args = sys.argv[1:] with open(os.environ["STUB_LOG"], "a") as stream: stream.write(json.dumps([command, *args]) + "\\n") approved = "0b939723c1db3ef59175535998b470cbcd4b8824" +http_snapshot_candidate = "d8314a61df697cae1dd03a78ddc5c2fc4179ec67" +staged_source_revision = "longbridge-quant-paper-service-r36423178119" +staged_source_image = ( + "asia-east1-docker.pkg.dev/synthetic-project/images/longbridgeplatform/synthetic-paper" + "@sha256:9a3260ca8255873309b1c27cd2fd7403e6006a4e9088241e6f89a4e5e65f7a10" +) +history = { + "ACCOUNT_HISTORY_RECORDING_ENABLED": "true", + "ACCOUNT_HISTORY_GCS_PREFIX": "gs://paper-bucket/account_snapshots", + "ACCOUNT_HISTORY_TARGET_ID": "paper", + "ACCOUNT_HISTORY_EXPECTED_SCOPE": "PAPER", +} serving_sha = "1" * 40 ues = "e" * 40 @@ -511,16 +524,21 @@ def base_env(): ] def service_document(ingress): + env = base_env() + template = {"spec": {"serviceAccountName": "runtime@example.invalid", "containers": [{"env": env, "image": "serving-image"}]}} + status = {"traffic": [{"revisionName": "serving-rev", "percent": 100}], "latestReadyRevisionName": "ignored-latest"} + if os.environ.get("SOURCE_COMMIT") == http_snapshot_candidate: + env.extend({"name": key, "value": value} for key, value in history.items()) + template["metadata"] = {"name": staged_source_revision} + template["spec"]["containers"][0]["image"] = staged_source_image + status["latestCreatedRevisionName"] = staged_source_revision return { "metadata": { "name": os.environ["CLOUD_RUN_SERVICE"], "annotations": {"run.googleapis.com/ingress": ingress}, }, - "spec": {"template": {"spec": { - "serviceAccountName": "runtime@example.invalid", - "containers": [{"env": base_env(), "image": "serving-image"}], - }}}, - "status": {"traffic": [{"revisionName": "serving-rev", "percent": 100}], "latestReadyRevisionName": "ignored-latest"}, + "spec": {"template": template}, + "status": status, } def serving_revision(): @@ -533,6 +551,20 @@ def serving_revision(): "status": {"conditions": [{"type": "Ready", "status": "True"}]}, } +def staged_source_revision_payload(): + service = os.environ["CLOUD_RUN_SERVICE"] + env = base_env() + [{"name": key, "value": value} for key, value in history.items()] + image = ( + "asia-east1-docker.pkg.dev/synthetic-project/images/longbridgeplatform/" + + service + + "@sha256:9a3260ca8255873309b1c27cd2fd7403e6006a4e9088241e6f89a4e5e65f7a10" + ) + return { + "metadata": {"name": staged_source_revision, "labels": {"commit-sha": approved}}, + "spec": {"serviceAccountName": "runtime@example.invalid", "containers": [{"env": env, "image": image}]}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}, + } + def staged_revision(): service = os.environ["CLOUD_RUN_SERVICE"] image_repo = "registry.invalid/synthetic-project/synthetic-images/longbridgeplatform/" + service @@ -543,6 +575,8 @@ def staged_revision(): "ACCOUNT_HISTORY_TARGET_ID", "ACCOUNT_HISTORY_EXPECTED_SCOPE", ) + if os.environ.get("SOURCE_COMMIT") == http_snapshot_candidate: + env.extend({"name": key, "value": value} for key, value in history.items()) if os.environ.get("CONFIG_DRIFT") == "1": env.append({"name": "UNRELATED_SETTING", "value": "1"}) update_path = Path(os.environ["HOME"]) / "updated-env.json" @@ -568,21 +602,23 @@ def staged_revision(): if command == "git" and args == ["rev-parse", "HEAD"]: print(os.environ["CHECKOUT_SHA"]) -elif command == "git" and args == ["fetch", "--depth", "1", "origin", approved]: +elif command == "git" and args[:4] == ["fetch", "--depth", "1", "origin"] and args[4] in (approved, http_snapshot_candidate): pass -elif command == "git" and args == ["cat-file", "-t", approved]: +elif command == "git" and args[:2] == ["cat-file", "-t"] and args[2] in (approved, http_snapshot_candidate): print("commit") -elif command == "git" and args == ["rev-parse", approved + "^{commit}"]: - print(approved) +elif command == "git" and args[:1] == ["rev-parse"] and len(args) == 2 and args[1] in (approved + "^{commit}", http_snapshot_candidate + "^{commit}"): + print(args[1].split("^", 1)[0]) elif command == "git" and args == ["archive", "HEAD"]: print("synthetic tracked source archive") elif command == "git" and args == ["archive", approved]: print("synthetic candidate archive") +elif command == "git" and args == ["archive", http_snapshot_candidate]: + print("synthetic HTTP snapshot candidate archive") elif command == "git" and args[:1] == ["show"] and len(args) == 2 and ":" in args[1]: sha, name = args[1].split(":", 1) if name not in ("uv.lock", "pyproject.toml", "qsl.toml"): raise SystemExit("unexpected git show") - if sha not in ("a" * 40, serving_sha, approved): + if sha not in ("a" * 40, serving_sha, approved, http_snapshot_candidate): raise SystemExit("admission read an unapproved source lock") pin = ("f" * 40) if sha == serving_sha and os.environ.get("BAD_SERVING_LOCK") == "1" else ues print(declaration(name, pin), end="") @@ -616,7 +652,9 @@ elif command == "gcloud" and args[:3] == ["run", "revisions", "describe"]: revision_name = args[3] if os.environ.get("UNKNOWN_READBACK") == "1" and revision_name.endswith("-r" + os.environ["GITHUB_RUN_ID"]): sys.exit(1) - if revision_name.endswith("-r" + os.environ["GITHUB_RUN_ID"]): + if revision_name == staged_source_revision: + print(json.dumps(staged_source_revision_payload())) + elif revision_name.endswith("-r" + os.environ["GITHUB_RUN_ID"]): print(json.dumps(staged_revision())) else: print(json.dumps(serving_revision())) @@ -659,6 +697,12 @@ else: "ADMISSION_PYTHON": str(Path(sys.argv[1]).resolve().parents[2] / ".venv" / "bin" / "python"), } candidate = "0b939723c1db3ef59175535998b470cbcd4b8824" + http_snapshot_candidate = "d8314a61df697cae1dd03a78ddc5c2fc4179ec67" + staged_source_revision = "longbridge-quant-paper-service-r36423178119" + staged_source_image = ( + "asia-east1-docker.pkg.dev/synthetic-project/images/longbridgeplatform/" + "synthetic-paper@sha256:9a3260ca8255873309b1c27cd2fd7403e6006a4e9088241e6f89a4e5e65f7a10" + ) history = { "ACCOUNT_HISTORY_RECORDING_ENABLED": "true", "ACCOUNT_HISTORY_GCS_PREFIX": "gs://paper-bucket/account_snapshots", @@ -760,6 +804,38 @@ else: assert not any("record_daily" in part for call in calls for part in call) assert sum(call[:4] == ["gcloud", "run", "services", "update"] for call in calls) == 1 cases += 1 + code, calls = execute(SOURCE_COMMIT=http_snapshot_candidate, CHECKOUT_SHA="a" * 40) + assert code == 0, (code, calls) + updates = [call for call in calls if call[:4] == ["gcloud", "run", "services", "update"]] + assert len(updates) == 1 + assert "--update-env-vars=" not in " ".join(updates[0]) + assert updates[0][-1] == "--revision-suffix=r123" + assert f"--image=registry.invalid/synthetic-project/synthetic-images/longbridgeplatform/synthetic-paper@{base['IMAGE_DIGEST']}" in updates[0] + assert f"--update-labels=commit-sha={http_snapshot_candidate},github-run-id=123" in updates[0] + assert ["docker", "build", "--pull", "-t", + f"registry.invalid/synthetic-project/synthetic-images/longbridgeplatform/synthetic-paper:{http_snapshot_candidate}-123", "-"] in calls + assert ["git", "fetch", "--depth", "1", "origin", http_snapshot_candidate] in calls + assert ["git", "archive", http_snapshot_candidate] in calls + assert ["git", "archive", "HEAD"] not in calls + assert ["gcloud", "run", "revisions", "describe", staged_source_revision, + "--project=synthetic-project", "--region=synthetic-region", "--format=json"] in calls + assert not any(call[:4] == ["gcloud", "run", "services", "update"] and "ACCOUNT_HISTORY_" in " ".join(call) for call in calls) + cases += 1 + code, calls = execute(SOURCE_COMMIT=http_snapshot_candidate, CHECKOUT_SHA="a" * 40, ACCOUNT_SNAPSHOT_ENABLED_INPUT="true") + assert code == 0, (code, calls) + updates = [call for call in calls if call[:4] == ["gcloud", "run", "services", "update"]] + assert len(updates) == 1 + update_args = [arg for arg in updates[0] if arg.startswith("--update-env-vars=")] + assert update_args == ["--update-env-vars=LONGBRIDGE_ACCOUNT_SNAPSHOT_ENABLED=true"] + assert ["git", "archive", http_snapshot_candidate] in calls + cases += 1 + for overrides in ( + {"SOURCE_COMMIT": http_snapshot_candidate, "WORKFLOW_TARGET": "HK"}, + {"SOURCE_COMMIT": http_snapshot_candidate, **history}, + ): + code, calls = execute(**overrides) + assert code != 0 and calls == [], overrides + cases += 1 for overrides in ( { "GITHUB_REF": "refs/heads/codex/natural-cycle-history-20260928",