Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 19 additions & 3 deletions .github/workflows/sync-cloud-run-env.yml
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,7 @@ jobs:
approved_probe_financing_candidate=82788aa7c73690d2b87f63540a9102943e9372eb
approved_sghk_snapshot_candidate=df3d29d13daeffc7b4af9fec5db0e9ec1f07b60f
approved_hk_probe_diagnostics_candidate=779d8e1c39d161144bc34ae8503ce34d87971b16
approved_sghk_stock_inventory_candidate=b9dbd2f697c84182a8959fcbbd360a5211f502c6
history_count=0
for history_value in \
"${ACCOUNT_HISTORY_RECORDING_ENABLED:-}" \
Expand Down Expand Up @@ -295,6 +296,10 @@ jobs:
&& [ -z "${snapshot_setting}" ] \
&& [ "${SOURCE_COMMIT}" = "${approved_hk_probe_diagnostics_candidate}" ]; then
image_mode=hk-probe-diagnostics
elif { [ "${WORKFLOW_TARGET}" = "HK" ] || [ "${WORKFLOW_TARGET}" = "SG" ]; } \
&& [ "${history_count}" -eq 0 ] && [ -z "${snapshot_setting}" ] \
&& [ "${SOURCE_COMMIT}" = "${approved_sghk_stock_inventory_candidate}" ]; then
image_mode=stock-inventory
else
echo "Image source is not approved." >&2
exit 1
Expand Down Expand Up @@ -352,6 +357,7 @@ jobs:
approved_probe_financing_candidate=82788aa7c73690d2b87f63540a9102943e9372eb
approved_sghk_snapshot_candidate=df3d29d13daeffc7b4af9fec5db0e9ec1f07b60f
approved_hk_probe_diagnostics_candidate=779d8e1c39d161144bc34ae8503ce34d87971b16
approved_sghk_stock_inventory_candidate=b9dbd2f697c84182a8959fcbbd360a5211f502c6
case "${WORKFLOW_TARGET}" in
HK) expected_sghk_target=hk ;;
SG) expected_sghk_target=sg ;;
Expand Down Expand Up @@ -401,6 +407,10 @@ jobs:
&& [ -z "${snapshot_setting}" ] \
&& [ "${SOURCE_COMMIT}" = "${approved_hk_probe_diagnostics_candidate}" ]; then
image_mode=hk-probe-diagnostics
elif { [ "${WORKFLOW_TARGET}" = "HK" ] || [ "${WORKFLOW_TARGET}" = "SG" ]; } \
&& [ "${history_count}" -eq 0 ] && [ -z "${snapshot_setting}" ] \
&& [ "${SOURCE_COMMIT}" = "${approved_sghk_stock_inventory_candidate}" ]; then
image_mode=stock-inventory
else
echo "Image source is not approved." >&2
exit 1
Expand All @@ -423,6 +433,9 @@ jobs:
elif [ "${image_mode}" = "hk-probe-diagnostics" ]; then
archive_ref="${approved_hk_probe_diagnostics_candidate}"
image_tag="${approved_hk_probe_diagnostics_candidate}"
elif [ "${image_mode}" = "stock-inventory" ]; then
archive_ref="${approved_sghk_stock_inventory_candidate}"
image_tag="${approved_sghk_stock_inventory_candidate}"
else
archive_ref="${approved_candidate}"
image_tag="${approved_candidate}"
Expand All @@ -447,7 +460,8 @@ jobs:
fi
if [ "${WORKFLOW_TARGET}" = "PAPER" ] || [ "${image_mode}" = "account-history" ] \
|| [ "${image_mode}" = "hk-account-snapshot" ] \
|| [ "${image_mode}" = "hk-probe-diagnostics" ]; then
|| [ "${image_mode}" = "hk-probe-diagnostics" ] \
|| [ "${image_mode}" = "stock-inventory" ]; then
plan="$(mktemp)"
uv run --no-sync python "${GITHUB_WORKSPACE}/scripts/verify_deployed_runtime_target_admission.py" \
--project="${GCP_PROJECT_ID}" \
Expand Down Expand Up @@ -488,7 +502,8 @@ jobs:
fi
if [ "${WORKFLOW_TARGET}" = "PAPER" ] || [ "${image_mode}" = "account-history" ] \
|| [ "${image_mode}" = "hk-account-snapshot" ] \
|| [ "${image_mode}" = "hk-probe-diagnostics" ]; then
|| [ "${image_mode}" = "hk-probe-diagnostics" ] \
|| [ "${image_mode}" = "stock-inventory" ]; then
revision_suffix="r${GITHUB_RUN_ID}"
revision_name="${CLOUD_RUN_SERVICE}-${revision_suffix}"
if [[ ! "${GITHUB_RUN_ID}" =~ ^[0-9]+$ ]] \
Expand All @@ -501,7 +516,8 @@ jobs:
"${update_command[@]}"
if [ "${WORKFLOW_TARGET}" = "PAPER" ] || [ "${image_mode}" = "account-history" ] \
|| [ "${image_mode}" = "hk-account-snapshot" ] \
|| [ "${image_mode}" = "hk-probe-diagnostics" ]; then
|| [ "${image_mode}" = "hk-probe-diagnostics" ] \
|| [ "${image_mode}" = "stock-inventory" ]; then
uv run --no-sync python "${GITHUB_WORKSPACE}/scripts/verify_deployed_runtime_target_admission.py" \
--project="${GCP_PROJECT_ID}" \
--region="${CLOUD_RUN_REGION}" \
Expand Down
13 changes: 12 additions & 1 deletion scripts/verify_deployed_runtime_target_admission.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ class AdmissionError(ValueError):
APPROVED_PAPER_PROBE_FINANCING_CANDIDATE = "82788aa7c73690d2b87f63540a9102943e9372eb"
# Reviewed read-only SG/HK account-snapshot producer. It is not a PAPER candidate.
APPROVED_SGHK_ACCOUNT_SNAPSHOT_CANDIDATE = "df3d29d13daeffc7b4af9fec5db0e9ec1f07b60f"
# Reviewed read-only SG/HK stock-inventory image. It carries no history or snapshot update.
APPROVED_SGHK_STOCK_INVENTORY_CANDIDATE = "b9dbd2f697c84182a8959fcbbd360a5211f502c6"
# Fixed 922-based HK candidate carrying only bounded /probe failure diagnostics.
APPROVED_HK_PROBE_DIAGNOSTICS_CANDIDATE = "779d8e1c39d161144bc34ae8503ce34d87971b16"
# One already-staged PAPER revision that may hold history env while serving still runs an older image.
Expand Down Expand Up @@ -725,7 +727,7 @@ def prepare_image_only_staging(
image_commit: str,
run: Callable[[Sequence[str]], str],
) -> dict[str, Any]:
"""Admit a fixed PAPER candidate or the signed main image for image-only staging."""
"""Admit a fixed PAPER candidate, a fixed SG/HK candidate, or the signed main image."""

history, snapshot_value = _validate_image_only_source(
image_commit=image_commit, env=env, project=project
Expand All @@ -734,6 +736,7 @@ def prepare_image_only_staging(
if image_commit in {
APPROVED_SGHK_ACCOUNT_SNAPSHOT_CANDIDATE,
APPROVED_HK_PROBE_DIAGNOSTICS_CANDIDATE,
APPROVED_SGHK_STOCK_INVENTORY_CANDIDATE,
}:
_require_sghk_candidate_identity(
target_label=str(env.get("WORKFLOW_TARGET") or ""),
Expand Down Expand Up @@ -878,6 +881,14 @@ def _validate_image_only_source(
or snapshot_value is not None
):
raise AdmissionError("image source is not approved")
elif image_commit == APPROVED_SGHK_STOCK_INVENTORY_CANDIDATE:
# Read-only SG/HK inventory image: no PAPER target and no environment update.
if (
str(env.get("WORKFLOW_TARGET") or "") not in {"HK", "SG"}
or history is not None
or snapshot_value is not None
):
raise AdmissionError("image source is not approved")
elif _is_exact_main_image(image_commit, env):
if history is not None:
raise AdmissionError("image source is not approved")
Expand Down
231 changes: 231 additions & 0 deletions tests/test_deployed_runtime_target_admission.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ def test_verify_service_rejects_target_drift(target, profile, message):
PROBE_CANDIDATES = (PROBE_SNAPSHOT_CANDIDATE, PROBE_FINANCING_CANDIDATE)
SGHK_SNAPSHOT_CANDIDATE = admission.APPROVED_SGHK_ACCOUNT_SNAPSHOT_CANDIDATE
HK_PROBE_DIAGNOSTICS_CANDIDATE = admission.APPROVED_HK_PROBE_DIAGNOSTICS_CANDIDATE
STOCK_INVENTORY_CANDIDATE = admission.APPROVED_SGHK_STOCK_INVENTORY_CANDIDATE
SGHK_PREFIX = "gs://qsl-runtime-logs-shared/longbridge/account_snapshots"
STAGED_SOURCE_REVISION = "longbridge-quant-paper-service-r36423178119"
STAGED_SOURCE_IMAGE = (
Expand Down Expand Up @@ -1206,3 +1207,233 @@ def run(command):
run=run,
)
assert not any(command[:3] == ["gcloud", "run", "services"] and "update" in command for command in calls)


def _stock_inventory_service(target_label: str, *, target_overrides=None, extra_env=None):
target = _sghk_target(target_label)
target.update(target_overrides or {})
env = [
{"name": "RUNTIME_TARGET_JSON", "value": json.dumps(target)},
{"name": "STRATEGY_PROFILE", "value": target["strategy_profile"]},
{"name": "LONGBRIDGE_DRY_RUN_ONLY", "value": "false"},
{"name": "RUNTIME_TARGET_ENABLED", "value": "true"},
] + list(extra_env or [])
service = {
"metadata": {"annotations": {"run.googleapis.com/ingress": "internal"}},
"spec": {
"template": {
"spec": {
"serviceAccountName": "runtime@example.invalid",
"containers": [{"env": env}],
}
}
},
"status": {"traffic": [{"revisionName": "serving-rev", "percent": 100}]},
}
return service, env


def _prepare_stock_inventory(
target_label: str = "HK",
*,
target_overrides=None,
env_overrides=None,
project: str = "longbridgequant",
region: str | None = None,
serving_extra_env=None,
serving_pin: str = UES,
calls: list | None = None,
):
service, env = _stock_inventory_service(target_label, target_overrides=target_overrides)
target = json.loads(env[0]["value"])
if region is None:
region = {"HK": "asia-east2", "SG": "asia-southeast1"}[target_label]
candidate_env = {
"WORKFLOW_TARGET": target_label,
"SOURCE_COMMIT": STOCK_INVENTORY_CANDIDATE,
}
candidate_env.update(env_overrides or {})

def run(command):
if calls is not None:
calls.append(list(command))
if command[:3] == ["gcloud", "run", "revisions"]:
return json.dumps({
"metadata": {"name": "serving-rev", "labels": {"commit-sha": SERVING}},
"spec": {
"serviceAccountName": "runtime@example.invalid",
"containers": [
{"env": env + list(serving_extra_env or []), "image": "serving-image"}
],
},
"status": {"conditions": [{"type": "Ready", "status": "True"}]},
})
pin = serving_pin if command[2].startswith(SERVING + ":") else UES
return _declaration(command[2].split(":", 1)[1], pin)

return admission.prepare_image_only_staging(
service=target["service_name"],
project=project,
region=region,
service_json=service,
env=candidate_env,
image_commit=STOCK_INVENTORY_CANDIDATE,
run=run,
)


@pytest.mark.parametrize("target_label", ["HK", "SG"])
def test_stock_inventory_candidate_admits_exact_sghk_identity_without_updates(target_label):
calls = []
plan = _prepare_stock_inventory(target_label, calls=calls)

assert plan["image_commit"] == STOCK_INVENTORY_CANDIDATE
assert plan["history_arg"] == ""
assert plan["snapshot_arg"] == ""
assert plan["history_values"] == {}
assert plan["retained_history_values"] == {}
assert plan["snapshot_value"] is None
assert plan["serving_traffic"] == [{"revisionName": "serving-rev", "percent": 100}]
assert admission._validate_image_only_source(
image_commit=STOCK_INVENTORY_CANDIDATE,
env={"WORKFLOW_TARGET": target_label},
project="longbridgequant",
) == (None, None)
assert not any(
command[:3] == ["gcloud", "run", "services"] and "update" in command for command in calls
)


@pytest.mark.parametrize(
"env",
[
{"WORKFLOW_TARGET": "PAPER"},
{"WORKFLOW_TARGET": "configured"},
_sghk_history("HK"),
_sghk_history("SG"),
{"WORKFLOW_TARGET": "HK", "ACCOUNT_SNAPSHOT_ENABLED_INPUT": "true"},
{"WORKFLOW_TARGET": "HK", "ACCOUNT_SNAPSHOT_ENABLED_INPUT": "false"},
{"WORKFLOW_TARGET": "SG", "ACCOUNT_SNAPSHOT_ENABLED_INPUT": "true"},
],
)
def test_stock_inventory_source_rejects_paper_history_or_snapshot(env):
with pytest.raises(admission.AdmissionError):
admission._validate_image_only_source(
image_commit=STOCK_INVENTORY_CANDIDATE,
env=env,
project="longbridgequant",
)


def test_stock_inventory_candidate_does_not_admit_arbitrary_sha():
with pytest.raises(admission.AdmissionError, match="not approved"):
admission._validate_image_only_source(
image_commit="b" * 40,
env={"WORKFLOW_TARGET": "HK"},
project="longbridgequant",
)


@pytest.mark.parametrize(
("target_label", "env_overrides", "commit"),
[
("PAPER", {}, STOCK_INVENTORY_CANDIDATE),
("HK", _sghk_history("HK"), STOCK_INVENTORY_CANDIDATE),
("HK", {"ACCOUNT_SNAPSHOT_ENABLED_INPUT": "true"}, STOCK_INVENTORY_CANDIDATE),
("HK", {}, "b" * 40),
],
)
def test_stock_inventory_rejects_unapproved_dispatch_before_cloud_reads(
target_label, env_overrides, commit
):
calls = []
env = {"WORKFLOW_TARGET": target_label, "SOURCE_COMMIT": commit, **env_overrides}
with pytest.raises(admission.AdmissionError):
admission.prepare_image_only_staging(
service="longbridge-quant-hk-service",
project="longbridgequant",
region="asia-east2",
service_json={},
env=env,
image_commit=commit,
run=lambda command: calls.append(command) or "",
)
assert calls == []


@pytest.mark.parametrize(
("target_label", "target_overrides", "project", "region"),
[
("HK", {"account_scope": "SG"}, "longbridgequant", "asia-east2"),
("HK", {"account_selector": "hk"}, "longbridgequant", "asia-east2"),
("SG", {"deployment_selector": "sg"}, "longbridgequant", "asia-southeast1"),
("SG", None, "other-project", "asia-southeast1"),
("HK", None, "longbridgequant", "asia-east1"),
],
)
def test_stock_inventory_candidate_rejects_identity_mismatch(
target_label, target_overrides, project, region
):
with pytest.raises(admission.AdmissionError, match="approved account"):
_prepare_stock_inventory(
target_label,
target_overrides=target_overrides,
project=project,
region=region,
)


def test_stock_inventory_candidate_rejects_template_drift():
with pytest.raises(admission.AdmissionError, match="template does not match"):
_prepare_stock_inventory(
"HK", serving_extra_env=[{"name": "UNEXPECTED", "value": "1"}]
)


def test_stock_inventory_candidate_rejects_serving_ues_mismatch():
calls = []
with pytest.raises(admission.AdmissionError, match="differs from serving"):
_prepare_stock_inventory("SG", serving_pin=OTHER_UES, calls=calls)
assert not any(
command[:3] == ["gcloud", "run", "services"] and "update" in command for command in calls
)


def test_stock_inventory_readback_requires_unchanged_configuration():
service, env = _stock_inventory_service("HK")
plan = _prepare_stock_inventory("HK")
image = "repo@sha256:" + "b" * 64
revision = {
"metadata": {
"name": "longbridge-quant-hk-service-r123",
"labels": {"commit-sha": STOCK_INVENTORY_CANDIDATE},
},
"spec": {
"serviceAccountName": "runtime@example.invalid",
"containers": [{"env": env, "image": image}],
},
"status": {"conditions": [{"type": "Ready", "status": "True"}]},
}

def confirm(service_payload, revision_payload):
def run(command):
if command[:3] == ["gcloud", "run", "services"]:
return json.dumps(service_payload)
return json.dumps(revision_payload)

admission.confirm_image_only_readback(
service="longbridge-quant-hk-service",
project="longbridgequant",
region="asia-east2",
plan=plan,
expected_image=image,
expected_commit=STOCK_INVENTORY_CANDIDATE,
expected_revision="longbridge-quant-hk-service-r123",
run=run,
)

confirm(service, revision)
drifted = json.loads(json.dumps(revision))
drifted["spec"]["containers"][0]["env"].append({"name": "UNEXPECTED", "value": "1"})
with pytest.raises(admission.AdmissionError, match="configuration"):
confirm(service, drifted)
Loading
Loading