diff --git a/.github/workflows/execution-report-heartbeat.yml b/.github/workflows/execution-report-heartbeat.yml index 4538e2c..cbc6ac8 100644 --- a/.github/workflows/execution-report-heartbeat.yml +++ b/.github/workflows/execution-report-heartbeat.yml @@ -60,9 +60,9 @@ on: - cron: "20 22 * * *" env: - GCP_PROJECT_ID: longbridgequant - GCP_WORKLOAD_IDENTITY_PROVIDER: projects/252919773759/locations/global/workloadIdentityPools/github-actions/providers/github-main - GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT: longbridge-platform-deploy@longbridgequant.iam.gserviceaccount.com + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT: ${{ secrets.GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }} concurrency: group: ${{ github.workflow }}-${{ github.ref_name }} @@ -151,6 +151,7 @@ jobs: RUNTIME_HEARTBEAT_EXPECTED_TIMEZONE: ${{ vars.RUNTIME_HEARTBEAT_EXPECTED_TIMEZONE }} RUNTIME_TARGET_ENABLED: ${{ vars.RUNTIME_TARGET_ENABLED }} ACCOUNT_HISTORY_OBSERVATION_MODE: ${{ vars.ACCOUNT_HISTORY_OBSERVATION_MODE || 'scheduler_archive' }} + ACCOUNT_HISTORY_ALLOWED_REF: ${{ format('refs/heads/{0}', github.event.repository.default_branch) }} ACCOUNT_HISTORY_REQUESTED_TARGET: ${{ inputs.target || 'all' }} NOTIFY_LANG: ${{ vars.NOTIFY_LANG || 'zh' }} RUNTIME_TARGET_JSON: ${{ vars.RUNTIME_TARGET_JSON || secrets.RUNTIME_TARGET_JSON }} @@ -180,11 +181,12 @@ jobs: continue-on-error: true env: ACCOUNT_HISTORY_RECORDING_ENABLED: ${{ vars.ACCOUNT_HISTORY_RECORDING_ENABLED }} - ACCOUNT_HISTORY_SERVICE_URL: ${{ vars.ACCOUNT_HISTORY_SERVICE_URL }} + ACCOUNT_HISTORY_SERVICE_URL: ${{ secrets.ACCOUNT_HISTORY_SERVICE_URL }} ACCOUNT_HISTORY_TARGET_ID: ${{ matrix.target.id }} ACCOUNT_HISTORY_EXPECTED_SCOPE: ${{ matrix.target.label }} - ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID: ${{ secrets.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID || vars.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID }} + ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID: ${{ secrets.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID }} GOOGLE_CLOUD_PROJECT: ${{ env.GCP_PROJECT_ID }} + GOOGLE_CLOUD_PROJECT_NUMBER: ${{ secrets.GCP_PROJECT_NUMBER }} SNAPSHOT_SERVICE: ${{ secrets.CLOUD_RUN_SERVICE }} SNAPSHOT_REGION: ${{ vars.CLOUD_RUN_REGION }} run: uv run --no-sync python scripts/record_daily_account_snapshot.py --validate-independent-audience >> "$GITHUB_OUTPUT" @@ -235,16 +237,17 @@ jobs: continue-on-error: true env: ACCOUNT_HISTORY_RECORDING_ENABLED: ${{ vars.ACCOUNT_HISTORY_RECORDING_ENABLED }} - ACCOUNT_HISTORY_SERVICE_URL: ${{ env.ACCOUNT_HISTORY_OBSERVATION_MODE != 'independent_get' && vars.ACCOUNT_HISTORY_SERVICE_URL || steps.independent_audience.outputs.service_url }} + ACCOUNT_HISTORY_SERVICE_URL: ${{ env.ACCOUNT_HISTORY_OBSERVATION_MODE != 'independent_get' && secrets.ACCOUNT_HISTORY_SERVICE_URL || steps.independent_audience.outputs.service_url }} ACCOUNT_HISTORY_GCS_PREFIX: ${{ vars.ACCOUNT_HISTORY_GCS_PREFIX }} ACCOUNT_HISTORY_TARGET_ID: ${{ matrix.target.id }} ACCOUNT_HISTORY_EXPECTED_SCOPE: ${{ matrix.target.label }} - ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID: ${{ secrets.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID || vars.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID }} + ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID: ${{ secrets.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID }} GOOGLE_CLOUD_PROJECT: ${{ env.GCP_PROJECT_ID }} ACCOUNT_FACTS_SYNC_ENABLED: ${{ vars.ACCOUNT_FACTS_SYNC_ENABLED }} ACCOUNT_FACTS_SYNC_URL: ${{ vars.ACCOUNT_FACTS_SYNC_URL }} ACCOUNT_FACTS_SYNC_TOKEN: ${{ vars.ACCOUNT_FACTS_SYNC_ENABLED == 'true' && secrets.ACCOUNT_FACTS_SYNC_TOKEN || '' }} ACCOUNT_HISTORY_INDEPENDENT_ID_TOKEN: ${{ steps.gcp_auth_primary.outputs.id_token || steps.gcp_auth_retry.outputs.id_token }} + GOOGLE_CLOUD_PROJECT_NUMBER: ${{ secrets.GCP_PROJECT_NUMBER }} SNAPSHOT_SERVICE: ${{ secrets.CLOUD_RUN_SERVICE }} SNAPSHOT_REGION: ${{ vars.CLOUD_RUN_REGION }} run: | @@ -329,14 +332,14 @@ jobs: id-token: write environment: ${{ matrix.target.environment }} env: - GCP_PROJECT_ID: longbridgequant - GCP_WORKLOAD_IDENTITY_PROVIDER: projects/252919773759/locations/global/workloadIdentityPools/github-actions/providers/github-main - GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT: longbridge-platform-deploy@longbridgequant.iam.gserviceaccount.com + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT: ${{ secrets.GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }} ACCOUNT_HISTORY_TARGET_ID: ${{ matrix.target.id }} ACCOUNT_HISTORY_EXPECTED_SCOPE: ${{ matrix.target.label }} - ACCOUNT_HISTORY_SERVICE_URL: ${{ vars.ACCOUNT_HISTORY_SERVICE_URL }} + ACCOUNT_HISTORY_SERVICE_URL: ${{ secrets.ACCOUNT_HISTORY_SERVICE_URL }} ACCOUNT_HISTORY_GCS_PREFIX: ${{ vars.ACCOUNT_HISTORY_GCS_PREFIX }} - ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID: ${{ secrets.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID || vars.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID }} + ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID: ${{ secrets.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID }} ACCOUNT_HISTORY_INSPECTION_TRIGGERED_AT: ${{ inputs.snapshot_triggered_at }} ACCOUNT_HISTORY_INSPECTION_COMPLETED_AT: ${{ inputs.snapshot_inspection_completed_at }} steps: diff --git a/docs/read_only_account_observation.zh-CN.md b/docs/read_only_account_observation.zh-CN.md new file mode 100644 index 0000000..370e5d8 --- /dev/null +++ b/docs/read_only_account_observation.zh-CN.md @@ -0,0 +1,33 @@ +# 只读账户观察器的配置与边界 + +独立观察器支持 LongBridge 的 PAPER、HK、SG 协议范围。目标名称可以自定义；账户槽位、策略名称和券商原生账户身份分别处理，策略名称不能代替账户身份。 + +本实现复用普通配置对象、只读券商入口和现有账户事实接收接口。配置解析、服务身份核验、券商响应投影和发送分别由已有函数承担，不增加插件框架或第二套账户注册表。 + +## 配置来源 + +独立观察器从当前受保护任务环境读取部署信息，不从公开运行目标清单推导生产映射。 + +| 参数 | 用途 | +| --- | --- | +| `GOOGLE_CLOUD_PROJECT`、`GOOGLE_CLOUD_PROJECT_NUMBER` | 核对服务所在项目；数字项目号在返回数字 namespace 时必填 | +| `SNAPSHOT_SERVICE`、`SNAPSHOT_REGION` | 定位并核对实际承载流量的服务版本 | +| `ACCOUNT_HISTORY_TARGET_ID`、`ACCOUNT_HISTORY_EXPECTED_SCOPE` | 自定义账户槽位与协议范围 | +| `ACCOUNT_HISTORY_SERVICE_URL` | 精确服务地址与 ID token audience | +| `ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID` | 核对该账户的预期来源绑定 | +| `ACCOUNT_HISTORY_ALLOWED_REF` | 允许执行的分支；工作流使用仓库默认分支 | +| `ACCOUNT_FACTS_SYNC_URL`、`ACCOUNT_FACTS_SYNC_TOKEN` | 既有账户事实接收接口及专用同步凭据 | + +工作流的 `GCP_PROJECT_ID`、`GCP_PROJECT_NUMBER`、`GCP_WORKLOAD_IDENTITY_PROVIDER` 和 `GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT` 使用 GitHub Secrets。每个环境的 `ACCOUNT_HISTORY_SERVICE_URL` 和 `ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID` 同样只从 Secrets 读取。真实地址、来源绑定及可关联用户的映射应放在受保护配置中；源码、文档、fixture 和公开日志只用不可关联真实部署的示例。部署参数不得在应用代码中设生产默认值。 + +`GITHUB_REPOSITORY`、`GITHUB_REF`、`GITHUB_WORKFLOW_REF` 使用 GitHub 提供的执行上下文。保护环境、云端 WIF 条件及精确来源绑定共同限制调用者；仓库名称不写死在 Python 中。 + +## 访问与数据处理 + +观察器先核对服务、项目、区域、地址及实际承载 100% 流量版本的只读开关，再发送 audience token。失败的新版本不能代替现役版本，也不能阻断一个已核准的健康现役版本。 + +`independent_get` 适用于可由当前执行器访问的已鉴权 Cloud Run 服务。内部 ingress 返回固定原因 `internal_observer_required`，在调用券商或发送账户事实前停止；不得为采集开放 ingress。内部服务应复用受支持的内部执行入口，另行验证只读采集、私有归档与接收闭环。此文档不表示该内部闭环已经完成。 + +单次操作最多一次快照 GET 和一次事实 POST，不自动重试未知结果。投影保留券商实际币种、金额、观察时间及来源；账户来源不匹配时拒收，不补零、不改旧数据时间。交易启停、风险预算和通知不由此观察器控制。 + +本次配置迁移只覆盖独立观察器与该心跳工作流。历史 Scheduler 观察器及其他部署流程尚有旧配置约定，不能据此宣称全仓已完成配置与隐私整改。 diff --git a/scripts/record_daily_account_snapshot.py b/scripts/record_daily_account_snapshot.py index 6ffb283..c32ef3d 100644 --- a/scripts/record_daily_account_snapshot.py +++ b/scripts/record_daily_account_snapshot.py @@ -300,24 +300,35 @@ class _IndependentConfig: source_binding_id: str target_id: str account_scope: str + project_number: str = "" def _independent_config(env: Mapping[str, str], *, require_id_token: bool = True) -> _IndependentConfig: """Read one configured target without depending on its trading activation.""" - target_id = env.get("ACCOUNT_HISTORY_TARGET_ID") - expected = _EXPECTED_TARGETS.get(target_id) + target_id = str(env.get("ACCOUNT_HISTORY_TARGET_ID") or "") + scope = str(env.get("ACCOUNT_HISTORY_EXPECTED_SCOPE") or "") + project = str(env.get("GOOGLE_CLOUD_PROJECT") or "") + project_number = str(env.get("GOOGLE_CLOUD_PROJECT_NUMBER") or "") + service = str(env.get("SNAPSHOT_SERVICE") or "") + region = str(env.get("SNAPSHOT_REGION") or "") + repository = str(env.get("GITHUB_REPOSITORY") or "") + allowed_ref = str(env.get("ACCOUNT_HISTORY_ALLOWED_REF") or "refs/heads/main") if ( env.get("ACCOUNT_HISTORY_OBSERVATION_MODE") != "independent_get" or env.get("ACCOUNT_HISTORY_RECORDING_ENABLED") != "true" - or expected is None - or env.get("ACCOUNT_HISTORY_EXPECTED_SCOPE") != expected[0] - or env.get("GOOGLE_CLOUD_PROJECT") != "longbridgequant" + or not _TARGET_ID.fullmatch(target_id) + or scope not in {"PAPER", "HK", "SG"} + or not _PROJECT_ID.fullmatch(project) + or (project_number and not re.fullmatch(r"[1-9]\d{5,19}", project_number)) + or not re.fullmatch(r"[a-z][a-z0-9-]{0,62}", service) + or not re.fullmatch(r"[a-z]+-[a-z]+\d+", region) + or not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repository) + or not allowed_ref.startswith("refs/heads/") or any(c.isspace() for c in allowed_ref) or env.get("GITHUB_EVENT_NAME") not in {"workflow_dispatch", "schedule"} or env.get("ACCOUNT_HISTORY_REQUESTED_TARGET") not in {target_id, "all"} - or env.get("GITHUB_REF") != "refs/heads/main" + or env.get("GITHUB_REF") != allowed_ref or env.get("GITHUB_WORKFLOW_REF") != ( - "QuantStrategyLab/LongBridgePlatform/.github/workflows/" - "execution-report-heartbeat.yml@refs/heads/main" + f"{repository}/.github/workflows/execution-report-heartbeat.yml@{allowed_ref}" ) ): raise _Rejected("independent_config_invalid") @@ -327,22 +338,13 @@ def _independent_config(env: Mapping[str, str], *, require_id_token: bool = True require_id_token and (not token or len(token) > 16384 or any(c.isspace() for c in token)) ): raise _Rejected("independent_config_invalid") - from application.runtime_target_manifest import load_runtime_target_manifest - - targets = [target for target in load_runtime_target_manifest().targets if target.id == target_id] - if len(targets) != 1 or targets[0].mode != expected[1] or targets[0].account_scope != expected[0]: - raise _Rejected("independent_config_invalid") - target = targets[0] - if env.get("SNAPSHOT_SERVICE") != target.service or env.get("SNAPSHOT_REGION") != target.region: - raise _Rejected("independent_config_invalid") raw_url = env.get("ACCOUNT_HISTORY_SERVICE_URL") - service_url = _service_url(str(raw_url or ""), service=target.service, region=target.region) + service_url = _service_url(str(raw_url or ""), service=service, region=region) # Unlike the legacy Scheduler route, this exact string is also the token audience. if raw_url != service_url: raise _Rejected("independent_config_invalid") return _IndependentConfig( - "longbridgequant", target.region, target.service, - service_url, binding, target_id, expected[0], + project, region, service, service_url, binding, target_id, scope, project_number, ) @@ -360,10 +362,8 @@ def _validate_independent_service(value: object, config: _IndependentConfig) -> revision_name = revision["metadata"]["name"] if ( metadata["name"] != config.service - or str(metadata["namespace"]) not in {config.project_id, "252919773759"} + or str(metadata["namespace"]) not in {v for v in (config.project_id, config.project_number) if v} or metadata["labels"]["cloud.googleapis.com/location"] != config.region - or annotations.get("run.googleapis.com/ingress") != "all" - or annotations.get("run.googleapis.com/ingress-status", "all") != "all" or status["url"] != config.service_url or len(ready) != 1 or ready[0].get("status") != "True" or not isinstance(revision_name, str) or not revision_name.startswith(config.service + "-") @@ -380,6 +380,14 @@ def _validate_independent_service(value: object, config: _IndependentConfig) -> ] if len(snapshot_flags) != 1 or snapshot_flags[0].get("value") != "true" or "valueFrom" in snapshot_flags[0]: raise ValueError + ingress = annotations.get("run.googleapis.com/ingress") + ingress_status = annotations.get("run.googleapis.com/ingress-status", ingress) + if ingress in {"internal", "internal-and-cloud-load-balancing"} or ingress_status in { + "internal", "internal-and-cloud-load-balancing", + }: + raise _Rejected("internal_observer_required") + if ingress != "all" or ingress_status != "all": + raise ValueError except (KeyError, TypeError, AttributeError, ValueError): raise _Rejected("independent_service_unqualified") from None diff --git a/tests/test_daily_account_snapshot.py b/tests/test_daily_account_snapshot.py index bf10ff8..9c0fb1c 100644 --- a/tests/test_daily_account_snapshot.py +++ b/tests/test_daily_account_snapshot.py @@ -1622,14 +1622,18 @@ def test_history_validator_accepts_optional_financing_and_rejects_malformed(): def _independent_env(**overrides): value = _env( "hk", + GOOGLE_CLOUD_PROJECT="example-project", + GOOGLE_CLOUD_PROJECT_NUMBER="111222333444", + GITHUB_REPOSITORY="example-owner/example-platform", + ACCOUNT_HISTORY_SERVICE_URL="https://example-hk-reader-abc123-uc.a.run.app", RUNTIME_TARGET_ENABLED="false", ACCOUNT_HISTORY_OBSERVATION_MODE="independent_get", - SNAPSHOT_SERVICE="longbridge-quant-hk-service", + SNAPSHOT_SERVICE="example-hk-reader", SNAPSHOT_REGION="asia-east2", ACCOUNT_HISTORY_REQUESTED_TARGET="hk", GITHUB_EVENT_NAME="workflow_dispatch", GITHUB_REF="refs/heads/main", - GITHUB_WORKFLOW_REF="QuantStrategyLab/LongBridgePlatform/.github/workflows/execution-report-heartbeat.yml@refs/heads/main", + GITHUB_WORKFLOW_REF="example-owner/example-platform/.github/workflows/execution-report-heartbeat.yml@refs/heads/main", ACCOUNT_HISTORY_INDEPENDENT_ID_TOKEN="synthetic-cloud-run-id-token", ACCOUNT_FACTS_SYNC_ENABLED="true", ACCOUNT_FACTS_SYNC_URL=QRS_URL, @@ -1643,22 +1647,22 @@ def _independent_env(**overrides): def _service_metadata(): value = { "metadata": { - "name": "longbridge-quant-hk-service", - "namespace": "252919773759", + "name": "example-hk-reader", + "namespace": "111222333444", "generation": 7, "labels": {"cloud.googleapis.com/location": "asia-east2"}, "annotations": {"run.googleapis.com/ingress": "all"}, }, - "spec": {"template": {"metadata": {"name": "longbridge-quant-hk-service-00001"}, "spec": {"containers": [{ + "spec": {"template": {"metadata": {"name": "example-hk-reader-00001"}, "spec": {"containers": [{ "env": [{"name": "LONGBRIDGE_ACCOUNT_SNAPSHOT_ENABLED", "value": "true"}], }]}}}, "status": { "url": _independent_env()["ACCOUNT_HISTORY_SERVICE_URL"], - "latestReadyRevisionName": "longbridge-quant-hk-service-00001", - "latestCreatedRevisionName": "longbridge-quant-hk-service-00001", + "latestReadyRevisionName": "example-hk-reader-00001", + "latestCreatedRevisionName": "example-hk-reader-00001", "observedGeneration": 7, "conditions": [{"type": "Ready", "status": "True"}], - "traffic": [{"revisionName": "longbridge-quant-hk-service-00001", "percent": 100}], + "traffic": [{"revisionName": "example-hk-reader-00001", "percent": 100}], }, } @@ -1772,10 +1776,10 @@ def test_independent_hk_preserves_native_currency_and_only_publishes_existing_hi {"ACCOUNT_HISTORY_OBSERVATION_MODE": ""}, {"ACCOUNT_HISTORY_OBSERVATION_MODE": "scheduler_archive"}, {"ACCOUNT_HISTORY_OBSERVATION_MODE": "invalid"}, - {"ACCOUNT_HISTORY_TARGET_ID": "sg"}, - {"ACCOUNT_HISTORY_EXPECTED_SCOPE": "SG"}, + {"ACCOUNT_HISTORY_TARGET_ID": ""}, + {"ACCOUNT_HISTORY_EXPECTED_SCOPE": "invalid"}, {"ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID": ""}, - {"GOOGLE_CLOUD_PROJECT": "other-project"}, + {"GOOGLE_CLOUD_PROJECT": ""}, {"GITHUB_REF": "refs/heads/candidate"}, {"GITHUB_WORKFLOW_REF": "other/workflow@refs/heads/main"}, {"ACCOUNT_HISTORY_INDEPENDENT_ID_TOKEN": ""}, @@ -1797,7 +1801,7 @@ def test_independent_invalid_config_has_no_get_or_publish(overrides): lambda v: v["metadata"]["annotations"].update({"run.googleapis.com/ingress-status": "internal"}), lambda v: v["status"].update(url="https://different.run.app"), lambda v: v["status"].update(traffic=[{"revisionName": "old-revision", "percent": 100}]), - lambda v: v["status"].update(traffic=[{"revisionName": "longbridge-quant-hk-service-00001", "percent": 50}]), + lambda v: v["status"].update(traffic=[{"revisionName": "example-hk-reader-00001", "percent": 50}]), lambda v: v["serving_revision"]["status"].update(conditions=[{"type": "Ready", "status": "False"}]), lambda v: v["serving_revision"]["spec"]["containers"][0].update(env=[]), lambda v: v["serving_revision"]["spec"]["containers"][0]["env"][0].update(value="false"), @@ -2092,7 +2096,7 @@ def test_independent_workflow_validates_one_audience_before_auth_and_reuses_it() assert workflow.count(audience) == 2 assert "token_format: \u0024{{ steps.independent_audience.outcome == 'success' && 'id_token' || '' }}" in workflow record_step = workflow[workflow.index(" - name: Record and sync daily account snapshot"):workflow.index(" - name: Check recent execution report")] - assert "ACCOUNT_HISTORY_SERVICE_URL: \u0024{{ env.ACCOUNT_HISTORY_OBSERVATION_MODE != 'independent_get' && vars.ACCOUNT_HISTORY_SERVICE_URL || steps.independent_audience.outputs.service_url }}" in record_step + assert "ACCOUNT_HISTORY_SERVICE_URL: \u0024{{ env.ACCOUNT_HISTORY_OBSERVATION_MODE != 'independent_get' && secrets.ACCOUNT_HISTORY_SERVICE_URL || steps.independent_audience.outputs.service_url }}" in record_step def test_scheduled_read_only_snapshot_does_not_depend_on_strategy_activation(): @@ -2110,9 +2114,9 @@ def test_failed_candidate_does_not_block_healthy_serving_snapshot(): "spec": metadata["spec"]["template"]["spec"], "status": {"conditions": [{"type": "Ready", "status": "True"}]}, } - metadata["status"]["latestCreatedRevisionName"] = "longbridge-quant-hk-service-failed-candidate" + metadata["status"]["latestCreatedRevisionName"] = "example-hk-reader-failed-candidate" metadata["status"]["conditions"] = [{"type": "Ready", "status": "False"}] - metadata["spec"]["template"] = {"metadata": {"name": "longbridge-quant-hk-service-failed-candidate"}, "spec": {"containers": []}} + metadata["spec"]["template"] = {"metadata": {"name": "example-hk-reader-failed-candidate"}, "spec": {"containers": []}} result, calls, _ = _observe_independent(metadata=metadata) assert result.status == "observed" assert len(calls["gets"]) == 1 @@ -2122,11 +2126,12 @@ def test_failed_candidate_does_not_block_healthy_serving_snapshot(): @pytest.mark.parametrize("event", ["workflow_dispatch", "schedule"]) @pytest.mark.parametrize("trading_enabled", ["true", "false"]) def test_independent_snapshot_targets_keep_scope_and_ignore_trading_activation(target_id, event, trading_enabled): - from application.runtime_target_manifest import load_runtime_target_manifest - target = next(t for t in load_runtime_target_manifest().targets if t.id == target_id) - env = _independent_env(**{**_env(target_id), "SNAPSHOT_SERVICE": target.service, "SNAPSHOT_REGION": target.region, - "ACCOUNT_HISTORY_REQUESTED_TARGET": target_id if event == "workflow_dispatch" else "all", - "GITHUB_EVENT_NAME": event, "RUNTIME_TARGET_ENABLED": trading_enabled}) + scope = {"paper": "PAPER", "hk": "HK", "sg": "SG"}[target_id] + target = SimpleNamespace(service=f"example-{target_id}-reader", region="us-central1", account_scope=scope) + env = _independent_env(SNAPSHOT_SERVICE=target.service, SNAPSHOT_REGION=target.region, + ACCOUNT_HISTORY_TARGET_ID=target_id, ACCOUNT_HISTORY_EXPECTED_SCOPE=scope, + ACCOUNT_HISTORY_REQUESTED_TARGET=target_id if event == "workflow_dispatch" else "all", + GITHUB_EVENT_NAME=event, RUNTIME_TARGET_ENABLED=trading_enabled) metadata = _service_metadata() metadata["metadata"]["name"] = target.service metadata["metadata"]["labels"]["cloud.googleapis.com/location"] = target.region @@ -2145,7 +2150,7 @@ def test_independent_snapshot_targets_keep_scope_and_ignore_trading_activation(t def test_independent_cli_reads_only_the_actual_serving_revision(monkeypatch, capsys): metadata = _service_metadata() serving = metadata.pop("serving_revision") - metadata["status"]["latestCreatedRevisionName"] = "longbridge-quant-hk-service-failed-candidate" + metadata["status"]["latestCreatedRevisionName"] = "example-hk-reader-failed-candidate" calls = [] monkeypatch.setattr(snapshots.sys, "stdin", SimpleNamespace(buffer=io.BytesIO(json.dumps(metadata).encode()))) def run(command, **kwargs): @@ -2158,7 +2163,7 @@ def run(command, **kwargs): assert len(calls) == 1 command, kwargs = calls[0] assert command[:5] == ["gcloud", "run", "revisions", "describe", serving["metadata"]["name"]] - assert command[command.index("--project") + 1] == "longbridgequant" + assert command[command.index("--project") + 1] == "example-project" assert command[command.index("--region") + 1] == "asia-east2" assert kwargs["stderr"] == snapshots.subprocess.DEVNULL assert kwargs["timeout"] == snapshots.HTTP_TIMEOUT_SECONDS @@ -2188,3 +2193,65 @@ def test_manual_snapshot_only_skips_heartbeat_notifications_and_runtime_publishe "Publish daily runtime projection", "Publish existing reports as unified digest candidates"]: block = workflow[workflow.index(" - name: " + step):].split("\n - name:", 1)[0] assert "!inputs.snapshot_only" in block.split("\n", 2)[1] + + +@pytest.mark.parametrize("namespace_kind", ["project_id", "project_number"]) +def test_independent_observer_can_use_another_private_deployment_without_manifest(monkeypatch, namespace_kind): + from application import runtime_target_manifest + + monkeypatch.setattr(runtime_target_manifest, "load_runtime_target_manifest", lambda: pytest.fail("private deployment must not use public mappings")) + env = _independent_env( + GOOGLE_CLOUD_PROJECT="another-example-project", + GOOGLE_CLOUD_PROJECT_NUMBER="999888777666", + SNAPSHOT_SERVICE="custom-readonly-service", + SNAPSHOT_REGION="europe-west1", + ACCOUNT_HISTORY_SERVICE_URL="https://custom-readonly-service-example-ew.a.run.app", + ACCOUNT_HISTORY_TARGET_ID="custom-account-slot", + ACCOUNT_HISTORY_REQUESTED_TARGET="custom-account-slot", + ACCOUNT_HISTORY_EXPECTED_SCOPE="SG", + ACCOUNT_HISTORY_ALLOWED_REF="refs/heads/release", + GITHUB_REF="refs/heads/release", + GITHUB_REPOSITORY="another-owner/another-platform", + GITHUB_WORKFLOW_REF="another-owner/another-platform/.github/workflows/execution-report-heartbeat.yml@refs/heads/release", + ) + metadata = _service_metadata() + namespace = env["GOOGLE_CLOUD_PROJECT" if namespace_kind == "project_id" else "GOOGLE_CLOUD_PROJECT_NUMBER"] + revision = env["SNAPSHOT_SERVICE"] + "-00001" + metadata["metadata"].update(name=env["SNAPSHOT_SERVICE"], namespace=namespace) + metadata["metadata"]["labels"]["cloud.googleapis.com/location"] = env["SNAPSHOT_REGION"] + metadata["status"].update(url=env["ACCOUNT_HISTORY_SERVICE_URL"], traffic=[{"revisionName": revision, "percent": 100}]) + metadata["serving_revision"]["metadata"].update(name=revision, namespace=namespace) + result, calls, _ = _observe_independent(env, metadata=metadata, snapshot=_independent_snapshot(account_scope="SG")) + assert result.status == "observed" and result.publish_status == "published" + assert calls["gets"][0][0] == env["ACCOUNT_HISTORY_SERVICE_URL"] + "/account-snapshot" + published = json.loads(calls["posts"][0][1]["data"]) + assert published["target_id"] == "custom-account-slot" and published["account_scope"] == "SG" + assert published["source_binding"]["id"] == BINDING + assert published["observed_finished_at"] == _independent_snapshot()["observed_finished_at"] + + +@pytest.mark.parametrize("ingress", ["internal", "internal-and-cloud-load-balancing"]) +def test_private_ingress_requires_internal_observer_without_broker_request(ingress): + metadata = _service_metadata() + metadata["metadata"]["annotations"].update({"run.googleapis.com/ingress": ingress, "run.googleapis.com/ingress-status": ingress}) + result, calls, _ = _observe_independent(metadata=metadata) + assert result.category == "internal_observer_required" + assert calls == {"gets": [], "posts": []} + + +@pytest.mark.parametrize("overrides", [ + {"SNAPSHOT_SERVICE": ""}, {"SNAPSHOT_REGION": ""}, + {"GOOGLE_CLOUD_PROJECT_NUMBER": "not-a-number"}, {"GITHUB_REPOSITORY": ""}, + {"GITHUB_REPOSITORY": "another-owner/another-platform"}, +]) +def test_missing_or_cross_repository_private_settings_do_not_send_tokens(overrides): + result, calls, _ = _observe_independent(_independent_env(**overrides)) + assert result.status == "error" + assert calls == {"gets": [], "posts": []} + + +def test_heartbeat_cloud_identity_comes_from_protected_configuration(): + workflow = (ROOT / ".github/workflows/execution-report-heartbeat.yml").read_text() + for setting in ["GCP_PROJECT_ID", "GCP_WORKLOAD_IDENTITY_PROVIDER", "GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT"]: + assert workflow.count(f"{setting}: ${{{{ secrets.{setting} }}}}") == 2 + assert "GOOGLE_CLOUD_PROJECT_NUMBER: ${{ secrets.GCP_PROJECT_NUMBER }}" in workflow diff --git a/tests/test_execution_report_heartbeat.py b/tests/test_execution_report_heartbeat.py index 2d11fe9..7d7ef16 100644 --- a/tests/test_execution_report_heartbeat.py +++ b/tests/test_execution_report_heartbeat.py @@ -17,9 +17,11 @@ from scripts import execution_report_heartbeat as heartbeat # noqa: E402 -def test_account_history_workflow_prefers_and_masks_protected_source_binding(): +def test_account_history_workflow_only_reads_and_masks_protected_source_binding(): workflow = WORKFLOW.read_text(encoding="utf-8") - assert "${{ secrets.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID || vars.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID }}" in workflow + assert "vars.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID" not in workflow + assert "vars.ACCOUNT_HISTORY_SERVICE_URL" not in workflow + assert "${{ secrets.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID }}" in workflow assert "printf '::add-mask::%s\\n' \"$ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID\"" in workflow record_step = workflow[workflow.index("name: Record and sync daily account snapshot") :] assert record_step.index("printf '::add-mask::%s\\n'") < record_step.index( @@ -65,7 +67,7 @@ def test_archive_inspection_is_manual_target_limited_and_skips_heartbeat_job(): assert "environment: ${{ matrix.target.environment }}" in job assert "ACCOUNT_HISTORY_TARGET_ID: ${{ matrix.target.id }}" in job assert "ACCOUNT_HISTORY_EXPECTED_SCOPE: ${{ matrix.target.label }}" in job - assert "ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID: ${{ secrets.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID || vars.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID }}" in job + assert "ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID: ${{ secrets.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID }}" in job assert "--inspect-archived-account-snapshot --target \"$ACCOUNT_HISTORY_TARGET_ID\" --triggered-at \"$ACCOUNT_HISTORY_INSPECTION_TRIGGERED_AT\"" in job assert "--completed-at \"$ACCOUNT_HISTORY_INSPECTION_COMPLETED_AT\"" in job assert "cloudscheduler" not in job.lower() diff --git a/tests/test_runtime_monitor_workflows.py b/tests/test_runtime_monitor_workflows.py index 5b4ee31..74a9fd0 100644 --- a/tests/test_runtime_monitor_workflows.py +++ b/tests/test_runtime_monitor_workflows.py @@ -171,8 +171,7 @@ def test_account_snapshot_sync_uses_matrix_identity_and_preserves_heartbeat_fail assert "ACCOUNT_HISTORY_TARGET_ID: ${{ matrix.target.id }}" in workflow assert "ACCOUNT_HISTORY_EXPECTED_SCOPE: ${{ matrix.target.label }}" in workflow assert ( - "ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID: ${{ secrets.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID || " - "vars.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID }}" + "ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID: ${{ secrets.ACCOUNT_HISTORY_EXPECTED_SOURCE_BINDING_ID }}" ) in workflow assert "matrix.target.service" not in workflow assert "matrix.target.region" not in workflow