From 8a6e10d0ef5508100b9979b6caad21461f61dd1a Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Sun, 30 Aug 2026 05:22:43 -0700 Subject: [PATCH 01/39] feat: require signed production e2e evidence --- CHANGELOG.md | 7 + artifacts/p1_acceptance_report.json | 27 +- contracts/schemas/certification-run.v1.json | 21 +- deploy/deploy_forge.sh | 2 +- docs/OPERATIONS.md | 8 + docs/PROJECT_CONTRACT.md | 16 + policies/mandatory-rules.v1.json | 8 + policies/mandatory-rules.v2.json | 8 + pyproject.toml | 2 +- scripts/certification_journey.py | 2 +- scripts/p1_acceptance.py | 29 +- src/echo_certification_forge/__init__.py | 2 +- src/echo_certification_forge/executor.py | 40 +- src/echo_certification_forge/intake.py | 66 +++- src/echo_certification_forge/models.py | 6 + .../production_e2e.py | 351 ++++++++++++++++++ src/echo_certification_forge/run_worker.py | 45 ++- src/echo_certification_forge/verdict.py | 45 +++ tests/conftest.py | 5 +- tests/production_e2e_support.py | 46 +++ tests/test_additional_contracts.py | 9 +- tests/test_evidence_and_verdict.py | 58 ++- tests/test_p5_executor_adapter_gate.py | 5 +- tests/test_p6_deployment_enforcement.py | 6 +- tests/test_p6_platform_integration.py | 32 +- tests/test_p7_subscriber_governance.py | 3 + tests/test_production_e2e.py | 214 +++++++++++ tests/test_runner.py | 7 +- tests/test_t4_live_run.py | 4 +- tests/test_t4p5_executor.py | 6 +- tests/test_t4p7_e2e.py | 2 + tests/test_t4p8_sandbox.py | 2 + 32 files changed, 1027 insertions(+), 57 deletions(-) create mode 100644 src/echo_certification_forge/production_e2e.py create mode 100644 tests/production_e2e_support.py create mode 100644 tests/test_production_e2e.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a8b23b..4d2f4c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.2.0] - 2026-08-30 + +### Added + +- A pinned-key-signed production E2E gate required before any `PRODUCTION_READY` verdict. +- Exact E2E bindings for source, deployment, target, environment, accounts, and clients. + All notable changes to Echo Certification Forge are documented here. Versions follow Semantic Versioning; dates use ISO 8601. diff --git a/artifacts/p1_acceptance_report.json b/artifacts/p1_acceptance_report.json index 771a206..1e54769 100644 --- a/artifacts/p1_acceptance_report.json +++ b/artifacts/p1_acceptance_report.json @@ -1,10 +1,10 @@ { - "manifest_digest": "0ca325c3e6d39b7f22cdd3d01068c8f99dcd976c4422200b5b4d4cfe452f29f7", + "manifest_digest": "9e9f963377374e827ccc9c8b07ea82cc7966e25a2392838997d3c1371f5a4e5c", "manifest_id": "certforge.release-strict.v1", "passed": true, "phase": "P1", - "platform": "Windows-11-10.0.22631-SP0", - "python": "3.13.0 (tags/v3.13.0:60403a5, Oct 7 2024, 09:38:07) [MSC v.1941 64 bit (AMD64)]", + "platform": "Windows-11-10.0.26200-SP0", + "python": "3.12.10 (tags/v3.12.10:0cc8128, Apr 8 2025, 12:21:36) [MSC v.1943 64 bit (AMD64)]", "scenarios": { "default_block": { "passed": true, @@ -17,27 +17,30 @@ "mandatory_rule_missing:environment_identity", "mandatory_rule_missing:evidence_integrity", "mandatory_rule_missing:immutable_target_identity", + "mandatory_rule_missing:production_e2e", "mandatory_rule_missing:runner_control_channel", "mandatory_rule_missing:signing_authority_separation", "mandatory_rule_missing:tenant_isolation", + "production_e2e_attestation_missing", "run_outcome_inconclusive" ], "verdict": "NOT_READY" }, + "source_only_blocks": { + "deploy_gate_allowed": false, + "evidence_merkle_root": "41a4fc282a478767dc16f6c64128c93ab2535866e620a3f5b0ae922220ffb449", + "passed": true, + "payload_sha256": "a7255f5bae53fcd49b1e049b3edd1feb6c9b03f4520b0e2abb0e44f0da7138e7", + "signing_key_id": "ed25519:e222f1c31e85f1b8a6d8cfed30fd2300", + "verdict": "NOT_READY" + }, "tamper_blocks": { "deploy_gate_allowed": false, "passed": true, "reasons": [ - "current_evidence_invalid" + "current_evidence_invalid", + "verdict_not_production_ready" ] - }, - "verified_ready": { - "deploy_gate_allowed": true, - "evidence_merkle_root": "438efda29a515669d0db0472f6527c201c55d0d80afebf2963123d11c77a7d73", - "passed": true, - "payload_sha256": "db3fe179c7c950a9ffaf90cfef6d475fc915af87f0d1b10a85071c7d8411b975", - "signing_key_id": "ed25519:4592da038060823a957506d477e49114", - "verdict": "PRODUCTION_READY" } }, "schema_version": "1.0.0" diff --git a/contracts/schemas/certification-run.v1.json b/contracts/schemas/certification-run.v1.json index 3e4f4b0..4797ffd 100644 --- a/contracts/schemas/certification-run.v1.json +++ b/contracts/schemas/certification-run.v1.json @@ -3,7 +3,7 @@ "$id": "https://cert.echoforge.com/contracts/certification-run.v1.json", "title": "CertificationRunV1", "type": "object", - "required": ["run_id", "tenant_id", "state", "run_outcome", "release_verdict", "target_identity_digest", "environment_identity_digest", "policy_version"], + "required": ["run_id", "tenant_id", "state", "run_outcome", "release_verdict", "target_identity_digest", "environment_identity_digest", "policy_version", "production_e2e"], "properties": { "run_id": {"type": "string", "minLength": 1, "maxLength": 128}, "tenant_id": {"type": "string", "minLength": 1, "maxLength": 128}, @@ -14,6 +14,25 @@ "environment_identity_digest": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, "policy_version": {"type": "string", "minLength": 1, "maxLength": 128}, "evidence_merkle_root": {"type": ["string", "null"], "pattern": "^[0-9a-f]{64}$"}, + "production_e2e": { + "type": "object", + "required": ["verified", "profile", "attestation_id", "source_commit", "deployment_sha", "target_identity_digest", "environment_identity_digest", "canonical_target", "tool_count", "signature_verified", "collector_key_id", "attestation_envelope_sha256"], + "properties": { + "verified": {"type": "boolean"}, + "profile": {"type": ["string", "null"]}, + "attestation_id": {"type": ["string", "null"]}, + "source_commit": {"type": ["string", "null"], "pattern": "^[0-9a-f]{40}(?:[0-9a-f]{24})?$"}, + "deployment_sha": {"type": ["string", "null"], "pattern": "^[0-9a-f]{40}(?:[0-9a-f]{24})?$"}, + "target_identity_digest": {"type": ["string", "null"], "pattern": "^[0-9a-f]{64}$"}, + "environment_identity_digest": {"type": ["string", "null"], "pattern": "^[0-9a-f]{64}$"}, + "canonical_target": {"type": ["string", "null"]}, + "tool_count": {"type": ["integer", "null"], "minimum": 0}, + "signature_verified": {"type": "boolean"}, + "collector_key_id": {"type": ["string", "null"], "pattern": "^ed25519:[0-9a-f]{32}$"}, + "attestation_envelope_sha256": {"type": ["string", "null"], "pattern": "^[0-9a-f]{64}$"} + }, + "additionalProperties": false + }, "created_at": {"type": "string", "format": "date-time"}, "updated_at": {"type": "string", "format": "date-time"} }, diff --git a/deploy/deploy_forge.sh b/deploy/deploy_forge.sh index c280297..2e649b5 100644 --- a/deploy/deploy_forge.sh +++ b/deploy/deploy_forge.sh @@ -16,7 +16,7 @@ CURRENT_LINK="${CERTFORGE_CURRENT_LINK:-/home/forge/echo-certification-forge-cur STATE_ROOT="${CERTFORGE_STATE_ROOT:-/home/forge/echo-certification-forge/var}" ADAPTER_DIR="${ECHO_CERTFORGE_PROD_ADAPTER_DIR:-$STATE_ROOT/p5}" ADAPTER_MODE="${CERTFORGE_ADAPTER_MODE:-required}" -TRUSTED_MANIFEST_SHA256="${ECHO_CERTFORGE_TRUSTED_MANIFEST_SHA256:-7dc98e0e95e6dd2c000ec069a8c46c4d1d49a4fe869ad4eae25e059d103644f4}" +TRUSTED_MANIFEST_SHA256="${ECHO_CERTFORGE_TRUSTED_MANIFEST_SHA256:-965106b00917268d556b325719f26f5096e6c3746551658ffecb9fd4a95ec342}" UNIT_PATH="/etc/systemd/system/$SERVICE.service" DISPATCH_UNIT_PATH="/etc/systemd/system/$DISPATCH_SERVICE.service" RELEASE_DROPIN="/etc/systemd/system/$SERVICE.service.d/10-release.conf" diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 66e4b6f..7e47fee 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -10,6 +10,14 @@ ## Verification sequence +Before a worker may issue `PRODUCTION_READY`, mount a collector envelope through +`--production-e2e-attestation` (or +`ECHO_CERTFORGE_PRODUCTION_E2E_ATTESTATION`) and a directory containing only the +pinned collector public keys through `--trusted-production-e2e-keys` (or +`ECHO_CERTFORGE_TRUSTED_PRODUCTION_E2E_KEYS`). The private collector key must not +be present on the worker. Absence, signature failure, expiration, or any exact +identity/E2E mismatch is a normal `NOT_READY` result, never a bypass. + 1. Confirm hosted CI succeeded for the exact source SHA. 2. Execute the declared production-shaped journey with the pinned runner image. 3. Verify evidence custody, receipt chains, signatures, expiry, and revocation state. diff --git a/docs/PROJECT_CONTRACT.md b/docs/PROJECT_CONTRACT.md index eb191dd..e452ded 100644 --- a/docs/PROJECT_CONTRACT.md +++ b/docs/PROJECT_CONTRACT.md @@ -41,3 +41,19 @@ GS343 may propose discovery, classification, and bounded harness repairs. Determ ## Completion rule Implementation, source validation, contracts, real runtime, failure and recovery, evidence package, scoped Git record, and Commander acceptance are distinct gates. A registered capability is not presumed visible, healthy, schema-complete, authorized, or runnable. + +## Production E2E verdict gate + +`PRODUCTION_READY` requires a current Ed25519-signed `certforge.production-e2e.v1` +attestation from an independently pinned collector. The attestation must bind the +exact target and environment identities, exact deployed source revision, real +critical journeys, negative controls, repeated stability probes, and external +acceptance. Source checks, local journeys, HTTP 200, unsigned status fields, and +target-authored evidence cannot satisfy this gate. Missing, expired, untrusted, +partially passing, or identity-mismatched attestations remain `NOT_READY`. + +For Echo GitHub Autonomy the target-specific profile additionally requires the +canonical MCP and OAuth surface, 30-tool schema, repeated discovery plus +invocation without registry loss, reconciled public/private inventory and +read/write/certify authority for all four exact GitHub account IDs, and matching +private-repository fingerprints from ChatGPT, Claude, Codex, and Grok. diff --git a/policies/mandatory-rules.v1.json b/policies/mandatory-rules.v1.json index ae17648..25e8723 100644 --- a/policies/mandatory-rules.v1.json +++ b/policies/mandatory-rules.v1.json @@ -83,6 +83,14 @@ "conditional_allowed": false, "minimum_evidence": 1, "description": "Deployment authorization checks the exact certified digest, policy, environment, and lifecycle." + }, + { + "id": "production_e2e", + "severity": "BLOCKER", + "mandatory": true, + "conditional_allowed": false, + "minimum_evidence": 1, + "description": "A trusted independent collector proves the exact deployed identity, real critical journeys, negative controls, stability, and external client acceptance end to end." } ] } diff --git a/policies/mandatory-rules.v2.json b/policies/mandatory-rules.v2.json index 45c4ee9..30a31dc 100644 --- a/policies/mandatory-rules.v2.json +++ b/policies/mandatory-rules.v2.json @@ -91,6 +91,14 @@ "conditional_allowed": false, "minimum_evidence": 1, "description": "Deployment authorization checks the exact certified digest, policy, environment, and lifecycle." + }, + { + "id": "production_e2e", + "severity": "BLOCKER", + "mandatory": true, + "conditional_allowed": false, + "minimum_evidence": 1, + "description": "A trusted independent collector proves the exact deployed identity, real critical journeys, negative controls, stability, and external client acceptance end to end." } ] } diff --git a/pyproject.toml b/pyproject.toml index 754c58c..3f1a10e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "echo-certification-forge" -version = "1.1.0" +version = "1.2.0" description = "Deterministic, evidence-backed release authority for EchoForge" readme = "README.md" requires-python = ">=3.12" diff --git a/scripts/certification_journey.py b/scripts/certification_journey.py index 712b631..38ca793 100644 --- a/scripts/certification_journey.py +++ b/scripts/certification_journey.py @@ -45,7 +45,7 @@ def require(condition: bool, message: str) -> None: def main() -> int: - require(__version__ == "1.1.0", "package version is not synchronized") + require(__version__ == "1.2.0", "package version is not synchronized") python_files = sorted(SRC.rglob("*.py")) require(bool(python_files), "source tree contains no Python modules") for path in python_files: diff --git a/scripts/p1_acceptance.py b/scripts/p1_acceptance.py index 77c43e4..163a603 100644 --- a/scripts/p1_acceptance.py +++ b/scripts/p1_acceptance.py @@ -70,30 +70,37 @@ def main() -> int: "reasons": list(default.reasons), } - store.register_run("cert-verified-ready", target, environment, manifest.manifest_id, manifest.digest) + # P1 is source/control-plane acceptance, not an external production E2E run. Even with + # every local rule row marked true, it must remain NOT_READY without the independently + # signed production_e2e attestation. + store.register_run("cert-source-only", target, environment, manifest.manifest_id, manifest.digest) for index, rule in enumerate(manifest.rules, start=1): artifact_id = f"proof-{index:02d}" content = json.dumps({"rule": rule.id, "passed": True}, sort_keys=True).encode() store.append_artifact( - "cert-verified-ready", target.tenant_id, artifact_id, content, + "cert-source-only", target.tenant_id, artifact_id, content, "application/json", "p1-independent-harness", RedactionStatus.COMPLETE, ) store.record_rule_result( - "cert-verified-ready", target.tenant_id, + "cert-source-only", target.tenant_id, RuleResult(rule.id, True, (artifact_id,), {"fixture": "known-good"}), ) - store.set_run_outcome("cert-verified-ready", target.tenant_id, RunOutcome.COMPLETE) - ready = engine.evaluate(store, "cert-verified-ready", target.tenant_id, manifest, signer.key_id) + store.set_run_outcome("cert-source-only", target.tenant_id, RunOutcome.COMPLETE) + ready = engine.evaluate(store, "cert-source-only", target.tenant_id, manifest, signer.key_id) envelope = signer.sign(ready) - store.save_signed_verdict("cert-verified-ready", target.tenant_id, envelope) + store.save_signed_verdict("cert-source-only", target.tenant_id, envelope) trusted = TrustedPublicKeyRegistry.empty() trusted.add_pem(signer.public_key_pem) gate = DeployGate(store, trusted).evaluate( - target.tenant_id, "cert-verified-ready", target.identity_digest, + target.tenant_id, "cert-source-only", target.identity_digest, environment.identity_digest, manifest.digest, ) - report["scenarios"]["verified_ready"] = { - "passed": ready.release_verdict.value == "PRODUCTION_READY" and gate.allowed, + report["scenarios"]["source_only_blocks"] = { + "passed": ( + ready.release_verdict.value == "NOT_READY" + and not gate.allowed + and "production_e2e_schema_invalid" in ready.reasons + ), "verdict": ready.release_verdict.value, "deploy_gate_allowed": gate.allowed, "evidence_merkle_root": ready.evidence_merkle_root, @@ -101,10 +108,10 @@ def main() -> int: "signing_key_id": envelope.key_id, } - tampered_path = workspace / "evidence" / target.tenant_id / "cert-verified-ready" / "artifacts" / "proof-01.bin" + tampered_path = workspace / "evidence" / target.tenant_id / "cert-source-only" / "artifacts" / "proof-01.bin" tampered_path.write_bytes(b"tampered") tampered_gate = DeployGate(store, trusted).evaluate( - target.tenant_id, "cert-verified-ready", target.identity_digest, + target.tenant_id, "cert-source-only", target.identity_digest, environment.identity_digest, manifest.digest, ) report["scenarios"]["tamper_blocks"] = { diff --git a/src/echo_certification_forge/__init__.py b/src/echo_certification_forge/__init__.py index c8c894b..ddc7799 100644 --- a/src/echo_certification_forge/__init__.py +++ b/src/echo_certification_forge/__init__.py @@ -3,4 +3,4 @@ from .models import ReleaseVerdict, RunOutcome, RunState __all__ = ["ReleaseVerdict", "RunOutcome", "RunState"] -__version__ = "1.1.0" +__version__ = "1.2.0" diff --git a/src/echo_certification_forge/executor.py b/src/echo_certification_forge/executor.py index 4105c7e..e909dc3 100644 --- a/src/echo_certification_forge/executor.py +++ b/src/echo_certification_forge/executor.py @@ -29,8 +29,11 @@ from .canonical import to_utc_iso, utc_now from .evidence import EvidenceStore from .hostile import scan_target_source -from .models import RedactionStatus, RunOutcome, RuleResult, RunState +from .models import EnvironmentIdentity, RedactionStatus, RuleResult, RunOutcome, RunState, TargetIdentity from .policy import RuleManifest +from .production_e2e import RULE_ID as PRODUCTION_E2E_RULE_ID +from .production_e2e import VerifiedProductionE2E +from .production_e2e import validate_production_e2e from .signing import Ed25519VerdictSigner from .supply_chain import scan_dockerfile from .verdict import DeterministicVerdictEngine @@ -193,6 +196,7 @@ def execute( control_attestations: dict[str, bool] | None = None, adapter_records: tuple[AdapterExecutionRecord, ...] | None = None, adapter_policy: AdapterAcceptancePolicy | None = None, + production_e2e_attestation: VerifiedProductionE2E | None = None, execution_guard: Callable[[], None] | None = None, completion_callback: Callable[[Any], None] | None = None, ) -> ExecutionResult: @@ -393,6 +397,32 @@ def bound(digest_key: str) -> bool: # Includes adapter records/assessment when supplied. evidence_chain_ok = self.store.verify_evidence(run_id, tenant_id).valid + production_e2e_passed = False + production_e2e_detail: dict[str, Any] = { + "validation": "production_e2e_attestation_missing" + } + try: + target_identity = TargetIdentity(**json.loads(run["target_identity_json"])) + environment_identity = EnvironmentIdentity(**json.loads(run["environment_identity_json"])) + production_e2e_payload = ( + production_e2e_attestation.payload + if isinstance(production_e2e_attestation, VerifiedProductionE2E) + else None + ) + production_e2e_passed, validation = validate_production_e2e( + production_e2e_payload, + target_identity, + environment_identity, + now=utc_now(), + ) + production_e2e_detail = ( + production_e2e_attestation.result_details() + if production_e2e_passed and production_e2e_attestation is not None + else {"validation": validation} + ) + except (TypeError, ValueError): + production_e2e_detail = {"validation": "production_e2e_identity_invalid"} + checks: dict[str, tuple[bool, dict]] = { "immutable_target_identity": ( bound("target_identity_digest"), @@ -429,6 +459,7 @@ def bound(digest_key: str) -> bool: bool(control_attestations.get("signing_authority_separation", False)), {"check": "trusted_caller_attestation"}, ), + PRODUCTION_E2E_RULE_ID: (production_e2e_passed, production_e2e_detail), } for rule in self.manifest.rules: if execution_guard is not None: @@ -444,7 +475,12 @@ def bound(digest_key: str) -> bool: self.store.record_rule_result( run_id, tenant_id, - RuleResult(rule.id, passed, (artifact_id,), {"source": "executor"}), + RuleResult( + rule.id, + passed, + (artifact_id,), + detail if rule.id == PRODUCTION_E2E_RULE_ID else {"source": "executor"}, + ), ) self._t(run_id, tenant_id, RunState.CLASSIFYING_FINDINGS, "classify") diff --git a/src/echo_certification_forge/intake.py b/src/echo_certification_forge/intake.py index 2842cf5..67cf2af 100644 --- a/src/echo_certification_forge/intake.py +++ b/src/echo_certification_forge/intake.py @@ -12,9 +12,17 @@ from pydantic import BaseModel, ConfigDict, Field -from .canonical import sha256_json +from .canonical import sha256_json, utc_now from .evidence import EvidenceStore -from .models import ReleaseVerdict, RunState, declared_target_identity_digest +from .models import ( + EnvironmentIdentity, + ReleaseVerdict, + RunState, + TargetIdentity, + declared_target_identity_digest, +) +from .production_e2e import RULE_ID as PRODUCTION_E2E_RULE_ID +from .production_e2e import validate_production_e2e from .policy import RuleManifest # Internal fine-grained RunState -> coarse public state (contracts/schemas/certification-run.v1.json). @@ -152,6 +160,59 @@ def project_run(store: EvidenceStore, row: dict[str, Any]) -> dict[str, Any]: payload = json.loads(verdict_row["payload_json"]) release_verdict = payload.get("release_verdict", release_verdict) evidence_merkle_root = payload.get("evidence_merkle_root") + production_e2e = { + "verified": False, + "profile": None, + "attestation_id": None, + "source_commit": None, + "deployment_sha": None, + "target_identity_digest": None, + "environment_identity_digest": None, + "canonical_target": None, + "tool_count": None, + "signature_verified": False, + "collector_key_id": None, + "attestation_envelope_sha256": None, + } + result = store.list_rule_results(run_id, tenant_id).get(PRODUCTION_E2E_RULE_ID) + if result is not None: + try: + target = TargetIdentity(**json.loads(row["target_identity_json"])) + environment = EnvironmentIdentity(**json.loads(row["environment_identity_json"])) + verified, _reason = validate_production_e2e( + result.details, + target, + environment, + now=utc_now(), + ) + production_e2e = { + "verified": verified, + "profile": result.details.get("profile") if verified else None, + "attestation_id": result.details.get("attestation_id") if verified else None, + "source_commit": result.details.get("source_commit") if verified else None, + "deployment_sha": result.details.get("deployment_sha") if verified else None, + "target_identity_digest": ( + result.details.get("target_identity_digest") if verified else None + ), + "environment_identity_digest": ( + result.details.get("environment_identity_digest") if verified else None + ), + "canonical_target": ( + result.details.get("canonical_target") if verified else None + ), + "tool_count": result.details.get("tool_count") if verified else None, + "signature_verified": ( + result.details.get("signature_verified") is True if verified else False + ), + "collector_key_id": ( + result.details.get("collector_key_id") if verified else None + ), + "attestation_envelope_sha256": ( + result.details.get("attestation_envelope_sha256") if verified else None + ), + } + except (TypeError, ValueError): + pass projected = { "run_id": run_id, "tenant_id": tenant_id, @@ -162,6 +223,7 @@ def project_run(store: EvidenceStore, row: dict[str, Any]) -> dict[str, Any]: "environment_identity_digest": row["environment_identity_digest"], "policy_version": row["policy_version"] or row["rule_manifest_id"], "evidence_merkle_root": evidence_merkle_root, + "production_e2e": production_e2e, "created_at": row["created_at"], "updated_at": row["updated_at"], } diff --git a/src/echo_certification_forge/models.py b/src/echo_certification_forge/models.py index 546faa7..74602d6 100644 --- a/src/echo_certification_forge/models.py +++ b/src/echo_certification_forge/models.py @@ -182,6 +182,9 @@ class VerdictDecision: rule_manifest_id: str rule_manifest_digest: str evidence_merkle_root: str + production_e2e_attestation_id: str | None + production_e2e_profile: str | None + production_e2e_envelope_sha256: str | None signing_key_id: str issued_at: datetime expires_at: datetime @@ -199,6 +202,9 @@ def to_dict(self) -> dict[str, Any]: "rule_manifest_id": self.rule_manifest_id, "rule_manifest_digest": self.rule_manifest_digest, "evidence_merkle_root": self.evidence_merkle_root, + "production_e2e_attestation_id": self.production_e2e_attestation_id, + "production_e2e_profile": self.production_e2e_profile, + "production_e2e_envelope_sha256": self.production_e2e_envelope_sha256, "signing_key_id": self.signing_key_id, "issued_at": to_utc_iso(self.issued_at), "expires_at": to_utc_iso(self.expires_at), diff --git a/src/echo_certification_forge/production_e2e.py b/src/echo_certification_forge/production_e2e.py new file mode 100644 index 0000000..b35e4eb --- /dev/null +++ b/src/echo_certification_forge/production_e2e.py @@ -0,0 +1,351 @@ +"""Fail-closed validation for signed production end-to-end attestations. + +Source checks, a successful process exit, and a reachable URL are not production +evidence. This contract is evaluated by the deterministic verdict authority +after a trusted collector has signed the attestation. Target-controlled code +cannot manufacture the attestation or select its trust key. +""" +from __future__ import annotations + +import json +import re +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from .canonical import sha256_json +from .models import EnvironmentIdentity, SignedVerdictEnvelope, TargetIdentity +from .signing import TrustedPublicKeyRegistry + +RULE_ID = "production_e2e" +SCHEMA_VERSION = "certforge.production-e2e.v1" +GENERIC_PROFILE = "generic-production-v1" +ECHO_GITHUB_AUTONOMY_PROFILE = "echo-github-autonomy-remote-mcp-v1" +ECHO_GITHUB_AUTONOMY_REPOSITORY = "echoomegaprime/echo-github-autonomy" +ECHO_GITHUB_AUTONOMY_CANONICAL_MCP = "https://echo-ghub.grok.me/api/plugin/mcp" + +BASE_CHECKS = frozenset( + { + "runtime_or_artifact_executed", + "exact_identity_readback", + "critical_journeys_complete", + "negative_controls_pass", + "stability_verified", + "external_acceptance_verified", + } +) +ECHO_GITHUB_AUTONOMY_CHECKS = BASE_CHECKS | frozenset( + { + "canonical_mcp_health", + "oauth_discovery", + "mcp_initialize", + "tool_schema", + "repeated_tool_invocation", + "registry_persistence", + "four_account_reconciliation", + "private_public_visibility", + "read_write_certify", + "cross_client_consistency", + } +) +ECHO_GITHUB_ACCOUNTS = { + "echoomegaprime": 314902331, + "ECHO-OMEGA-PRIME": 264607697, + "Bmcbob76": 203470412, + "bobmcwilliams4": 235318155, +} +ECHO_CLIENTS = frozenset({"chatgpt", "claude", "codex", "grok"}) + +_SHA = re.compile(r"^[0-9a-f]{40}(?:[0-9a-f]{24})?$") +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$") +_SIGNING_KEY_ID = re.compile(r"^ed25519:[0-9a-f]{32}$") +_SECRET_VALUE = re.compile( + r"(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|" + r"-----BEGIN [A-Z ]*PRIVATE KEY-----|Bearer\s+[A-Za-z0-9._~-]{16,})" +) +_RESTRICTED_KEYS = { + "password", + "passwd", + "secret", + "token", + "private_key", + "authorization", + "cookie", + "api_key", +} +_VERIFIED_MARKER = object() + + +@dataclass(frozen=True, slots=True) +class VerifiedProductionE2E: + """An attestation whose envelope was verified by an independently pinned key registry.""" + + payload: dict[str, Any] + collector_key_id: str + envelope_sha256: str + _marker: object + + def __post_init__(self) -> None: + if self._marker is not _VERIFIED_MARKER: + raise ValueError("VerifiedProductionE2E must come from signature verification") + + def result_details(self) -> dict[str, Any]: + return { + **self.payload, + "signature_verified": True, + "collector_key_id": self.collector_key_id, + "attestation_envelope_sha256": self.envelope_sha256, + } + + +def _parse_time(value: object) -> datetime | None: + if not isinstance(value, str): + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return parsed.astimezone(UTC) if parsed.tzinfo is not None else None + + +def _contains_restricted_value(value: Any, *, key_name: str = "") -> bool: + normalized = key_name.casefold().replace("-", "_") + if normalized in _RESTRICTED_KEYS: + return True + if isinstance(value, Mapping): + return any( + _contains_restricted_value(item, key_name=str(key)) for key, item in value.items() + ) + if isinstance(value, (list, tuple, set)): + return any(_contains_restricted_value(item) for item in value) + return isinstance(value, str) and bool(_SECRET_VALUE.search(value)) + + +def _source_repository(target: TargetIdentity) -> str | None: + reference = target.canonical_ref.casefold() + marker = "github.com/" + if marker not in reference: + return None + suffix = reference.split(marker, 1)[1].split("@", 1)[0] + if suffix.endswith(".git"): + suffix = suffix[:-4] + return suffix + + +def _validate_common( + payload: Mapping[str, Any], + target: TargetIdentity, + environment: EnvironmentIdentity, + *, + now: datetime, +) -> str | None: + if _contains_restricted_value(payload): + return "production_e2e_contains_restricted_value" + if payload.get("schema_version") != SCHEMA_VERSION: + return "production_e2e_schema_invalid" + if not _IDENTIFIER.fullmatch(str(payload.get("attestation_id") or "")): + return "production_e2e_attestation_id_invalid" + if payload.get("target_identity_digest") != target.identity_digest: + return "production_e2e_target_identity_mismatch" + if payload.get("environment_identity_digest") != environment.identity_digest: + return "production_e2e_environment_identity_mismatch" + if not _SIGNING_KEY_ID.fullmatch(str(payload.get("signing_key_id") or "")): + return "production_e2e_signing_key_id_invalid" + + observed_at = _parse_time(payload.get("observed_at")) + expires_at = _parse_time(payload.get("expires_at")) + current = now.astimezone(UTC) if now.tzinfo is not None else None + if current is None or observed_at is None or expires_at is None: + return "production_e2e_time_invalid" + if expires_at <= observed_at or (expires_at - observed_at).total_seconds() > 3600: + return "production_e2e_lifetime_invalid" + if observed_at > current or current >= expires_at: + return "production_e2e_attestation_not_current" + + source_commit = payload.get("source_commit") + deployment_sha = payload.get("deployment_sha") + if target.source_commit is not None: + expected = target.source_commit.casefold() + if not _SHA.fullmatch(expected): + return "production_e2e_target_source_commit_not_exact" + if source_commit != expected or deployment_sha != expected: + return "production_e2e_deployment_sha_mismatch" + elif source_commit is not None or deployment_sha is not None: + if source_commit != deployment_sha or not _SHA.fullmatch(str(source_commit)): + return "production_e2e_deployment_sha_invalid" + + required_checks = payload.get("required_checks") + checks = payload.get("checks") + if not isinstance(required_checks, list) or not isinstance(checks, Mapping): + return "production_e2e_checks_invalid" + if len(required_checks) != len(set(required_checks)): + return "production_e2e_checks_duplicated" + check_set = frozenset(required_checks) + if set(checks) != set(check_set) or any(checks.get(name) is not True for name in check_set): + return "production_e2e_checks_incomplete" + if not BASE_CHECKS <= check_set: + return "production_e2e_baseline_checks_missing" + if not isinstance(payload.get("stability_probe_count"), int) or payload["stability_probe_count"] < 3: + return "production_e2e_stability_probe_count_insufficient" + return None + + +def _validate_echo_github_autonomy(payload: Mapping[str, Any]) -> str | None: + if payload.get("canonical_target") != ECHO_GITHUB_AUTONOMY_CANONICAL_MCP: + return "production_e2e_canonical_mcp_mismatch" + if frozenset(payload.get("required_checks") or ()) != ECHO_GITHUB_AUTONOMY_CHECKS: + return "production_e2e_github_autonomy_checks_incomplete" + if payload.get("tool_count") != 30: + return "production_e2e_tool_count_mismatch" + + repositories = payload.get("sample_private_repositories") + if not isinstance(repositories, Mapping) or set(repositories) != set(ECHO_GITHUB_ACCOUNTS): + return "production_e2e_sample_repositories_incomplete" + sample_digests: dict[str, str] = {} + for login, sample in repositories.items(): + if not isinstance(sample, Mapping): + return "production_e2e_sample_repository_invalid" + if ( + not isinstance(sample.get("repository_id"), int) + or isinstance(sample["repository_id"], bool) + or sample["repository_id"] <= 0 + ): + return "production_e2e_sample_repository_id_invalid" + if not str(sample.get("node_id") or "").strip(): + return "production_e2e_sample_node_id_invalid" + if not str(sample.get("default_branch") or "").strip(): + return "production_e2e_sample_default_branch_invalid" + if not _SHA.fullmatch(str(sample.get("head_sha") or "")): + return "production_e2e_sample_head_sha_invalid" + digest = str(sample.get("fingerprint_sha256") or "") + if not _SHA256.fullmatch(digest): + return "production_e2e_sample_fingerprint_invalid" + sample_digests[login] = digest + + accounts = payload.get("accounts") + if not isinstance(accounts, Mapping) or set(accounts) != set(ECHO_GITHUB_ACCOUNTS): + return "production_e2e_accounts_incomplete" + for login, account_id in ECHO_GITHUB_ACCOUNTS.items(): + account = accounts.get(login) + if not isinstance(account, Mapping) or account.get("account_id") != account_id: + return "production_e2e_account_identity_mismatch" + counts = ( + account.get("enumerated_count"), + account.get("upstream_total_count"), + account.get("public_count"), + account.get("private_count"), + ) + if any(not isinstance(value, int) or isinstance(value, bool) or value < 0 for value in counts): + return "production_e2e_account_counts_invalid" + if counts[0] != counts[1] or counts[2] <= 0 or counts[3] <= 0: + return "production_e2e_account_reconciliation_failed" + if any(account.get(name) is not True for name in ("read", "write", "certify")): + return "production_e2e_account_authority_incomplete" + if account.get("credential_source") not in { + "github_app_installation", + "vault_user_token_fallback", + }: + return "production_e2e_credential_source_invalid" + if account.get("secret_exposed") is not False: + return "production_e2e_secret_boundary_failed" + + clients = payload.get("clients") + if not isinstance(clients, Mapping) or set(clients) != set(ECHO_CLIENTS): + return "production_e2e_clients_incomplete" + for client in clients.values(): + if not isinstance(client, Mapping) or client.get("accepted") is not True: + return "production_e2e_client_not_accepted" + fingerprints = client.get("repository_fingerprints") + if not isinstance(fingerprints, Mapping) or dict(fingerprints) != sample_digests: + return "production_e2e_cross_client_repository_mismatch" + return None + + +def validate_production_e2e( + payload: Mapping[str, Any] | None, + target: TargetIdentity, + environment: EnvironmentIdentity, + *, + now: datetime, +) -> tuple[bool, str]: + """Validate one trusted-collector payload against the exact run identities.""" + + if not isinstance(payload, Mapping): + return False, "production_e2e_attestation_missing" + error = _validate_common(payload, target, environment, now=now) + if error is not None: + return False, error + repository = _source_repository(target) + profile = payload.get("profile") + if repository == ECHO_GITHUB_AUTONOMY_REPOSITORY: + if profile != ECHO_GITHUB_AUTONOMY_PROFILE: + return False, "production_e2e_profile_mismatch" + error = _validate_echo_github_autonomy(payload) + return (error is None, error or "production_e2e_verified") + if profile != GENERIC_PROFILE: + return False, "production_e2e_profile_mismatch" + if frozenset(payload.get("required_checks") or ()) != BASE_CHECKS: + return False, "production_e2e_generic_checks_incomplete" + return True, "production_e2e_verified" + + +def validate_attestation_trust_metadata(payload: Mapping[str, Any]) -> tuple[bool, str]: + """Require the signature-verification metadata added by ``VerifiedProductionE2E``.""" + + if payload.get("signature_verified") is not True: + return False, "production_e2e_signature_not_verified" + if payload.get("collector_key_id") != payload.get("signing_key_id"): + return False, "production_e2e_collector_key_mismatch" + if not _SIGNING_KEY_ID.fullmatch(str(payload.get("collector_key_id") or "")): + return False, "production_e2e_collector_key_invalid" + if not _SHA256.fullmatch(str(payload.get("attestation_envelope_sha256") or "")): + return False, "production_e2e_envelope_sha256_invalid" + return True, "production_e2e_signature_verified" + + +def verify_signed_attestation( + envelope: SignedVerdictEnvelope, + trusted_keys: TrustedPublicKeyRegistry, +) -> VerifiedProductionE2E: + """Verify one collector envelope against an independently selected key registry.""" + + verified, reason = trusted_keys.verify(envelope) + if not verified: + raise ValueError(f"production E2E attestation is not trusted: {reason}") + if envelope.payload.get("schema_version") != SCHEMA_VERSION: + raise ValueError("production E2E attestation schema is unsupported") + if envelope.payload.get("signing_key_id") != envelope.key_id: + raise ValueError("production E2E attestation key binding is invalid") + envelope_sha256 = sha256_json( + { + "payload": envelope.payload, + "signature_b64": envelope.signature_b64, + "key_id": envelope.key_id, + } + ) + return VerifiedProductionE2E( + payload=dict(envelope.payload), + collector_key_id=envelope.key_id, + envelope_sha256=envelope_sha256, + _marker=_VERIFIED_MARKER, + ) + + +def load_signed_attestation(path: Path, trusted_key_directory: Path) -> VerifiedProductionE2E: + """Load and independently verify a collector envelope using pinned public keys.""" + + try: + document = json.loads(path.read_text(encoding="utf-8")) + envelope = SignedVerdictEnvelope( + payload=dict(document["payload"]), + signature_b64=str(document["signature_b64"]), + key_id=str(document["key_id"]), + public_key_pem=str(document.get("public_key_pem") or ""), + ) + except (OSError, ValueError, TypeError, KeyError, json.JSONDecodeError) as exc: + raise ValueError("production E2E attestation envelope is invalid") from exc + registry = TrustedPublicKeyRegistry.from_directory(trusted_key_directory) + return verify_signed_attestation(envelope, registry) diff --git a/src/echo_certification_forge/run_worker.py b/src/echo_certification_forge/run_worker.py index 3774b5d..e2fc1a6 100644 --- a/src/echo_certification_forge/run_worker.py +++ b/src/echo_certification_forge/run_worker.py @@ -17,6 +17,7 @@ import sqlite3 import tempfile import threading +from collections.abc import Callable from pathlib import Path from cryptography.hazmat.primitives import serialization @@ -38,6 +39,7 @@ from .executor import RetentionPolicy, RunExecutor, StaticEntitlement from .models import EnvironmentIdentity, RunOutcome, RunState, TargetIdentity from .policy import RuleManifest +from .production_e2e import VerifiedProductionE2E, load_signed_attestation from .runner import RunnerResponse from .sandbox import DEFAULT_IMAGE, DockerSandbox, sandboxed_journey_runner from .signing import Ed25519VerdictSigner @@ -47,7 +49,7 @@ _ADAPTER_RULE = "adapter_identity_and_quality" _PRODUCTION_MANIFEST_ID = "certforge.release-strict.v2" _PRODUCTION_MANIFEST_SHA256 = ( - "7dc98e0e95e6dd2c000ec069a8c46c4d1d49a4fe869ad4eae25e059d103644f4" + "08ba068ceb3e14bfed2690337edbb94c546e3e0a1a89b1321f7657653d8eea43" ) class _ClaimHeartbeat: def __init__(self, governance: SubscriberGovernance, claim) -> None: @@ -160,6 +162,10 @@ def run( adapter_records: tuple[AdapterExecutionRecord, ...] | None = None, adapter_policy: AdapterAcceptancePolicy | None = None, adapter_bundle_response: RunnerResponse | None = None, + production_e2e_attestation: VerifiedProductionE2E | None = None, + production_e2e_provider: ( + Callable[[TargetIdentity, EnvironmentIdentity], VerifiedProductionE2E] | None + ) = None, worker_id: str | None = None, worker_attestation_sha256: str | None = None, execution_location: str = "local", @@ -348,6 +354,8 @@ def run( else None ) environment = _worker_environment(adapter_digest, adapter_execution_profile_sha256) + if production_e2e_attestation is None and production_e2e_provider is not None: + production_e2e_attestation = production_e2e_provider(target, environment) if subscribers is not None: if existing is None or existing["state"] != RunState.QUEUED.value: if claim is not None: @@ -530,6 +538,7 @@ def complete_subscriber_execution(envelope): }, adapter_records=adapter_records, adapter_policy=adapter_policy, + production_e2e_attestation=production_e2e_attestation, execution_guard=execution_guard, completion_callback=completion_callback, ) @@ -669,6 +678,26 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--tenant", required=True) parser.add_argument("--target-json", required=True, help='e.g. {"type":"local","path":"/abs/dir"}') parser.add_argument("--journey-json", default=None, help='e.g. ["python3","hello.py"]') + parser.add_argument( + "--production-e2e-attestation", + type=Path, + default=( + Path(os.environ["ECHO_CERTFORGE_PRODUCTION_E2E_ATTESTATION"]) + if os.environ.get("ECHO_CERTFORGE_PRODUCTION_E2E_ATTESTATION") + else None + ), + help="trusted-collector signed production E2E envelope", + ) + parser.add_argument( + "--trusted-production-e2e-keys", + type=Path, + default=( + Path(os.environ["ECHO_CERTFORGE_TRUSTED_PRODUCTION_E2E_KEYS"]) + if os.environ.get("ECHO_CERTFORGE_TRUSTED_PRODUCTION_E2E_KEYS") + else None + ), + help="directory of pinned Ed25519 collector public keys", + ) parser.add_argument("--policy-version", default=None) parser.add_argument( "--db", @@ -875,6 +904,19 @@ def main(argv: list[str] | None = None) -> int: ) target_spec = json.loads(args.target_json) journey = json.loads(args.journey_json) if args.journey_json else None + production_e2e_attestation = None + if args.production_e2e_attestation is not None: + if args.trusted_production_e2e_keys is None: + print(json.dumps({"error": "trusted_production_e2e_keys_missing"})) + return 2 + try: + production_e2e_attestation = load_signed_attestation( + args.production_e2e_attestation, + args.trusted_production_e2e_keys, + ) + except ValueError as exc: + print(json.dumps({"error": "production_e2e_attestation_invalid", "detail": str(exc)})) + return 2 sandbox = None if args.sandbox: sandbox = DockerSandbox( @@ -896,6 +938,7 @@ def main(argv: list[str] | None = None) -> int: adapter_records=adapter_records, adapter_policy=adapter_policy, adapter_bundle_response=adapter_rebound_response, + production_e2e_attestation=production_e2e_attestation, worker_id=args.worker_id, worker_attestation_sha256=args.worker_attestation_sha256, execution_location=args.execution_location, diff --git a/src/echo_certification_forge/verdict.py b/src/echo_certification_forge/verdict.py index c739357..457751c 100644 --- a/src/echo_certification_forge/verdict.py +++ b/src/echo_certification_forge/verdict.py @@ -9,6 +9,9 @@ from .evidence import EvidenceStore from .models import EnvironmentIdentity, ReleaseVerdict, RunOutcome, TargetIdentity, VerdictDecision from .policy import RuleManifest +from .production_e2e import RULE_ID as PRODUCTION_E2E_RULE_ID +from .production_e2e import validate_attestation_trust_metadata +from .production_e2e import validate_production_e2e class DeterministicVerdictEngine: @@ -34,6 +37,8 @@ def evaluate( # ({tenant_id, target_type, declared_identity_digest, reference}), not the full canonical # TargetIdentity. Such a run has not been reconciled to an acquired artifact, so it cannot be # certified: fail-closed with an explicit reason instead of crashing on TargetIdentity(**...). + target: TargetIdentity | None = None + environment: EnvironmentIdentity | None = None if "declared_identity_digest" in target_data or "artifact_sha256" not in target_data: reasons.append("target_identity_not_reconciled") else: @@ -72,6 +77,35 @@ def evaluate( if invalid_references: reasons.append(f"invalid_rule_evidence:{rule.id}") + # PRODUCTION_READY is impossible without a current, signed, exact-identity E2E + # attestation. This is an engine invariant, not an optional policy convention; + # legacy/custom manifests that omit the rule fail closed as well. + if not any(rule.id == PRODUCTION_E2E_RULE_ID for rule in manifest.rules): + reasons.append("production_e2e_rule_missing_from_policy") + e2e_result = results.get(PRODUCTION_E2E_RULE_ID) + verified_e2e: dict[str, Any] | None = None + if target is None or environment is None: + reasons.append("production_e2e_identity_unavailable") + elif e2e_result is None: + reasons.append("production_e2e_attestation_missing") + else: + e2e_valid, e2e_reason = validate_production_e2e( + e2e_result.details, + target, + environment, + now=utc_now(), + ) + if not e2e_valid: + reasons.append(e2e_reason) + else: + trust_valid, trust_reason = validate_attestation_trust_metadata( + e2e_result.details + ) + if not trust_valid: + reasons.append(trust_reason) + else: + verified_e2e = e2e_result.details + if store.blocking_findings(run_id, tenant_id): reasons.append("blocking_findings_present") @@ -97,6 +131,17 @@ def evaluate( rule_manifest_id=manifest.manifest_id, rule_manifest_digest=manifest.digest, evidence_merkle_root=verification.merkle_root, + production_e2e_attestation_id=( + str(verified_e2e["attestation_id"]) if verified_e2e is not None else None + ), + production_e2e_profile=( + str(verified_e2e["profile"]) if verified_e2e is not None else None + ), + production_e2e_envelope_sha256=( + str(verified_e2e["attestation_envelope_sha256"]) + if verified_e2e is not None + else None + ), signing_key_id=signing_key_id, issued_at=issued_at, expires_at=issued_at + timedelta(seconds=manifest.verdict_ttl_seconds), diff --git a/tests/conftest.py b/tests/conftest.py index f674a9a..138dac5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,12 +26,13 @@ def store(tmp_path: Path) -> EvidenceStore: @pytest.fixture def target() -> TargetIdentity: + source_commit = "a" * 40 return TargetIdentity( tenant_id="tenant-alpha", target_type="git", - canonical_ref="https://github.com/example/project@abc123", + canonical_ref=f"https://github.com/example/project@{source_commit}", artifact_sha256=digest("artifact"), - source_commit="abc123", + source_commit=source_commit, dependency_sha256=digest("dependencies"), configuration_sha256=digest("configuration"), ) diff --git a/tests/production_e2e_support.py b/tests/production_e2e_support.py new file mode 100644 index 0000000..8c3d2c7 --- /dev/null +++ b/tests/production_e2e_support.py @@ -0,0 +1,46 @@ +"""Trusted production-E2E attestations used by tests that exercise ready paths.""" + +from __future__ import annotations + +from datetime import timedelta + +from echo_certification_forge.canonical import to_utc_iso, utc_now +from echo_certification_forge.models import EnvironmentIdentity, TargetIdentity +from echo_certification_forge.production_e2e import ( + BASE_CHECKS, + GENERIC_PROFILE, + SCHEMA_VERSION, + VerifiedProductionE2E, + verify_signed_attestation, +) +from echo_certification_forge.signing import Ed25519VerdictSigner, TrustedPublicKeyRegistry + + +def trusted_generic_production_e2e( + target: TargetIdentity, + environment: EnvironmentIdentity, +) -> VerifiedProductionE2E: + """Create a fresh signed generic attestation through the real trust verifier.""" + + observed_at = utc_now() + signer = Ed25519VerdictSigner.generate() + payload = { + "schema_version": SCHEMA_VERSION, + "attestation_id": f"test-e2e-{target.identity_digest[:16]}", + "profile": GENERIC_PROFILE, + "target_identity_digest": target.identity_digest, + "environment_identity_digest": environment.identity_digest, + "source_commit": target.source_commit, + "deployment_sha": target.source_commit, + "canonical_target": target.canonical_ref, + "required_checks": sorted(BASE_CHECKS), + "checks": {name: True for name in sorted(BASE_CHECKS)}, + "stability_probe_count": 3, + "observed_at": to_utc_iso(observed_at), + "expires_at": to_utc_iso(observed_at + timedelta(minutes=30)), + "signing_key_id": signer.key_id, + } + envelope = signer.sign_payload(payload) + trusted = TrustedPublicKeyRegistry.empty() + trusted.add_pem(signer.public_key_pem) + return verify_signed_attestation(envelope, trusted) diff --git a/tests/test_additional_contracts.py b/tests/test_additional_contracts.py index 7687852..362b3f7 100644 --- a/tests/test_additional_contracts.py +++ b/tests/test_additional_contracts.py @@ -22,10 +22,12 @@ from echo_certification_forge.service import ServiceContext, create_app from echo_certification_forge.signing import Ed25519VerdictSigner, TrustedPublicKeyRegistry from echo_certification_forge.verdict import DeterministicVerdictEngine +from production_e2e_support import trusted_generic_production_e2e def complete_run(store, manifest, target, environment, run_id="cert-extra"): store.register_run(run_id, target, environment, manifest.manifest_id, manifest.digest) + production_e2e = trusted_generic_production_e2e(target, environment).result_details() for index, rule in enumerate(manifest.rules, start=1): artifact_id = f"extra-{index:02d}" store.append_artifact( @@ -34,7 +36,12 @@ def complete_run(store, manifest, target, environment, run_id="cert-extra"): ) store.record_rule_result( run_id, target.tenant_id, - RuleResult(rule.id, True, (artifact_id,), {"verified": True}), + RuleResult( + rule.id, + True, + (artifact_id,), + production_e2e if rule.id == "production_e2e" else {"verified": True}, + ), ) store.set_run_outcome(run_id, target.tenant_id, RunOutcome.COMPLETE) return run_id diff --git a/tests/test_evidence_and_verdict.py b/tests/test_evidence_and_verdict.py index 716f488..2265075 100644 --- a/tests/test_evidence_and_verdict.py +++ b/tests/test_evidence_and_verdict.py @@ -3,6 +3,7 @@ import json import sqlite3 from contextlib import closing +from datetime import timedelta from pathlib import Path import pytest @@ -13,10 +14,13 @@ ReleaseVerdict, RuleResult, RunOutcome, + EnvironmentIdentity, TargetIdentity, VerdictLifecycleEvent, ) from echo_certification_forge.signing import Ed25519VerdictSigner, TrustedPublicKeyRegistry +from echo_certification_forge.canonical import to_utc_iso, utc_now +from echo_certification_forge.production_e2e import BASE_CHECKS, GENERIC_PROFILE, SCHEMA_VERSION from echo_certification_forge.verdict import DeterministicVerdictEngine @@ -25,8 +29,33 @@ def register(store, manifest, target, environment, run_id="cert-001"): return run_id -def satisfy_all_rules(store, manifest, run_id, tenant_id): +def satisfy_all_rules(store, manifest, run_id, tenant_id, *, include_production_e2e=True): + run = store.get_run(run_id, tenant_id) + target = TargetIdentity(**json.loads(run["target_identity_json"])) + environment = EnvironmentIdentity(**json.loads(run["environment_identity_json"])) + observed_at = utc_now() + production_e2e = { + "schema_version": SCHEMA_VERSION, + "attestation_id": f"e2e-{run_id}", + "profile": GENERIC_PROFILE, + "target_identity_digest": target.identity_digest, + "environment_identity_digest": environment.identity_digest, + "source_commit": target.source_commit, + "deployment_sha": target.source_commit, + "canonical_target": "https://example.test/runtime", + "required_checks": sorted(BASE_CHECKS), + "checks": {name: True for name in sorted(BASE_CHECKS)}, + "stability_probe_count": 3, + "observed_at": to_utc_iso(observed_at), + "expires_at": to_utc_iso(observed_at + timedelta(minutes=30)), + "signing_key_id": "ed25519:" + "a" * 32, + "signature_verified": True, + "collector_key_id": "ed25519:" + "a" * 32, + "attestation_envelope_sha256": "f" * 64, + } for index, rule in enumerate(manifest.rules, start=1): + if rule.id == "production_e2e" and not include_production_e2e: + continue artifact_id = f"evidence-{index:02d}" store.append_artifact( run_id, @@ -40,7 +69,12 @@ def satisfy_all_rules(store, manifest, run_id, tenant_id): store.record_rule_result( run_id, tenant_id, - RuleResult(rule.id, True, (artifact_id,), {"source": "acceptance"}), + RuleResult( + rule.id, + True, + (artifact_id,), + production_e2e if rule.id == "production_e2e" else {"source": "acceptance"}, + ), ) @@ -81,6 +115,26 @@ def test_complete_run_with_verified_evidence_gets_signed_ready(store, manifest, assert gate.allowed +def test_source_and_rule_rows_without_production_e2e_never_get_ready( + store, manifest, target, environment +): + run_id = register(store, manifest, target, environment) + satisfy_all_rules( + store, + manifest, + run_id, + target.tenant_id, + include_production_e2e=False, + ) + store.set_run_outcome(run_id, target.tenant_id, RunOutcome.COMPLETE) + signer = Ed25519VerdictSigner.generate() + decision = DeterministicVerdictEngine().evaluate( + store, run_id, target.tenant_id, manifest, signer.key_id + ) + assert decision.release_verdict is ReleaseVerdict.NOT_READY + assert "production_e2e_attestation_missing" in decision.reasons + + def test_bare_database_rows_cannot_manufacture_ready(store, manifest, target, environment): run_id = register(store, manifest, target, environment) now = "2026-07-16T00:00:00Z" diff --git a/tests/test_p5_executor_adapter_gate.py b/tests/test_p5_executor_adapter_gate.py index 9f1b9fd..8607e44 100644 --- a/tests/test_p5_executor_adapter_gate.py +++ b/tests/test_p5_executor_adapter_gate.py @@ -17,6 +17,7 @@ from echo_certification_forge.models import EnvironmentIdentity from echo_certification_forge.policy import RuleManifest from echo_certification_forge.signing import Ed25519VerdictSigner +from production_e2e_support import trusted_generic_production_e2e RUN = "cert-p5-executor" @@ -89,7 +90,8 @@ def manifest_v2() -> RuleManifest: def execute(store, target, tmp_path: Path, *, records, adapter_policy, adapter_digest: str): manifest = manifest_v2() - store.register_run(RUN, target, environment(adapter_digest), manifest.manifest_id, manifest.digest) + run_environment = environment(adapter_digest) + store.register_run(RUN, target, run_environment, manifest.manifest_id, manifest.digest) executor = RunExecutor(store, manifest, Ed25519VerdictSigner.generate()) return executor.execute( RUN, @@ -103,6 +105,7 @@ def execute(store, target, tmp_path: Path, *, records, adapter_policy, adapter_d }, adapter_records=records, adapter_policy=adapter_policy, + production_e2e_attestation=trusted_generic_production_e2e(target, run_environment), ) diff --git a/tests/test_p6_deployment_enforcement.py b/tests/test_p6_deployment_enforcement.py index 5c4496a..937915b 100644 --- a/tests/test_p6_deployment_enforcement.py +++ b/tests/test_p6_deployment_enforcement.py @@ -41,6 +41,7 @@ VerdictLifecycleEvent, ) from echo_certification_forge.signing import Ed25519VerdictSigner, TrustedPublicKeyRegistry +from production_e2e_support import trusted_generic_production_e2e TENANT = "tenant-alpha" OTHER_TENANT = "tenant-beta" @@ -57,7 +58,7 @@ def _target(artifact_label: str, tenant_id: str = TENANT) -> TargetIdentity: target_type="container", canonical_ref=f"registry.echo/app@{artifact_label}", artifact_sha256=_digest(artifact_label), - source_commit="abc123def456", + source_commit="abc123abc123abc123abc123abc123abc123abcd", dependency_sha256=_digest("dependencies"), configuration_sha256=_digest("configuration"), ) @@ -107,6 +108,7 @@ def _certify( entitlement=StaticEntitlement(frozenset({target.tenant_id})), journey=[sys.executable, "hello.py"], control_attestations={"runner_control_channel": True, "signing_authority_separation": True}, + production_e2e_attestation=trusted_generic_production_e2e(target, environment), ) assert result.release_verdict == "PRODUCTION_READY", result.blocking_findings @@ -1120,4 +1122,4 @@ def test_delayed_stale_binding_cannot_shadow_active_certification( _admission(target.artifact_sha256, "staging", environment, rolled_manifest, "deploy-shadow-s2"), ACTOR, ) - assert stale_decision.run_id == "cert-shadow-new" \ No newline at end of file + assert stale_decision.run_id == "cert-shadow-new" diff --git a/tests/test_p6_platform_integration.py b/tests/test_p6_platform_integration.py index e5406f3..267bd0a 100644 --- a/tests/test_p6_platform_integration.py +++ b/tests/test_p6_platform_integration.py @@ -51,12 +51,14 @@ from echo_certification_forge.service import ServiceContext, create_app from echo_certification_forge.sandbox import DockerSandbox from echo_certification_forge.signing import Ed25519VerdictSigner, TrustedPublicKeyRegistry +from production_e2e_support import trusted_generic_production_e2e TENANT = "tenant-alpha" OTHER_TENANT = "tenant-beta" SECRET = "p6-webhook-secret-0123456789abcdef" DEPLOY_SECRET = "p6-deploy-credential-alpha-0001" OTHER_DEPLOY_SECRET = "p6-deploy-credential-beta-0002" +SOURCE_COMMIT = "abc123abc123abc123abc123abc123abc123abcd" def _digest(label: str) -> str: @@ -101,6 +103,7 @@ def _certify(store, manifest, signer, environment, tmp_path, run_id, target) -> entitlement=StaticEntitlement(frozenset({target.tenant_id})), journey=[sys.executable, "hello.py"], control_attestations={"runner_control_channel": True, "signing_authority_separation": True}, + production_e2e_attestation=trusted_generic_production_e2e(target, environment), ) assert result.release_verdict == "PRODUCTION_READY", result.blocking_findings @@ -112,7 +115,7 @@ def certified_target(store, manifest, signer, environment, tmp_path) -> TargetId target_type="container", canonical_ref="registry.echo/app@v1", artifact_sha256=_digest("http-app-v1"), - source_commit="abc123def456", + source_commit=SOURCE_COMMIT, dependency_sha256=_digest("dependencies"), configuration_sha256=_digest("configuration"), ) @@ -401,7 +404,7 @@ def test_http_binding_refuses_unsigned_run_and_foreign_tenant( target_type="container", canonical_ref="registry.echo/app@u1", artifact_sha256=_digest("unsigned-app"), - source_commit="abc123def456", + source_commit=SOURCE_COMMIT, ) store.register_run("cert-unsigned", target, environment, manifest.manifest_id, manifest.digest) response = _post_signed(client, "/v1/certifications/cert-unsigned/bindings") @@ -442,7 +445,7 @@ def _build_event(event_id: str = "evt-00000001") -> dict: "event_type": "build.artifact.published", "tenant_id": TENANT, "artifact_sha256": _digest("built-artifact"), - "source_commit": "abc123def456", + "source_commit": SOURCE_COMMIT, "repository": "https://github.com/echo/app", "environment_identity_digest": _digest("build-env"), "policy_version": "certforge.release-strict.v1", @@ -488,7 +491,7 @@ def test_registry_webhook_maps_container_target(client): "tenant_id": TENANT, "image_digest": f"sha256:{_digest('pushed-image')}", "image_repository": "registry.echo/app", - "source_commit": "abc123def456", + "source_commit": SOURCE_COMMIT, "environment_identity_digest": _digest("registry-env"), "policy_version": "certforge.release-strict.v1", } @@ -557,7 +560,7 @@ def _declared_event(artifact: str, environment, manifest, event_id: str = "evt-r "event_type": "build.artifact.published", "tenant_id": TENANT, "artifact_sha256": artifact, - "source_commit": "abc123def456", + "source_commit": SOURCE_COMMIT, "repository": "https://github.com/echo/app", "environment_identity_digest": environment.identity_digest, "policy_version": manifest.manifest_id, @@ -582,9 +585,9 @@ def test_webhook_declared_run_reconciles_certifies_and_deploys_end_to_end( acquired = TargetIdentity( tenant_id=TENANT, target_type="container", - canonical_ref="registry.echo/app@abc123def456", + canonical_ref=f"registry.echo/app@{SOURCE_COMMIT}", artifact_sha256=artifact, - source_commit="abc123def456", + source_commit=SOURCE_COMMIT, ) store.reconcile_declared_target(run_id, TENANT, acquired, environment) row = store.get_run(run_id, TENANT) @@ -602,6 +605,7 @@ def test_webhook_declared_run_reconciles_certifies_and_deploys_end_to_end( entitlement=StaticEntitlement(frozenset({TENANT})), journey=[sys.executable, "hello.py"], control_attestations={"runner_control_channel": True, "signing_authority_separation": True}, + production_e2e_attestation=trusted_generic_production_e2e(acquired, environment), ) assert result.release_verdict == "PRODUCTION_READY", result.blocking_findings @@ -646,9 +650,9 @@ def test_reconciliation_refuses_mismatched_artifact_and_run_stays_undeployable( tampered = TargetIdentity( tenant_id=TENANT, target_type="container", - canonical_ref="registry.echo/app@abc123def456", + canonical_ref=f"registry.echo/app@{SOURCE_COMMIT}", artifact_sha256=_digest("DIFFERENT-acquired-artifact"), - source_commit="abc123def456", + source_commit=SOURCE_COMMIT, ) with pytest.raises(ValueError, match="target_declared_artifact_mismatch"): store.reconcile_declared_target(run_id, TENANT, tampered, environment) @@ -732,9 +736,9 @@ def test_reconciliation_window_closes_at_acquisition(client, store, manifest, en acquired = TargetIdentity( tenant_id=TENANT, target_type="container", - canonical_ref="registry.echo/app@abc123def456", + canonical_ref=f"registry.echo/app@{SOURCE_COMMIT}", artifact_sha256=artifact, - source_commit="abc123def456", + source_commit=SOURCE_COMMIT, ) with pytest.raises(ValueError, match="target_reconciliation_window_closed"): store.reconcile_declared_target(run_id, TENANT, acquired, environment) @@ -772,6 +776,7 @@ def test_run_worker_executes_webhook_declared_run_end_to_end( signer=signer, entitled=frozenset({TENANT}), journey=[sys.executable, "hello.py"], + production_e2e_provider=trusted_generic_production_e2e, ) assert outcome.get("error") is None, outcome assert outcome["release_verdict"] == "PRODUCTION_READY" @@ -1837,7 +1842,7 @@ def test_registry_webhook_oci_run_certifies_and_deploys_end_to_end( "tenant_id": TENANT, "image_digest": manifest_digest, "image_repository": repo, - "source_commit": "abc123def456", + "source_commit": SOURCE_COMMIT, # the platform declares the WORKER's environment commitment for the run "environment_identity_digest": run_worker._worker_environment().identity_digest, "policy_version": manifest.manifest_id, @@ -1862,6 +1867,7 @@ def test_registry_webhook_oci_run_certifies_and_deploys_end_to_end( entitled=frozenset({TENANT}), journey=["python3", "app/hello.py"], sandbox=DockerSandbox(docker=(sys.executable, str(stub))), + production_e2e_provider=trusted_generic_production_e2e, ) finally: os.environ.pop("CERTFORGE_TEST_DOCKER_LOG", None) @@ -2025,7 +2031,7 @@ def test_registry_webhook_oci_journey_without_sandbox_fails_closed( "tenant_id": TENANT, "image_digest": manifest_digest, "image_repository": repo, - "source_commit": "abc123def456", + "source_commit": SOURCE_COMMIT, "environment_identity_digest": run_worker._worker_environment().identity_digest, "policy_version": manifest.manifest_id, } diff --git a/tests/test_p7_subscriber_governance.py b/tests/test_p7_subscriber_governance.py index ce9ba38..a112bba 100644 --- a/tests/test_p7_subscriber_governance.py +++ b/tests/test_p7_subscriber_governance.py @@ -40,6 +40,7 @@ SubscriberGovernance, SubscriberPolicy, ) +from production_e2e_support import trusted_generic_production_e2e PEPPER = b"p7-acceptance-pepper-material-32-bytes-minimum" @@ -1166,6 +1167,7 @@ def test_run_worker_enforces_live_subscriber_entitlement( entitled=frozenset({org.organization_id}), subscribers=governance, journey=[sys.executable, "journey.py"], + production_e2e_provider=trusted_generic_production_e2e, ) assert result["error"] == "subscriber_worker_claim_denied" @@ -3578,6 +3580,7 @@ def test_legal_hold_lifecycle_and_public_verification_use_hardened_subscriber_au entitled=frozenset({owner.organization_id}), subscribers=governance, journey=[sys.executable, "journey.py"], + production_e2e_provider=trusted_generic_production_e2e, ) assert result["state"] == RunState.COMPLETED.value headers = _headers(owner.organization_id, owner.bootstrap_api_key) diff --git a/tests/test_production_e2e.py b/tests/test_production_e2e.py new file mode 100644 index 0000000..81460f5 --- /dev/null +++ b/tests/test_production_e2e.py @@ -0,0 +1,214 @@ +from __future__ import annotations + +import json +from datetime import UTC, datetime, timedelta + +import pytest + +from echo_certification_forge.models import EnvironmentIdentity, TargetIdentity +from echo_certification_forge.production_e2e import ( + ECHO_CLIENTS, + ECHO_GITHUB_ACCOUNTS, + ECHO_GITHUB_AUTONOMY_CANONICAL_MCP, + ECHO_GITHUB_AUTONOMY_CHECKS, + ECHO_GITHUB_AUTONOMY_PROFILE, + SCHEMA_VERSION, + load_signed_attestation, + validate_production_e2e, +) +from echo_certification_forge.signing import Ed25519VerdictSigner + +NOW = datetime(2026, 8, 30, 12, 0, tzinfo=UTC) +SOURCE_SHA = "5" * 40 + + +def _target() -> TargetIdentity: + return TargetIdentity( + tenant_id="echo-github-apps", + target_type="git", + canonical_ref=( + "https://github.com/echoomegaprime/echo-github-autonomy.git@" + SOURCE_SHA + ), + artifact_sha256="a" * 64, + source_commit=SOURCE_SHA, + dependency_sha256="b" * 64, + configuration_sha256="c" * 64, + ) + + +def _environment() -> EnvironmentIdentity: + return EnvironmentIdentity( + **{ + name: str(index) * 64 + for index, name in enumerate(EnvironmentIdentity.__dataclass_fields__, start=1) + } + ) + + +def _payload(*, signing_key_id: str = "ed25519:" + "d" * 32) -> dict: + target = _target() + environment = _environment() + samples = { + login: { + "repository_id": index, + "node_id": f"R_repo_{index}", + "default_branch": "main", + "head_sha": str(index) * 40, + "fingerprint_sha256": str(index) * 64, + } + for index, login in enumerate(ECHO_GITHUB_ACCOUNTS, start=1) + } + accounts = { + login: { + "account_id": account_id, + "enumerated_count": 10 + index, + "upstream_total_count": 10 + index, + "public_count": 5, + "private_count": 5 + index, + "read": True, + "write": True, + "certify": True, + "credential_source": ( + "github_app_installation" + if index == 1 + else "vault_user_token_fallback" + ), + "secret_exposed": False, + } + for index, (login, account_id) in enumerate(ECHO_GITHUB_ACCOUNTS.items(), start=1) + } + fingerprints = { + login: sample["fingerprint_sha256"] for login, sample in samples.items() + } + return { + "schema_version": SCHEMA_VERSION, + "attestation_id": "echo-github-autonomy-live-001", + "profile": ECHO_GITHUB_AUTONOMY_PROFILE, + "target_identity_digest": target.identity_digest, + "environment_identity_digest": environment.identity_digest, + "source_commit": SOURCE_SHA, + "deployment_sha": SOURCE_SHA, + "canonical_target": ECHO_GITHUB_AUTONOMY_CANONICAL_MCP, + "required_checks": sorted(ECHO_GITHUB_AUTONOMY_CHECKS), + "checks": {name: True for name in sorted(ECHO_GITHUB_AUTONOMY_CHECKS)}, + "stability_probe_count": 3, + "tool_count": 30, + "accounts": accounts, + "sample_private_repositories": samples, + "clients": { + name: {"accepted": True, "repository_fingerprints": fingerprints} + for name in sorted(ECHO_CLIENTS) + }, + "observed_at": NOW.isoformat(), + "expires_at": (NOW + timedelta(minutes=30)).isoformat(), + "signing_key_id": signing_key_id, + } + + +def test_echo_github_autonomy_requires_complete_exact_cross_client_e2e() -> None: + valid, reason = validate_production_e2e( + _payload(), _target(), _environment(), now=NOW + timedelta(minutes=1) + ) + assert valid, reason + + +@pytest.mark.parametrize( + ("mutation", "reason"), + [ + (lambda value: value.__setitem__("deployment_sha", "6" * 40), "production_e2e_deployment_sha_mismatch"), + (lambda value: value.__setitem__("tool_count", 27), "production_e2e_tool_count_mismatch"), + ( + lambda value: value["accounts"]["Bmcbob76"].__setitem__("private_count", 0), + "production_e2e_account_reconciliation_failed", + ), + ( + lambda value: value["clients"]["grok"].__setitem__("accepted", False), + "production_e2e_client_not_accepted", + ), + ( + lambda value: value["accounts"]["Bmcbob76"].__setitem__( + "credential_source", "model_config_pat" + ), + "production_e2e_credential_source_invalid", + ), + ( + lambda value: value["accounts"]["Bmcbob76"].__setitem__( + "secret_exposed", True + ), + "production_e2e_secret_boundary_failed", + ), + ( + lambda value: value["sample_private_repositories"]["Bmcbob76"].__setitem__( + "repository_id", True + ), + "production_e2e_sample_repository_id_invalid", + ), + ], +) +def test_incomplete_or_mismatched_e2e_fails_closed(mutation, reason: str) -> None: + payload = _payload() + mutation(payload) + valid, actual = validate_production_e2e( + payload, _target(), _environment(), now=NOW + timedelta(minutes=1) + ) + assert not valid + assert actual == reason + + +def test_stale_e2e_attestation_fails_closed() -> None: + valid, reason = validate_production_e2e( + _payload(), _target(), _environment(), now=NOW + timedelta(hours=2) + ) + assert not valid + assert reason == "production_e2e_attestation_not_current" + + +def test_attestation_loader_requires_a_pinned_collector_key(tmp_path) -> None: + signer = Ed25519VerdictSigner.generate() + payload = _payload(signing_key_id=signer.key_id) + envelope = signer.sign_payload(payload) + envelope_path = tmp_path / "attestation.json" + envelope_path.write_text( + json.dumps( + { + "payload": envelope.payload, + "signature_b64": envelope.signature_b64, + "key_id": envelope.key_id, + "public_key_pem": envelope.public_key_pem, + } + ), + encoding="utf-8", + ) + trusted = tmp_path / "trusted" + trusted.mkdir() + with pytest.raises(ValueError, match="untrusted_signing_key"): + load_signed_attestation(envelope_path, trusted) + (trusted / "collector.pem").write_text(signer.public_key_pem, encoding="ascii") + verified = load_signed_attestation(envelope_path, trusted) + assert verified.payload == payload + assert verified.collector_key_id == signer.key_id + assert len(verified.envelope_sha256) == 64 + + +def test_self_selected_public_key_cannot_replace_the_pinned_collector(tmp_path) -> None: + trusted_signer = Ed25519VerdictSigner.generate() + attacker = Ed25519VerdictSigner.generate() + payload = _payload(signing_key_id=attacker.key_id) + envelope = attacker.sign_payload(payload) + envelope_path = tmp_path / "attestation.json" + envelope_path.write_text( + json.dumps( + { + "payload": envelope.payload, + "signature_b64": envelope.signature_b64, + "key_id": envelope.key_id, + "public_key_pem": envelope.public_key_pem, + } + ), + encoding="utf-8", + ) + trusted = tmp_path / "trusted" + trusted.mkdir() + (trusted / "collector.pem").write_text(trusted_signer.public_key_pem, encoding="ascii") + with pytest.raises(ValueError, match="untrusted_signing_key"): + load_signed_attestation(envelope_path, trusted) diff --git a/tests/test_runner.py b/tests/test_runner.py index 7a16b63..0266497 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -634,7 +634,12 @@ def test_workspace_owner_marker_and_symlink_detection(tmp_path: Path) -> None: ) target = tmp_path / "outside.txt" target.write_text("outside") - os.symlink(target, workspace / "link") + try: + os.symlink(target, workspace / "link") + except OSError as exc: + if os.name == "nt" and getattr(exc, "winerror", None) == 1314: + pytest.skip("Windows symlink privilege is unavailable") + raise with pytest.raises(IsolationFailure, match="symlink_not_allowed"): assert_no_symlinks(workspace) diff --git a/tests/test_t4_live_run.py b/tests/test_t4_live_run.py index 3d7e6fa..02d6784 100644 --- a/tests/test_t4_live_run.py +++ b/tests/test_t4_live_run.py @@ -18,6 +18,7 @@ from echo_certification_forge.models import SignedVerdictEnvelope from echo_certification_forge.run_worker import run from echo_certification_forge.signing import Ed25519VerdictSigner, TrustedPublicKeyRegistry +from production_e2e_support import trusted_generic_production_e2e def _benign(tmp_path: Path) -> Path: @@ -99,7 +100,8 @@ def _run_worker(store, manifest, tenant, source, journey): signer = Ed25519VerdictSigner.generate() result = run("cert-live", tenant, {"type": "local", "path": str(source)}, store=store, manifest=manifest, signer=signer, - entitled=frozenset({tenant}), journey=journey) + entitled=frozenset({tenant}), journey=journey, + production_e2e_provider=trusted_generic_production_e2e) return result, signer diff --git a/tests/test_t4p5_executor.py b/tests/test_t4p5_executor.py index f4cd76d..60dbfc8 100644 --- a/tests/test_t4p5_executor.py +++ b/tests/test_t4p5_executor.py @@ -11,6 +11,7 @@ from echo_certification_forge.canonical import to_utc_iso, utc_now from echo_certification_forge.executor import RunExecutor, StaticEntitlement from echo_certification_forge.signing import Ed25519VerdictSigner +from production_e2e_support import trusted_generic_production_e2e RUN = "cert-exec" @@ -140,7 +141,10 @@ def test_retention_purge_removes_expired_content_but_keeps_signed_verdict( entitlement=StaticEntitlement(frozenset({target.tenant_id})), journey=[sys.executable, "hello.py"], control_attestations={"runner_control_channel": True, - "signing_authority_separation": True}) + "signing_authority_separation": True}, + production_e2e_attestation=trusted_generic_production_e2e( + target, environment + )) assert result.release_verdict == "PRODUCTION_READY", result.blocking_findings verdict_before = store.latest_signed_verdict(RUN, target.tenant_id) diff --git a/tests/test_t4p7_e2e.py b/tests/test_t4p7_e2e.py index 025c9a8..e733c57 100644 --- a/tests/test_t4p7_e2e.py +++ b/tests/test_t4p7_e2e.py @@ -15,6 +15,7 @@ from echo_certification_forge.executor import RunExecutor, StaticEntitlement from echo_certification_forge.models import SignedVerdictEnvelope from echo_certification_forge.signing import Ed25519VerdictSigner, TrustedPublicKeyRegistry +from production_e2e_support import trusted_generic_production_e2e RUN = "cert-e2e" @@ -46,6 +47,7 @@ def test_e2e_benign_target_certified_and_deploy_gate_allows(store, manifest, tar entitlement=StaticEntitlement(frozenset({target.tenant_id})), journey=[sys.executable, "hello.py"], control_attestations={"runner_control_channel": True, "signing_authority_separation": True}, + production_e2e_attestation=trusted_generic_production_e2e(target, environment), ) # certified through the full path diff --git a/tests/test_t4p8_sandbox.py b/tests/test_t4p8_sandbox.py index b0d3d45..a1e9dc5 100644 --- a/tests/test_t4p8_sandbox.py +++ b/tests/test_t4p8_sandbox.py @@ -15,6 +15,7 @@ DEFAULT_IMAGE, DockerSandbox, SandboxError, sandboxed_journey_runner, ) from echo_certification_forge.signing import Ed25519VerdictSigner +from production_e2e_support import trusted_generic_production_e2e def test_build_command_has_all_hardening_flags(tmp_path): @@ -72,6 +73,7 @@ def fake_runner(argv, workdir): entitlement=StaticEntitlement(frozenset({target.tenant_id})), journey=["python3", "hello.py"], journey_runner=fake_runner, control_attestations={"runner_control_channel": True, "signing_authority_separation": True}, + production_e2e_attestation=trusted_generic_production_e2e(target, environment), ) assert calls and calls[0][0] == ["python3", "hello.py"] # the injected runner was used assert result.release_verdict == "PRODUCTION_READY" From 61cebbb7b8dbd84e1cb444df5e69acb6b12ebaa1 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Sun, 30 Aug 2026 05:43:38 -0700 Subject: [PATCH 02/39] Test the P6 gate with signed production E2E --- artifacts/p6_acceptance.json | 22 +++++++++--------- artifacts/p6_acceptance.summary.json | 2 +- scripts/p6_acceptance.py | 34 ++++++++++++++++++++++++++-- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/artifacts/p6_acceptance.json b/artifacts/p6_acceptance.json index cfbd348..13c22df 100644 --- a/artifacts/p6_acceptance.json +++ b/artifacts/p6_acceptance.json @@ -1,8 +1,8 @@ { "phase": "P6", "title": "deployment enforcement and platform integration \u2014 live local acceptance", - "generated_at": "2026-07-26T02:18:04.411111Z", - "python": "3.13.0 (tags/v3.13.0:60403a5, Oct 7 2024, 09:38:07) [MSC v.1941 64 bit (AMD64)]", + "generated_at": "2026-08-30T12:40:31.960114Z", + "python": "3.12.10 (tags/v3.12.10:0cc8128, Apr 8 2025, 12:21:36) [MSC v.1943 64 bit (AMD64)]", "passed": true, "checks": [ { @@ -11,7 +11,7 @@ "passed": true, "detail": { "exit": 2, - "stdout": "{\"admission_id\": \"dep-3413cd5f5334d3b106b9ce81f7c7a26b\", \"allowed\": false, \"artifact_sha256\": \"cc9a65d96c25c8f96ba720e01628b734d6df4f5a1057238506ab6d504fb998fd\", \"chain_hash\": \"ab88eb7a583dc68353ca39d53cd4e37d9737ac4b6fa0e6c9b0ab8c659325a64d\", \"created_at\": \"2026-07-26T02:17:59.537952Z\", \"deployment_environment\": \"production\", \"deployment_id\": \"deploy-a1\", \"reasons\": [\"artifact_not_certified\", \"staging_acceptance_missing\"], \"rollback_candidate\": null, \"run_id\": null, \"tenant_id\": \"tenant-alpha\"}" + "stdout": "{\"admission_id\": \"dep-39149ed9c3643c355bbe8f4f09eec260\", \"allowed\": false, \"artifact_sha256\": \"cc9a65d96c25c8f96ba720e01628b734d6df4f5a1057238506ab6d504fb998fd\", \"chain_hash\": \"ff5f335a43355fdc85e2b64c11e7429ad3aa9070dca78655a74bf0d929da28b0\", \"created_at\": \"2026-08-30T12:40:27.987414Z\", \"deployment_environment\": \"production\", \"deployment_id\": \"deploy-a1\", \"reasons\": [\"artifact_not_certified\", \"staging_acceptance_missing\"], \"rollback_candidate\": null, \"run_id\": null, \"tenant_id\": \"tenant-alpha\"}" } }, { @@ -29,7 +29,7 @@ "passed": true, "detail": { "exit": 2, - "stdout": "{\"admission_id\": \"dep-f49980b48837ae58dec6fa3ffbc216e4\", \"allowed\": false, \"artifact_sha256\": \"11d7d626a19af7cf0b64c0862b3214a15fac77e3aa1400d70e6902f1a77fdb86\", \"chain_hash\": \"40c1d83374649745ee6fcd3c42c58ce5b6257e362aa6e22954f224c7284b61f5\", \"created_at\": \"2026-07-26T02:17:59.744469Z\", \"deployment_environment\": \"staging\", \"deployment_id\": \"deploy-a2\", \"reasons\": [\"artifact_not_certified\"], \"rollback_candidate\": null, \"run_id\": null, \"tenant_id\": \"tenant-alpha\"}" + "stdout": "{\"admission_id\": \"dep-de226c922ca25e4fe770043fa5cce489\", \"allowed\": false, \"artifact_sha256\": \"11d7d626a19af7cf0b64c0862b3214a15fac77e3aa1400d70e6902f1a77fdb86\", \"chain_hash\": \"1b08dbf5ba6527bbb6b743ab658e726bb487454d6ab95bd8ec1ec7889764501a\", \"created_at\": \"2026-08-30T12:40:28.129813Z\", \"deployment_environment\": \"staging\", \"deployment_id\": \"deploy-a2\", \"reasons\": [\"artifact_not_certified\"], \"rollback_candidate\": null, \"run_id\": null, \"tenant_id\": \"tenant-alpha\"}" } }, { @@ -38,7 +38,7 @@ "passed": true, "detail": { "exit": 2, - "stdout": "{\"admission_id\": \"dep-8ac38cbd2ade07ff1bb5c59d63d2582c\", \"allowed\": false, \"artifact_sha256\": \"67b7548d601f8ca9e8689fced7d40d8f407894ac7d6a2c09d2e524ee5514e7b0\", \"chain_hash\": \"5d31c6a43518753dc06259bee965f1e8fb91e4114f86c47bf61adde8a84ff4dd\", \"created_at\": \"2026-07-26T02:17:59.912909Z\", \"deployment_environment\": \"production\", \"deployment_id\": \"deploy-a4\", \"reasons\": [\"staging_acceptance_missing\"], \"rollback_candidate\": null, \"run_id\": \"cert-p6-v1\", \"tenant_id\": \"tenant-alpha\"}" + "stdout": "{\"admission_id\": \"dep-393b3f1c13aeaf883a659c67fe3a6062\", \"allowed\": false, \"artifact_sha256\": \"67b7548d601f8ca9e8689fced7d40d8f407894ac7d6a2c09d2e524ee5514e7b0\", \"chain_hash\": \"4adc5c6ca6c45d5664c517904d6b86304b4ae573fa525cef6397e6a315d30422\", \"created_at\": \"2026-08-30T12:40:28.260911Z\", \"deployment_environment\": \"production\", \"deployment_id\": \"deploy-a4\", \"reasons\": [\"staging_acceptance_missing\"], \"rollback_candidate\": null, \"run_id\": \"cert-p6-v1\", \"tenant_id\": \"tenant-alpha\"}" } }, { @@ -57,8 +57,8 @@ "passed": true, "detail": { "failure_candidate": { - "admission_id": "dep-c3f042d865f9fcd3c85e9d0a1c91484a", - "admitted_at": "2026-07-26T02:18:00.438507Z", + "admission_id": "dep-88177f711ac3ea76d71d3f5657753a8a", + "admitted_at": "2026-08-30T12:40:28.640308Z", "artifact_sha256": "67b7548d601f8ca9e8689fced7d40d8f407894ac7d6a2c09d2e524ee5514e7b0", "deployment_id": "deploy-a3-prd", "run_id": "cert-p6-v1" @@ -66,12 +66,12 @@ "rollback_target": { "artifact_sha256": "67b7548d601f8ca9e8689fced7d40d8f407894ac7d6a2c09d2e524ee5514e7b0", "run_id": "cert-p6-v1", - "admission_id": "dep-c3f042d865f9fcd3c85e9d0a1c91484a", + "admission_id": "dep-88177f711ac3ea76d71d3f5657753a8a", "deployment_id": "deploy-a3-prd", - "admitted_at": "2026-07-26T02:18:00.438507Z" + "admitted_at": "2026-08-30T12:40:28.640308Z" }, "rolled_back": { - "admission_id": "dep-d1f547a64a4ddb086ca7a521e0d1a4ce", + "admission_id": "dep-368a34d3be4daeb9870a2b93e41ebf45", "artifact_sha256": "6cf9eb2e75fbaf4037393188b98be82a05499dcfc0c45a6eeeb840bd7f775c20", "deployment_environment": "production", "detail": "restored v1", @@ -87,7 +87,7 @@ "passed": true, "detail": { "exit": 2, - "stdout": "{\"admission_id\": \"dep-4ec864d16bf5447abf4145975d62ed0a\", \"allowed\": false, \"artifact_sha256\": \"67b7548d601f8ca9e8689fced7d40d8f407894ac7d6a2c09d2e524ee5514e7b0\", \"chain_hash\": \"928bee7494d599c95422e4d3f6c8db2a7e80c2ec1ff5837056aead46be211115\", \"created_at\": \"2026-07-26T02:18:01.842335Z\", \"deployment_environment\": \"production\", \"deployment_id\": \"deploy-a5\", \"reasons\": [\"verdict_revoked\"], \"rollback_candidate\": null, \"run_id\": \"cert-p6-v1\", \"tenant_id\": \"tenant-alpha\"}" + "stdout": "{\"admission_id\": \"dep-1d051283684978484ff174d2f3cc5b7b\", \"allowed\": false, \"artifact_sha256\": \"67b7548d601f8ca9e8689fced7d40d8f407894ac7d6a2c09d2e524ee5514e7b0\", \"chain_hash\": \"8cbb0ce94045d583aaaaa7e150f5a997759f188f1de824d89f973711ef9a5754\", \"created_at\": \"2026-08-30T12:40:29.545903Z\", \"deployment_environment\": \"production\", \"deployment_id\": \"deploy-a5\", \"reasons\": [\"verdict_revoked\"], \"rollback_candidate\": null, \"run_id\": \"cert-p6-v1\", \"tenant_id\": \"tenant-alpha\"}" } }, { diff --git a/artifacts/p6_acceptance.summary.json b/artifacts/p6_acceptance.summary.json index 2c77777..ac231fd 100644 --- a/artifacts/p6_acceptance.summary.json +++ b/artifacts/p6_acceptance.summary.json @@ -3,5 +3,5 @@ "passed": true, "checks_total": 12, "checks_passed": 12, - "generated_at": "2026-07-26T02:18:04.411111Z" + "generated_at": "2026-08-30T12:40:31.960114Z" } diff --git a/scripts/p6_acceptance.py b/scripts/p6_acceptance.py index 05c4429..15490fe 100644 --- a/scripts/p6_acceptance.py +++ b/scripts/p6_acceptance.py @@ -41,7 +41,7 @@ import time import urllib.parse import urllib.request -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] @@ -64,6 +64,12 @@ VerdictLifecycleEvent, ) from echo_certification_forge.policy import RuleManifest # noqa: E402 +from echo_certification_forge.production_e2e import ( # noqa: E402 + BASE_CHECKS, + GENERIC_PROFILE, + SCHEMA_VERSION, + verify_signed_attestation, +) from echo_certification_forge.release_hooks import ( # noqa: E402 SIGNATURE_HEADER, TIMESTAMP_HEADER, @@ -187,6 +193,9 @@ def check(check_id: str, description: str, passed: bool, detail: object) -> None signer = Ed25519VerdictSigner.generate() trusted = TrustedPublicKeyRegistry.empty() trusted.add_pem(signer.public_key_pem) + e2e_signer = Ed25519VerdictSigner.generate() + e2e_trusted = TrustedPublicKeyRegistry.empty() + e2e_trusted.add_pem(e2e_signer.public_key_pem) environment = EnvironmentIdentity( runner_image_sha256=digest("runner"), @@ -205,6 +214,26 @@ def certify(run_id: str, target: TargetIdentity) -> None: workdir = tmp_path / f"src-{run_id}" workdir.mkdir() (workdir / "hello.py").write_text("print('service ok')\n", encoding="utf-8") + observed_at = utc_now() + e2e_payload = { + "schema_version": SCHEMA_VERSION, + "attestation_id": f"p6-e2e-{target.identity_digest[:16]}", + "profile": GENERIC_PROFILE, + "target_identity_digest": target.identity_digest, + "environment_identity_digest": environment.identity_digest, + "source_commit": target.source_commit, + "deployment_sha": target.source_commit, + "canonical_target": target.canonical_ref, + "required_checks": sorted(BASE_CHECKS), + "checks": {name: True for name in sorted(BASE_CHECKS)}, + "stability_probe_count": 3, + "observed_at": to_utc_iso(observed_at), + "expires_at": to_utc_iso(observed_at + timedelta(minutes=30)), + "signing_key_id": e2e_signer.key_id, + } + production_e2e = verify_signed_attestation( + e2e_signer.sign_payload(e2e_payload), e2e_trusted + ) result = RunExecutor(store, manifest, signer).execute( run_id, target.tenant_id, @@ -215,6 +244,7 @@ def certify(run_id: str, target: TargetIdentity) -> None: "runner_control_channel": True, "signing_authority_separation": True, }, + production_e2e_attestation=production_e2e, ) if result.release_verdict != "PRODUCTION_READY": raise RuntimeError(f"certification failed: {result.blocking_findings}") @@ -225,7 +255,7 @@ def make_target(label: str) -> TargetIdentity: target_type="container", canonical_ref=f"registry.echo/app@{label}", artifact_sha256=digest(label), - source_commit="abc123def456", + source_commit=digest("source-commit")[:40], dependency_sha256=digest("dependencies"), configuration_sha256=digest("configuration"), ) From 7a7a1c92b9ee4978bf27693620f4c503e524f51e Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Sun, 30 Aug 2026 05:57:39 -0700 Subject: [PATCH 03/39] Bind deployment checks to the effective database --- deploy/deploy_forge.sh | 11 ++++++----- tests/test_deploy_gate.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/deploy/deploy_forge.sh b/deploy/deploy_forge.sh index 2e649b5..72ee784 100644 --- a/deploy/deploy_forge.sh +++ b/deploy/deploy_forge.sh @@ -173,6 +173,7 @@ set -a set +a PROD_PEPPER="${ECHO_CERTFORGE_API_KEY_PEPPER:-}" STAGING_PEPPER="${ECHO_CERTFORGE_STAGING_API_KEY_PEPPER:-}" +PROD_DB_PATH="${ECHO_CERTFORGE_DB:-$STATE_ROOT/certforge.sqlite3}" test "${#PROD_PEPPER}" -ge 32 || { echo "!! production ECHO_CERTFORGE_API_KEY_PEPPER must be at least 32 bytes" exit 1 @@ -305,7 +306,7 @@ if sudo test -f "$DISPATCH_RELEASE_DROPIN"; then sudo cat "$DISPATCH_RELEASE_DROPIN" >"$DISPATCH_RELEASE_DROPIN_BACKUP" HAD_DISPATCH_RELEASE_DROPIN=1 fi -DB_PATH="$STATE_ROOT/certforge.sqlite3" +DB_PATH="$PROD_DB_PATH" DB_BACKUP="$STATE_ROOT/deploy-scratch/echo-certforge-db.$RELEASE_ID" HAD_DB=0 DB_SNAPSHOT_READY=0 @@ -503,7 +504,7 @@ Type=simple User=forge WorkingDirectory=$CURRENT_LINK Environment=PYTHONUNBUFFERED=1 -Environment=ECHO_CERTFORGE_DB=$STATE_ROOT/certforge.sqlite3 +Environment=ECHO_CERTFORGE_DB=$DB_PATH Environment=ECHO_CERTFORGE_EVIDENCE_ROOT=$STATE_ROOT/evidence Environment=ECHO_CERTFORGE_POLICY=$CURRENT_LINK/policies/mandatory-rules.v2.json Environment=ECHO_CERTFORGE_SUBSCRIBER_POLICY=$CURRENT_LINK/policies/subscriber-governance.v1.json @@ -546,7 +547,7 @@ Type=simple User=forge WorkingDirectory=$CURRENT_LINK Environment=PYTHONUNBUFFERED=1 -Environment=ECHO_CERTFORGE_DB=$STATE_ROOT/certforge.sqlite3 +Environment=ECHO_CERTFORGE_DB=$DB_PATH Environment=ECHO_CERTFORGE_EVIDENCE_ROOT=$STATE_ROOT/evidence Environment=ECHO_CERTFORGE_POLICY=$CURRENT_LINK/policies/mandatory-rules.v2.json Environment=ECHO_CERTFORGE_SUBSCRIBER_POLICY=$CURRENT_LINK/policies/subscriber-governance.v1.json @@ -593,7 +594,7 @@ for _ in $(seq 1 40); do } sleep 0.5 done -if [ "$ready" != 1 ] || ! ECHO_CERTFORGE_DB="$STATE_ROOT/certforge.sqlite3" \ +if [ "$ready" != 1 ] || ! ECHO_CERTFORGE_DB="$DB_PATH" \ ECHO_CERTFORGE_SUBSCRIBER_POLICY="$RELEASE_DIR/policies/subscriber-governance.v1.json" \ ECHO_CERTFORGE_API_KEY_PEPPER="$PROD_PEPPER" \ ECHO_CERTFORGE_EXPECT_PRODUCT_READY=1 \ @@ -621,7 +622,7 @@ if [ "$dispatcher_ready" != 1 ]; then exit 1 fi if [ "$ADAPTER_MODE" = required ]; then - ECHO_CERTFORGE_DB="$STATE_ROOT/certforge.sqlite3" \ + ECHO_CERTFORGE_DB="$DB_PATH" \ ECHO_CERTFORGE_SUBSCRIBER_POLICY="$RELEASE_DIR/policies/subscriber-governance.v1.json" \ ECHO_CERTFORGE_API_KEY_PEPPER="$PROD_PEPPER" \ "$RELEASE_DIR/.venv/bin/python" "$RELEASE_DIR/deploy/smoke_dispatch.py" \ diff --git a/tests/test_deploy_gate.py b/tests/test_deploy_gate.py index 8a0a1ed..67f7b9c 100644 --- a/tests/test_deploy_gate.py +++ b/tests/test_deploy_gate.py @@ -96,6 +96,17 @@ def test_deploy_gate_snapshots_and_restores_persistent_database() -> None: assert 'DB_SNAPSHOT_READY=1' in DEPLOY_SCRIPT +def test_deploy_gate_uses_the_effective_production_database_everywhere() -> None: + assert ( + 'PROD_DB_PATH="${ECHO_CERTFORGE_DB:-$STATE_ROOT/certforge.sqlite3}"' + in DEPLOY_SCRIPT + ) + assert 'DB_PATH="$PROD_DB_PATH"' in DEPLOY_SCRIPT + assert DEPLOY_SCRIPT.count("Environment=ECHO_CERTFORGE_DB=$DB_PATH") == 2 + assert DEPLOY_SCRIPT.count('ECHO_CERTFORGE_DB="$DB_PATH" \\') == 2 + assert 'ECHO_CERTFORGE_DB="$STATE_ROOT/certforge.sqlite3"' not in DEPLOY_SCRIPT + + def test_deploy_gate_restores_prior_service_lifecycle() -> None: assert ( 'PREV_ENABLED="$(systemctl is-enabled "$SERVICE.service"' From 917ffe933b2c6b36bbbaab5f621b1c5889b2fd25 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Sun, 30 Aug 2026 06:08:08 -0700 Subject: [PATCH 04/39] Inherit the live CertForge database during deploy --- deploy/deploy_forge.sh | 23 ++++++++++++++++++++++- tests/test_deploy_gate.py | 10 ++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/deploy/deploy_forge.sh b/deploy/deploy_forge.sh index 72ee784..c9c3b34 100644 --- a/deploy/deploy_forge.sh +++ b/deploy/deploy_forge.sh @@ -173,7 +173,28 @@ set -a set +a PROD_PEPPER="${ECHO_CERTFORGE_API_KEY_PEPPER:-}" STAGING_PEPPER="${ECHO_CERTFORGE_STAGING_API_KEY_PEPPER:-}" -PROD_DB_PATH="${ECHO_CERTFORGE_DB:-$STATE_ROOT/certforge.sqlite3}" +PROD_DB_PATH="${ECHO_CERTFORGE_DB:-}" +if [ -z "$PROD_DB_PATH" ]; then + CURRENT_SERVICE_PID="$(systemctl show "$SERVICE.service" --property MainPID --value 2>/dev/null || true)" + if [ "${CURRENT_SERVICE_PID:-0}" -gt 1 ] 2>/dev/null && \ + [ -r "/proc/$CURRENT_SERVICE_PID/environ" ]; then + PROD_DB_PATH="$(python3 - "$CURRENT_SERVICE_PID" <<'PY' +from pathlib import Path +import sys + +for entry in Path(f"/proc/{sys.argv[1]}/environ").read_bytes().split(b"\0"): + if entry.startswith(b"ECHO_CERTFORGE_DB="): + print(entry.split(b"=", 1)[1].decode("utf-8")) + break +PY +)" + fi +fi +PROD_DB_PATH="${PROD_DB_PATH:-$STATE_ROOT/certforge.sqlite3}" +case "$(realpath -m "$PROD_DB_PATH")" in + "$STATE_ROOT"/*|/mnt/documentation/echo/certforge/*) ;; + *) echo "!! effective production database is outside an approved state root"; exit 1 ;; +esac test "${#PROD_PEPPER}" -ge 32 || { echo "!! production ECHO_CERTFORGE_API_KEY_PEPPER must be at least 32 bytes" exit 1 diff --git a/tests/test_deploy_gate.py b/tests/test_deploy_gate.py index 67f7b9c..9f83072 100644 --- a/tests/test_deploy_gate.py +++ b/tests/test_deploy_gate.py @@ -97,10 +97,12 @@ def test_deploy_gate_snapshots_and_restores_persistent_database() -> None: def test_deploy_gate_uses_the_effective_production_database_everywhere() -> None: - assert ( - 'PROD_DB_PATH="${ECHO_CERTFORGE_DB:-$STATE_ROOT/certforge.sqlite3}"' - in DEPLOY_SCRIPT - ) + assert 'PROD_DB_PATH="${ECHO_CERTFORGE_DB:-}"' in DEPLOY_SCRIPT + assert 'systemctl show "$SERVICE.service" --property MainPID --value' in DEPLOY_SCRIPT + assert 'Path(f"/proc/{sys.argv[1]}/environ")' in DEPLOY_SCRIPT + assert 'entry.startswith(b"ECHO_CERTFORGE_DB=")' in DEPLOY_SCRIPT + assert 'PROD_DB_PATH="${PROD_DB_PATH:-$STATE_ROOT/certforge.sqlite3}"' in DEPLOY_SCRIPT + assert '/mnt/documentation/echo/certforge/*' in DEPLOY_SCRIPT assert 'DB_PATH="$PROD_DB_PATH"' in DEPLOY_SCRIPT assert DEPLOY_SCRIPT.count("Environment=ECHO_CERTFORGE_DB=$DB_PATH") == 2 assert DEPLOY_SCRIPT.count('ECHO_CERTFORGE_DB="$DB_PATH" \\') == 2 From fba53a4bd3a251d939d8a71f174a80c5f47b786e Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Sun, 30 Aug 2026 06:21:24 -0700 Subject: [PATCH 05/39] Make production deploy smoke prove fail-closed E2E gate --- deploy/smoke_dispatch.py | 39 +++++++++++++----- tests/test_deploy_dispatch_smoke_contract.py | 43 ++++++++++++++++++++ 2 files changed, 72 insertions(+), 10 deletions(-) create mode 100644 tests/test_deploy_dispatch_smoke_contract.py diff --git a/deploy/smoke_dispatch.py b/deploy/smoke_dispatch.py index b0410c1..1661749 100644 --- a/deploy/smoke_dispatch.py +++ b/deploy/smoke_dispatch.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 -"""Production subscriber-to-dispatcher certification smoke. +"""Production subscriber-to-dispatcher fail-closed certification smoke. This test runs only after the real API and dispatcher are active. It submits an exact-identity -local target through the customer HTTP contract, waits for the durable dispatcher, and verifies -the signed PRODUCTION_READY verdict and append-only evidence through the public API. +local target through the customer HTTP contract without a trusted production-E2E attestation, +waits for the durable dispatcher, and verifies that the service signs ``NOT_READY`` while keeping +the append-only evidence and public signature-verification paths healthy. A deploy smoke must not +manufacture the external evidence that the production gate is intended to require. """ from __future__ import annotations @@ -65,6 +67,22 @@ def _require(condition: bool, message: str, detail: object | None = None) -> Non print(f" [ok ] {message}") +def _require_attestation_gate_closed(terminal: dict) -> None: + """Prove an unattested run cannot be promoted by the production dispatcher.""" + + _require( + terminal.get("release_verdict") == "NOT_READY", + "unattested customer run is NOT_READY", + terminal.get("release_verdict"), + ) + production_e2e = terminal.get("production_e2e") + _require( + isinstance(production_e2e, dict) and production_e2e.get("verified") is False, + "unattested customer run has no verified production E2E", + production_e2e, + ) + + def main() -> int: base = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:8309" db_path = os.environ.get("ECHO_CERTFORGE_DB") @@ -177,11 +195,7 @@ def main() -> int: time.sleep(1) _require(terminal.get("state") == "COMPLETE", "dispatcher completed the customer run", terminal) _require(terminal.get("run_outcome") == "COMPLETE", "customer run outcome is COMPLETE") - _require( - terminal.get("release_verdict") == "PRODUCTION_READY", - "customer received PRODUCTION_READY", - terminal.get("release_verdict"), - ) + _require_attestation_gate_closed(terminal) _require( terminal.get("environment_identity_digest") == environment_digest, "verdict retained the published environment identity", @@ -198,7 +212,12 @@ def main() -> int: ) _require(verdict_status == 200, "signed verdict is retrievable", verdict_status) payload = verdict.get("payload", {}) - _require(payload.get("release_verdict") == "PRODUCTION_READY", "signed verdict is PRODUCTION_READY") + _require(payload.get("release_verdict") == "NOT_READY", "signed verdict is NOT_READY") + _require( + "production_e2e_attestation_missing" in payload.get("reasons", []), + "signed verdict records the missing production E2E attestation", + payload.get("reasons"), + ) _require(payload.get("environment_identity_digest") == environment_digest, "signed verdict binds environment") verify_status, verified = _request( @@ -209,7 +228,7 @@ def main() -> int: "public verdict verification succeeds", verified, ) - print(f"PRODUCTION DISPATCH SMOKE GREEN - run_id={run_id}") + print(f"PRODUCTION DISPATCH FAIL-CLOSED SMOKE GREEN - run_id={run_id}") return 0 finally: shutil.rmtree(target_root, ignore_errors=True) diff --git a/tests/test_deploy_dispatch_smoke_contract.py b/tests/test_deploy_dispatch_smoke_contract.py new file mode 100644 index 0000000..2118fa4 --- /dev/null +++ b/tests/test_deploy_dispatch_smoke_contract.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location( + "certforge_deploy_smoke_dispatch", ROOT / "deploy" / "smoke_dispatch.py" +) +assert SPEC is not None and SPEC.loader is not None +SMOKE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(SMOKE) + + +def test_production_dispatch_smoke_requires_fail_closed_unattested_verdict() -> None: + SMOKE._require_attestation_gate_closed( # noqa: SLF001 + { + "release_verdict": "NOT_READY", + "production_e2e": {"verified": False}, + } + ) + + +@pytest.mark.parametrize( + "terminal", + [ + { + "release_verdict": "PRODUCTION_READY", + "production_e2e": {"verified": False}, + }, + { + "release_verdict": "NOT_READY", + "production_e2e": {"verified": True}, + }, + {"release_verdict": "NOT_READY"}, + ], +) +def test_production_dispatch_smoke_rejects_open_or_ambiguous_gate(terminal: dict) -> None: + with pytest.raises(RuntimeError): + SMOKE._require_attestation_gate_closed(terminal) # noqa: SLF001 From c7982ee3de2ccd9e8b84da86b3021652d7e2ccab Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Sun, 30 Aug 2026 06:27:47 -0700 Subject: [PATCH 06/39] Preserve missing production E2E failure reason --- src/echo_certification_forge/verdict.py | 9 ++++++ tests/test_evidence_and_verdict.py | 42 +++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/src/echo_certification_forge/verdict.py b/src/echo_certification_forge/verdict.py index 457751c..82c1e12 100644 --- a/src/echo_certification_forge/verdict.py +++ b/src/echo_certification_forge/verdict.py @@ -88,6 +88,15 @@ def evaluate( reasons.append("production_e2e_identity_unavailable") elif e2e_result is None: reasons.append("production_e2e_attestation_missing") + elif ( + not e2e_result.passed + and e2e_result.details.get("validation") + == "production_e2e_attestation_missing" + ): + # The executor records a fail-closed rule row even when no attestation was supplied. + # Preserve that explicit cause instead of re-validating the diagnostic placeholder as + # though it were a malformed attestation payload. + reasons.append("production_e2e_attestation_missing") else: e2e_valid, e2e_reason = validate_production_e2e( e2e_result.details, diff --git a/tests/test_evidence_and_verdict.py b/tests/test_evidence_and_verdict.py index 2265075..c616c7c 100644 --- a/tests/test_evidence_and_verdict.py +++ b/tests/test_evidence_and_verdict.py @@ -135,6 +135,48 @@ def test_source_and_rule_rows_without_production_e2e_never_get_ready( assert "production_e2e_attestation_missing" in decision.reasons +def test_failed_executor_rule_preserves_missing_attestation_reason( + store, manifest, target, environment +): + run_id = register(store, manifest, target, environment) + for index, rule in enumerate(manifest.rules, start=1): + artifact_id = f"executor-evidence-{index:02d}" + store.append_artifact( + run_id, + target.tenant_id, + artifact_id, + json.dumps({"rule": rule.id}, sort_keys=True).encode(), + "application/json", + "executor-regression", + RedactionStatus.COMPLETE, + ) + is_production_e2e = rule.id == "production_e2e" + store.record_rule_result( + run_id, + target.tenant_id, + RuleResult( + rule.id, + not is_production_e2e, + (artifact_id,), + ( + {"validation": "production_e2e_attestation_missing"} + if is_production_e2e + else {"source": "executor-regression"} + ), + ), + ) + store.set_run_outcome(run_id, target.tenant_id, RunOutcome.COMPLETE) + + signer = Ed25519VerdictSigner.generate() + decision = DeterministicVerdictEngine().evaluate( + store, run_id, target.tenant_id, manifest, signer.key_id + ) + + assert decision.release_verdict is ReleaseVerdict.NOT_READY + assert "production_e2e_attestation_missing" in decision.reasons + assert "production_e2e_schema_invalid" not in decision.reasons + + def test_bare_database_rows_cannot_manufacture_ready(store, manifest, target, environment): run_id = register(store, manifest, target, environment) now = "2026-07-16T00:00:00Z" From 3bfe40b486520be31e4bc94cb83740dbc8b224be Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Sun, 30 Aug 2026 06:38:58 -0700 Subject: [PATCH 07/39] Canonicalize the GitHub App publish capability --- .echo/sdk.json | 1 + contracts/certforge-sdk-capabilities.v1.json | 29 +++++++++++++++++++- scripts/certification_journey.py | 4 +-- scripts/generate_certforge_sdk_contract.py | 10 +++++++ scripts/master_acceptance.py | 6 ++-- scripts/register_certforge_caps.sql | 6 ++++ scripts/register_certforge_sdk_schemas.sql | 5 ++-- tests/test_echo_sdk_manifest.py | 2 +- tests/test_sdk_capability_contract.py | 10 ++++++- 9 files changed, 63 insertions(+), 10 deletions(-) diff --git a/.echo/sdk.json b/.echo/sdk.json index 2d1fe37..f856e69 100644 --- a/.echo/sdk.json +++ b/.echo/sdk.json @@ -46,6 +46,7 @@ "echo.certforge.projects.create", "echo.certforge.projects.list", "echo.certforge.public_verification", + "echo.certforge.publish", "echo.certforge.r5.status", "echo.certforge.r5.submit_async", "echo.certforge.run", diff --git a/contracts/certforge-sdk-capabilities.v1.json b/contracts/certforge-sdk-capabilities.v1.json index e3522ea..66e2950 100644 --- a/contracts/certforge-sdk-capabilities.v1.json +++ b/contracts/certforge-sdk-capabilities.v1.json @@ -1412,6 +1412,33 @@ "type": "object" } }, + "echo.certforge.publish": { + "input_schema": { + "additionalProperties": false, + "properties": { + "command": { + "maxLength": 64, + "minLength": 1, + "type": "string" + }, + "run_id": { + "type": "string" + } + }, + "required": [ + "command", + "run_id" + ], + "type": "object" + }, + "operation": "POST /v1/subscriber/certifications/{run_id}/publish", + "output_schema": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + } + }, "echo.certforge.r5.status": { "input_schema": { "additionalProperties": false, @@ -2295,7 +2322,7 @@ } } }, - "capability_count": 60, + "capability_count": 61, "policy": "Every Certification Forge SDK action has a command-bearing strict input schema and an output schema derived from its production route contract.", "schema_version": "1.0.0" } diff --git a/scripts/certification_journey.py b/scripts/certification_journey.py index 38ca793..bf48bc6 100644 --- a/scripts/certification_journey.py +++ b/scripts/certification_journey.py @@ -62,8 +62,8 @@ def main() -> int: sdk = json.loads((ROOT / ".echo" / "sdk.json").read_text(encoding="utf-8")) capabilities = sdk.get("capabilities", []) - require(len(capabilities) == 60, "SDK capability declaration must contain exactly 60 entries") - require(len(set(capabilities)) == 60, "SDK capability declaration contains duplicates") + require(len(capabilities) == 61, "SDK capability declaration must contain exactly 61 entries") + require(len(set(capabilities)) == 61, "SDK capability declaration contains duplicates") require(all(isinstance(item, str) and item.startswith("echo.") for item in capabilities), "SDK capability declaration contains an invalid identifier") diff --git a/scripts/generate_certforge_sdk_contract.py b/scripts/generate_certforge_sdk_contract.py index 1537784..b061f51 100644 --- a/scripts/generate_certforge_sdk_contract.py +++ b/scripts/generate_certforge_sdk_contract.py @@ -216,6 +216,16 @@ def build_contract() -> dict[str, Any]: "input_schema": _operation_input(operation, components), "output_schema": _operation_output(operation, components), } + # Keep the least-privileged publishing capability used by the Certification Forge GitHub + # App as a first-class compatibility alias. It targets the same fail-closed route as the + # tier-2 administrative capability, but the service still re-evaluates the exact signed + # PRODUCTION_READY verdict before publishing public verification material. + publish = capabilities["echo.certforge.admin.publish"] + capabilities["echo.certforge.publish"] = { + "operation": publish["operation"], + "input_schema": publish["input_schema"], + "output_schema": publish["output_schema"], + } capabilities.update(_manual_capabilities()) return { "schema_version": "1.0.0", diff --git a/scripts/master_acceptance.py b/scripts/master_acceptance.py index e9f4a2e..4048db6 100644 --- a/scripts/master_acceptance.py +++ b/scripts/master_acceptance.py @@ -72,10 +72,10 @@ def _p7(value: dict[str, Any]) -> None: def _sdk(value: dict[str, Any]) -> None: capabilities = value.get("capabilities") - _require(value.get("capability_count") == 60, "SDK contract must contain exactly 60 capabilities") - _require(isinstance(capabilities, dict) and len(capabilities) == 60, "SDK capability rows are incomplete") + _require(value.get("capability_count") == 61, "SDK contract must contain exactly 61 capabilities") + _require(isinstance(capabilities, dict) and len(capabilities) == 61, "SDK capability rows are incomplete") names = list(capabilities) - _require(len(set(names)) == 60, "SDK capability IDs are not unique") + _require(len(set(names)) == 61, "SDK capability IDs are not unique") _require(all(isinstance(name, str) and name.startswith("echo.cert") for name in names), "SDK capability namespace drift") _require( all( diff --git a/scripts/register_certforge_caps.sql b/scripts/register_certforge_caps.sql index 80dbd57..6969503 100644 --- a/scripts/register_certforge_caps.sql +++ b/scripts/register_certforge_caps.sql @@ -136,6 +136,12 @@ VALUES '{"type":"object","required":["run_id"],"properties":{"run_id":{"type":"string","minLength":1,"maxLength":128}},"additionalProperties":false}'::jsonb, 'certforge.admin.mutate', 2, '{"X-Tenant-ID":"org-echo-sovereign","Authorization":"vault:certforge.desktop_admin_api_key"}'::jsonb, 15, 'active', 'unknown'), + ('echo.certforge.publish', + 'Certification Forge: publish public-only verification material for one already-current, signed, production-ready run. The service re-evaluates the deployment gate; this capability cannot change evidence, policy, verdicts, or lifecycle state.', + 'http', 'http://127.0.0.1:8309/v1/subscriber/certifications/{run_id}/publish', 'POST', 'path', 'forge', + '{"type":"object","required":["run_id"],"properties":{"run_id":{"type":"string","minLength":1,"maxLength":128}},"additionalProperties":false}'::jsonb, + 'certforge.read', 1, '{"X-Tenant-ID":"org-echo-sovereign","Authorization":"vault:certforge.desktop_admin_api_key"}'::jsonb, 15, 'active', 'unknown'), + ('echo.certforge.admin.quarantine', 'Certification Forge administration: quarantine one authenticated runner or adapter, removing it from eligible capacity/execution without deleting telemetry. Tier-2 HMAC and Desktop reauthentication are required.', 'http', 'http://127.0.0.1:8309/v1/subscriber/operational-quarantines', 'POST', 'json_body', 'forge', diff --git a/scripts/register_certforge_sdk_schemas.sql b/scripts/register_certforge_sdk_schemas.sql index fbfaae4..78f876b 100644 --- a/scripts/register_certforge_sdk_schemas.sql +++ b/scripts/register_certforge_sdk_schemas.sql @@ -50,6 +50,7 @@ INSERT INTO certforge_sdk_contract(id, input_schema_json, output_schema_json) VA ('echo.certforge.projects.create', '{"additionalProperties":false,"properties":{"command":{"maxLength":64,"minLength":1,"type":"string"},"name":{"maxLength":160,"minLength":1,"type":"string"},"slug":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$","type":"string"},"target_reference":{"maxLength":2048,"minLength":1,"type":"string"}},"required":["command","name","slug","target_reference"],"type":"object"}'::jsonb, '{"additionalProperties":true,"type":"object"}'::jsonb), ('echo.certforge.projects.list', '{"additionalProperties":false,"properties":{"command":{"maxLength":64,"minLength":1,"type":"string"}},"required":["command"],"type":"object"}'::jsonb, '{"items":{"additionalProperties":true,"type":"object"},"type":"array"}'::jsonb), ('echo.certforge.public_verification', '{"additionalProperties":false,"properties":{"command":{"maxLength":64,"minLength":1,"type":"string"},"verification_id":{"type":"string"}},"required":["command","verification_id"],"type":"object"}'::jsonb, '{"additionalProperties":true,"type":"object"}'::jsonb), + ('echo.certforge.publish', '{"additionalProperties":false,"properties":{"command":{"maxLength":64,"minLength":1,"type":"string"},"run_id":{"type":"string"}},"required":["command","run_id"],"type":"object"}'::jsonb, '{"additionalProperties":{"type":"string"},"type":"object"}'::jsonb), ('echo.certforge.r5.status', '{"additionalProperties":false,"properties":{"command":{"maxLength":64,"minLength":1,"type":"string"},"run_id":{"pattern":"^[A-Za-z0-9._-]{1,64}$","type":"string"}},"required":["command","run_id"],"type":"object"}'::jsonb, '{"additionalProperties":false,"properties":{"note":{"type":"string"},"result":{"anyOf":[{"additionalProperties":false,"properties":{"completion_marker":{"type":["string","null"]},"deployment_authorized":{"type":"boolean"},"evidence_run_id":{"type":"string"},"forge_verification":{"additionalProperties":true,"type":"object"},"mode":{"enum":["preflight","full"],"type":"string"},"operator_exit_code":{"type":"integer"},"operator_report":{"additionalProperties":true,"type":"object"},"r5_gate":{"enum":["PASS","BLOCK"],"type":"string"},"release_verdict":{"enum":["NOT_READY"],"type":"string"},"run_outcome":{"type":"string"},"schema":{"type":"string"},"stderr_tail":{"type":"string"}},"required":["schema","mode","evidence_run_id","run_outcome","release_verdict","r5_gate","deployment_authorized","completion_marker","operator_report","forge_verification","operator_exit_code"],"type":"object"},{"type":"null"}]},"run_id":{"type":"string"},"status":{"enum":["RUNNING","COMPLETE","FAILED"],"type":"string"}},"required":["run_id","status"],"type":"object"}'::jsonb), ('echo.certforge.r5.submit_async', '{"additionalProperties":false,"properties":{"command":{"maxLength":64,"minLength":1,"type":"string"},"dry_run":{"type":"boolean"},"evidence_run_id":{"pattern":"^[A-Za-z0-9._-]{1,64}$","type":"string"},"evidence_run_nonce":{"maxLength":128,"minLength":16,"type":"string"},"expected_base_model_digest":{"pattern":"^[0-9a-f]{64}$","type":"string"},"expected_base_model_revision":{"maxLength":128,"minLength":6,"type":"string"},"expected_gs343_digest":{"pattern":"^[0-9a-f]{64}$","type":"string"},"expected_r2d2_digest":{"pattern":"^[0-9a-f]{64}$","type":"string"},"expected_registry_revision":{"maxLength":128,"minLength":6,"type":"string"},"expected_registry_snapshot_digest":{"pattern":"^[0-9a-f]{64}$","type":"string"},"expected_server_build_digest":{"pattern":"^[0-9a-f]{64}$","type":"string"},"expected_signing_key_id":{"maxLength":128,"minLength":8,"type":"string"},"target_family":{"enum":["gs343","r2d2"],"type":"string"}},"required":["command","target_family","expected_server_build_digest","expected_registry_snapshot_digest","expected_registry_revision","expected_signing_key_id","expected_base_model_digest","expected_base_model_revision","expected_gs343_digest","expected_r2d2_digest","evidence_run_id","evidence_run_nonce"],"type":"object"}'::jsonb, '{"additionalProperties":false,"properties":{"idempotent":{"type":"boolean"},"mode":{"enum":["preflight","full"],"type":"string"},"poll_capability":{"enum":["echo.certforge.r5.status"],"type":"string"},"run_id":{"type":"string"},"status":{"enum":["RESERVED","RUNNING"],"type":"string"}},"required":["run_id","status","mode"],"type":"object"}'::jsonb), ('echo.certforge.run', '{"additionalProperties":false,"properties":{"command":{"maxLength":64,"minLength":1,"type":"string"},"run_id":{"pattern":"^[A-Za-z0-9._-]{1,128}$","type":"string"},"tenant":{"pattern":"^[A-Za-z0-9._-]{1,128}$","type":"string"}},"required":["command","run_id","tenant"],"type":"object"}'::jsonb, '{"additionalProperties":false,"properties":{"poll_capability":{"enum":["echo.certforge.status"],"type":"string"},"run_id":{"type":"string"},"sandboxed":{"type":"boolean"},"status":{"enum":["PENDING","CLAIMED"],"type":"string"},"tenant":{"type":"string"}},"required":["run_id","status","tenant","sandboxed","poll_capability"],"type":"object"}'::jsonb), @@ -74,8 +75,8 @@ BEGIN FROM arcanum_sdk.sdk_capabilities WHERE id LIKE 'echo.certforge.%' OR id = 'echo.certification_forge.r5.run_negative_controls'; - IF live_count <> 60 THEN - RAISE EXCEPTION 'Certification Forge SDK surface drift: live %, contract 60', live_count; + IF live_count <> 61 THEN + RAISE EXCEPTION 'Certification Forge SDK surface drift: live %, contract 61', live_count; END IF; IF EXISTS ( SELECT 1 FROM certforge_sdk_contract c diff --git a/tests/test_echo_sdk_manifest.py b/tests/test_echo_sdk_manifest.py index d3dfe13..9848804 100644 --- a/tests/test_echo_sdk_manifest.py +++ b/tests/test_echo_sdk_manifest.py @@ -13,7 +13,7 @@ def test_echo_sdk_manifest_matches_the_authoritative_contract() -> None: contract = json.loads(contract_path.read_text(encoding="utf-8")) assert manifest["version"] == 1 - assert len(manifest["capabilities"]) == contract["capability_count"] == 60 + assert len(manifest["capabilities"]) == contract["capability_count"] == 61 assert set(manifest["capabilities"]) == set(contract["capabilities"]) assert len(manifest["capabilities"]) == len(set(manifest["capabilities"])) assert all(capability.startswith("echo.") for capability in manifest["capabilities"]) diff --git a/tests/test_sdk_capability_contract.py b/tests/test_sdk_capability_contract.py index adcb6ef..a0afa0e 100644 --- a/tests/test_sdk_capability_contract.py +++ b/tests/test_sdk_capability_contract.py @@ -32,7 +32,7 @@ def test_generated_sdk_contract_is_current() -> None: def test_every_certforge_capability_has_command_and_output_schema() -> None: contract = json.loads(CONTRACT.read_text(encoding="utf-8")) capabilities = contract["capabilities"] - assert contract["capability_count"] == 60 == len(capabilities) + assert contract["capability_count"] == 61 == len(capabilities) for capability, item in capabilities.items(): input_schema = item["input_schema"] output_schema = item["output_schema"] @@ -48,6 +48,14 @@ def test_every_certforge_capability_has_command_and_output_schema() -> None: assert "type" in output_schema or "anyOf" in output_schema, capability +def test_github_app_publish_alias_is_exactly_schema_compatible() -> None: + contract = json.loads(CONTRACT.read_text(encoding="utf-8")) + capabilities = contract["capabilities"] + least_privileged = capabilities["echo.certforge.publish"] + administrative = capabilities["echo.certforge.admin.publish"] + assert least_privileged == administrative + + def test_registered_capability_set_equals_sdk_contract() -> None: contract = json.loads(CONTRACT.read_text(encoding="utf-8")) registered: set[str] = set() From d54d02a5c22a731ef736cde0d1ed9e26bea2996b Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Tue, 1 Sep 2026 00:20:36 -0700 Subject: [PATCH 08/39] fix: certify the governed three-account autonomy profile --- docs/OPERATIONS.md | 4 +++ docs/PROJECT_CONTRACT.md | 13 +++++--- .../production_e2e.py | 5 ++- tests/test_production_e2e.py | 31 ++++++++++++++++--- 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 7e47fee..e68ac5f 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -26,6 +26,10 @@ identity/E2E mismatch is a normal `NOT_READY` result, never a bypass. 6. Promote atomically, health-check production, and roll back on any mismatch. 7. Publish immutable machine certificates and the repository certificate graphic. +The Echo GitHub Autonomy production collector must use profile +`echo-github-autonomy-remote-mcp-v2` and exactly the three governed accounts. +Profile v1 and evidence containing the retired `Bmcbob76` identity fail closed. + ## Incident triage Capture the run ID, target SHA, environment digest, policy version, service health, dispatcher state, diff --git a/docs/PROJECT_CONTRACT.md b/docs/PROJECT_CONTRACT.md index e452ded..845bfb4 100644 --- a/docs/PROJECT_CONTRACT.md +++ b/docs/PROJECT_CONTRACT.md @@ -52,8 +52,11 @@ acceptance. Source checks, local journeys, HTTP 200, unsigned status fields, and target-authored evidence cannot satisfy this gate. Missing, expired, untrusted, partially passing, or identity-mismatched attestations remain `NOT_READY`. -For Echo GitHub Autonomy the target-specific profile additionally requires the -canonical MCP and OAuth surface, 30-tool schema, repeated discovery plus -invocation without registry loss, reconciled public/private inventory and -read/write/certify authority for all four exact GitHub account IDs, and matching -private-repository fingerprints from ChatGPT, Claude, Codex, and Grok. +For Echo GitHub Autonomy, target-specific profile +`echo-github-autonomy-remote-mcp-v2` additionally requires the canonical MCP +and OAuth surface, 30-tool schema, repeated discovery plus invocation without +registry loss, reconciled public/private inventory and read/write/certify +authority for exactly the three governed GitHub account IDs (`echoomegaprime`, +`ECHO-OMEGA-PRIME`, and `bobmcwilliams4`), and matching private-repository +fingerprints from ChatGPT, Claude, Codex, and Grok. The retired `Bmcbob76` +identity is invalid evidence for this profile. diff --git a/src/echo_certification_forge/production_e2e.py b/src/echo_certification_forge/production_e2e.py index b35e4eb..9712462 100644 --- a/src/echo_certification_forge/production_e2e.py +++ b/src/echo_certification_forge/production_e2e.py @@ -22,7 +22,7 @@ RULE_ID = "production_e2e" SCHEMA_VERSION = "certforge.production-e2e.v1" GENERIC_PROFILE = "generic-production-v1" -ECHO_GITHUB_AUTONOMY_PROFILE = "echo-github-autonomy-remote-mcp-v1" +ECHO_GITHUB_AUTONOMY_PROFILE = "echo-github-autonomy-remote-mcp-v2" ECHO_GITHUB_AUTONOMY_REPOSITORY = "echoomegaprime/echo-github-autonomy" ECHO_GITHUB_AUTONOMY_CANONICAL_MCP = "https://echo-ghub.grok.me/api/plugin/mcp" @@ -44,7 +44,7 @@ "tool_schema", "repeated_tool_invocation", "registry_persistence", - "four_account_reconciliation", + "three_account_reconciliation", "private_public_visibility", "read_write_certify", "cross_client_consistency", @@ -53,7 +53,6 @@ ECHO_GITHUB_ACCOUNTS = { "echoomegaprime": 314902331, "ECHO-OMEGA-PRIME": 264607697, - "Bmcbob76": 203470412, "bobmcwilliams4": 235318155, } ECHO_CLIENTS = frozenset({"chatgpt", "claude", "codex", "grok"}) diff --git a/tests/test_production_e2e.py b/tests/test_production_e2e.py index 81460f5..1c7dc19 100644 --- a/tests/test_production_e2e.py +++ b/tests/test_production_e2e.py @@ -118,7 +118,9 @@ def test_echo_github_autonomy_requires_complete_exact_cross_client_e2e() -> None (lambda value: value.__setitem__("deployment_sha", "6" * 40), "production_e2e_deployment_sha_mismatch"), (lambda value: value.__setitem__("tool_count", 27), "production_e2e_tool_count_mismatch"), ( - lambda value: value["accounts"]["Bmcbob76"].__setitem__("private_count", 0), + lambda value: value["accounts"]["bobmcwilliams4"].__setitem__( + "private_count", 0 + ), "production_e2e_account_reconciliation_failed", ), ( @@ -126,19 +128,19 @@ def test_echo_github_autonomy_requires_complete_exact_cross_client_e2e() -> None "production_e2e_client_not_accepted", ), ( - lambda value: value["accounts"]["Bmcbob76"].__setitem__( + lambda value: value["accounts"]["bobmcwilliams4"].__setitem__( "credential_source", "model_config_pat" ), "production_e2e_credential_source_invalid", ), ( - lambda value: value["accounts"]["Bmcbob76"].__setitem__( + lambda value: value["accounts"]["bobmcwilliams4"].__setitem__( "secret_exposed", True ), "production_e2e_secret_boundary_failed", ), ( - lambda value: value["sample_private_repositories"]["Bmcbob76"].__setitem__( + lambda value: value["sample_private_repositories"]["bobmcwilliams4"].__setitem__( "repository_id", True ), "production_e2e_sample_repository_id_invalid", @@ -163,6 +165,27 @@ def test_stale_e2e_attestation_fails_closed() -> None: assert reason == "production_e2e_attestation_not_current" +def test_retired_account_and_v1_profile_are_rejected() -> None: + retired_account = _payload() + retired_account["accounts"]["Bmcbob76"] = { + **retired_account["accounts"]["bobmcwilliams4"], + "account_id": 203470412, + } + valid, reason = validate_production_e2e( + retired_account, _target(), _environment(), now=NOW + timedelta(minutes=1) + ) + assert not valid + assert reason == "production_e2e_accounts_incomplete" + + retired_profile = _payload() + retired_profile["profile"] = "echo-github-autonomy-remote-mcp-v1" + valid, reason = validate_production_e2e( + retired_profile, _target(), _environment(), now=NOW + timedelta(minutes=1) + ) + assert not valid + assert reason == "production_e2e_profile_mismatch" + + def test_attestation_loader_requires_a_pinned_collector_key(tmp_path) -> None: signer = Ed25519VerdictSigner.generate() payload = _payload(signing_key_id=signer.key_id) From 3db2b984d8b102656662d5bd25c71ec21c9122f3 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Tue, 1 Sep 2026 08:28:54 -0700 Subject: [PATCH 09/39] fix: load exact production attestations in dispatcher --- deploy/deploy_forge.sh | 6 ++ docs/OPERATIONS.md | 12 ++++ .../dispatch_worker.py | 38 +++++++++++ .../production_e2e.py | 51 +++++++++++++++ src/echo_certification_forge/run_worker.py | 6 +- tests/test_p5_production_launchers.py | 11 ++++ tests/test_production_e2e.py | 64 +++++++++++++++++++ 7 files changed, 187 insertions(+), 1 deletion(-) diff --git a/deploy/deploy_forge.sh b/deploy/deploy_forge.sh index c9c3b34..d1abdaf 100644 --- a/deploy/deploy_forge.sh +++ b/deploy/deploy_forge.sh @@ -14,6 +14,8 @@ EXPECTED_COMMIT_SHA="${CERTFORGE_EXPECTED_COMMIT_SHA:-}" RELEASE_ROOT="${CERTFORGE_RELEASE_ROOT:-/home/forge/echo-certification-forge-releases}" CURRENT_LINK="${CERTFORGE_CURRENT_LINK:-/home/forge/echo-certification-forge-current}" STATE_ROOT="${CERTFORGE_STATE_ROOT:-/home/forge/echo-certification-forge/var}" +PRODUCTION_E2E_ATTESTATION_DIR="${ECHO_CERTFORGE_PRODUCTION_E2E_ATTESTATION_DIR:-$STATE_ROOT/production-e2e/attestations}" +PRODUCTION_E2E_TRUSTED_KEYS="${ECHO_CERTFORGE_TRUSTED_PRODUCTION_E2E_KEYS:-$STATE_ROOT/production-e2e/trusted-public-keys}" ADAPTER_DIR="${ECHO_CERTFORGE_PROD_ADAPTER_DIR:-$STATE_ROOT/p5}" ADAPTER_MODE="${CERTFORGE_ADAPTER_MODE:-required}" TRUSTED_MANIFEST_SHA256="${ECHO_CERTFORGE_TRUSTED_MANIFEST_SHA256:-965106b00917268d556b325719f26f5096e6c3746551658ffecb9fd4a95ec342}" @@ -62,6 +64,8 @@ mkdir -p \ "$STATE_ROOT/run-output" \ "$STATE_ROOT/dispatch-output" \ "$STATE_ROOT/deploy-scratch" +sudo install -d -o root -g root -m 0755 "$PRODUCTION_E2E_ATTESTATION_DIR" +sudo install -d -o root -g root -m 0755 "$PRODUCTION_E2E_TRUSTED_KEYS" trap 'rm -rf "$RELEASE_TMP"' EXIT mkdir "$RELEASE_TMP" git archive "$NEW_SHA" | tar -x -C "$RELEASE_TMP" @@ -581,6 +585,8 @@ Environment=ECHO_CERTFORGE_PROD_ADAPTER_POLICY=$ADAPTER_POLICY Environment=ECHO_CERTFORGE_ADAPTER_REGISTRY=$ADAPTER_REGISTRY Environment=ECHO_CERTFORGE_ADAPTER_RUNNER_SIGNING_KEY=$ADAPTER_SIGNING_KEY Environment=ECHO_CERTFORGE_ADAPTER_ACCEPTANCE_REPORT=$ADAPTER_ACCEPTANCE_REPORT +Environment=ECHO_CERTFORGE_PRODUCTION_E2E_ATTESTATION_DIR=$PRODUCTION_E2E_ATTESTATION_DIR +Environment=ECHO_CERTFORGE_TRUSTED_PRODUCTION_E2E_KEYS=$PRODUCTION_E2E_TRUSTED_KEYS Environment=ECHO_CERTFORGE_TRUSTED_MANIFEST_SHA256=$TRUSTED_MANIFEST_SHA256 EnvironmentFile=$ENV_FILE ExecStart=$CURRENT_LINK/.venv/bin/python -m echo_certification_forge.dispatch_worker --sandbox $DISPATCH_COMPAT_FLAG diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index e68ac5f..f5194a7 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -18,6 +18,18 @@ pinned collector public keys through `--trusted-production-e2e-keys` (or be present on the worker. Absence, signature failure, expiration, or any exact identity/E2E mismatch is a normal `NOT_READY` result, never a bypass. +The durable subscriber dispatcher resolves attestations from +`ECHO_CERTFORGE_PRODUCTION_E2E_ATTESTATION_DIR`. Each envelope is named +`..json`; both values are +the exact 64-character digests published for the queued run. The matching +collector public key is pinned independently under +`ECHO_CERTFORGE_TRUSTED_PRODUCTION_E2E_KEYS`. Both directories are root-owned +and readable by the dispatcher but are not writable by its `forge` service +identity. Install a collector envelope atomically (temporary file followed by +rename), never place a private key in either directory, and never accept a +target-selected path or target-supplied public key. Missing, untrusted, stale, +or identity-mismatched envelopes remain `NOT_READY`. + 1. Confirm hosted CI succeeded for the exact source SHA. 2. Execute the declared production-shaped journey with the pinned runner image. 3. Verify evidence custody, receipt chains, signatures, expiry, and revocation state. diff --git a/src/echo_certification_forge/dispatch_worker.py b/src/echo_certification_forge/dispatch_worker.py index 2bc2b88..e826ac1 100644 --- a/src/echo_certification_forge/dispatch_worker.py +++ b/src/echo_certification_forge/dispatch_worker.py @@ -11,6 +11,7 @@ from .evidence import EvidenceStore from .intake import SubmitRequest from .policy import RuleManifest +from .production_e2e import DirectoryProductionE2EProvider from .run_worker import _load_adapter_inputs, _load_signer, run from .sandbox import DEFAULT_IMAGE, DockerSandbox from .subscriber import SubscriberDispatch, SubscriberError, SubscriberGovernance, SubscriberPolicy @@ -241,6 +242,24 @@ def main(argv: list[str] | None = None) -> int: choices=("platform", "customer"), default=os.environ.get("ECHO_CERTFORGE_SIGNING_AUTHORITY", "platform"), ) + parser.add_argument( + "--production-e2e-attestation-dir", + type=Path, + default=( + Path(os.environ["ECHO_CERTFORGE_PRODUCTION_E2E_ATTESTATION_DIR"]) + if os.environ.get("ECHO_CERTFORGE_PRODUCTION_E2E_ATTESTATION_DIR") + else None + ), + ) + parser.add_argument( + "--trusted-production-e2e-keys", + type=Path, + default=( + Path(os.environ["ECHO_CERTFORGE_TRUSTED_PRODUCTION_E2E_KEYS"]) + if os.environ.get("ECHO_CERTFORGE_TRUSTED_PRODUCTION_E2E_KEYS") + else None + ), + ) args = parser.parse_args(argv) if args.poll_seconds <= 0: parser.error("--poll-seconds must be positive") @@ -270,6 +289,23 @@ def main(argv: list[str] | None = None) -> int: return 2 store = EvidenceStore(args.db, args.evidence_root) signer = _load_signer(args.signing_key) + production_e2e_paths = ( + args.production_e2e_attestation_dir, + args.trusted_production_e2e_keys, + ) + if sum(path is not None for path in production_e2e_paths) not in (0, 2): + print(json.dumps({"error": "production_e2e_trust_incomplete"})) + return 2 + production_e2e_provider = None + if all(path is not None for path in production_e2e_paths): + try: + production_e2e_provider = DirectoryProductionE2EProvider( + args.production_e2e_attestation_dir, + args.trusted_production_e2e_keys, + ) + except ValueError: + print(json.dumps({"error": "production_e2e_trust_unavailable"})) + return 2 sandbox = ( DockerSandbox( image=args.sandbox_image, @@ -289,6 +325,8 @@ def main(argv: list[str] | None = None) -> int: "execution_location": args.execution_location, "signing_authority": args.signing_authority, } + if production_e2e_provider is not None: + options["production_e2e_provider"] = production_e2e_provider if supplied_count == len(adapter_paths): options["adapter_inputs"] = { "response_path": args.adapter_response, diff --git a/src/echo_certification_forge/production_e2e.py b/src/echo_certification_forge/production_e2e.py index 9712462..490f180 100644 --- a/src/echo_certification_forge/production_e2e.py +++ b/src/echo_certification_forge/production_e2e.py @@ -348,3 +348,54 @@ def load_signed_attestation(path: Path, trusted_key_directory: Path) -> Verified raise ValueError("production E2E attestation envelope is invalid") from exc registry = TrustedPublicKeyRegistry.from_directory(trusted_key_directory) return verify_signed_attestation(envelope, registry) + + +@dataclass(frozen=True, slots=True) +class DirectoryProductionE2EProvider: + """Resolve a signed attestation for one exact target/environment pair. + + The directory and trust roots are selected by the control plane. Target code + can neither choose an arbitrary file nor supply a public key. Missing or + untrusted envelopes are treated as absent so the deterministic verdict stays + ``NOT_READY``; a successfully verified envelope is still identity- and + freshness-checked by the executor. + """ + + attestation_directory: Path + trusted_key_directory: Path + + def __post_init__(self) -> None: + try: + attestation_root = self.attestation_directory.resolve(strict=True) + trust_root = self.trusted_key_directory.resolve(strict=True) + except OSError as exc: + raise ValueError("production E2E trust root is unavailable") from exc + if not attestation_root.is_dir() or not trust_root.is_dir(): + raise ValueError("production E2E trust root is not a directory") + object.__setattr__(self, "attestation_directory", attestation_root) + object.__setattr__(self, "trusted_key_directory", trust_root) + + def __call__( + self, + target: TargetIdentity, + environment: EnvironmentIdentity, + ) -> VerifiedProductionE2E | None: + target_digest = target.identity_digest + environment_digest = environment.identity_digest + if not _SHA256.fullmatch(target_digest) or not _SHA256.fullmatch(environment_digest): + return None + candidate = self.attestation_directory / ( + f"{target_digest}.{environment_digest}.json" + ) + try: + if candidate.is_symlink() or not candidate.is_file(): + return None + resolved = candidate.resolve(strict=True) + except OSError: + return None + if resolved.parent != self.attestation_directory: + return None + try: + return load_signed_attestation(resolved, self.trusted_key_directory) + except (OSError, TypeError, ValueError): + return None diff --git a/src/echo_certification_forge/run_worker.py b/src/echo_certification_forge/run_worker.py index e2fc1a6..ae8d603 100644 --- a/src/echo_certification_forge/run_worker.py +++ b/src/echo_certification_forge/run_worker.py @@ -164,7 +164,11 @@ def run( adapter_bundle_response: RunnerResponse | None = None, production_e2e_attestation: VerifiedProductionE2E | None = None, production_e2e_provider: ( - Callable[[TargetIdentity, EnvironmentIdentity], VerifiedProductionE2E] | None + Callable[ + [TargetIdentity, EnvironmentIdentity], + VerifiedProductionE2E | None, + ] + | None ) = None, worker_id: str | None = None, worker_attestation_sha256: str | None = None, diff --git a/tests/test_p5_production_launchers.py b/tests/test_p5_production_launchers.py index bedf210..d0c594b 100644 --- a/tests/test_p5_production_launchers.py +++ b/tests/test_p5_production_launchers.py @@ -21,6 +21,9 @@ def test_production_dispatcher_requires_v2_and_all_adapter_inputs() -> None: assert "--adapter-runner-signing-key" in launch assert "--adapter-runner-signing-key" in dispatcher assert "production_adapter_inputs_unavailable" in dispatcher + assert "--production-e2e-attestation-dir" in dispatcher + assert "--trusted-production-e2e-keys" in dispatcher + assert "DirectoryProductionE2EProvider" in dispatcher def test_production_deploy_requires_v2_adapter_artifacts() -> None: @@ -53,6 +56,14 @@ def test_deploy_clears_stale_systemd_start_limits_before_start_and_rollback() -> assert promotion.index(dispatcher_reset) < promotion.index(dispatcher_restart) +def test_deploy_binds_dispatcher_to_root_owned_production_e2e_trust_roots() -> None: + text = (ROOT / "deploy" / "deploy_forge.sh").read_text(encoding="utf-8") + assert "ECHO_CERTFORGE_PRODUCTION_E2E_ATTESTATION_DIR" in text + assert "ECHO_CERTFORGE_TRUSTED_PRODUCTION_E2E_KEYS" in text + assert 'sudo install -d -o root -g root -m 0755 "$PRODUCTION_E2E_ATTESTATION_DIR"' in text + assert 'sudo install -d -o root -g root -m 0755 "$PRODUCTION_E2E_TRUSTED_KEYS"' in text + + def test_deploy_streams_private_sdk_sql_into_one_postgres_transaction() -> None: text = (ROOT / "deploy" / "deploy_forge.sh").read_text(encoding="utf-8") registration = text.split( diff --git a/tests/test_production_e2e.py b/tests/test_production_e2e.py index 1c7dc19..3358c55 100644 --- a/tests/test_production_e2e.py +++ b/tests/test_production_e2e.py @@ -7,6 +7,7 @@ from echo_certification_forge.models import EnvironmentIdentity, TargetIdentity from echo_certification_forge.production_e2e import ( + DirectoryProductionE2EProvider, ECHO_CLIENTS, ECHO_GITHUB_ACCOUNTS, ECHO_GITHUB_AUTONOMY_CANONICAL_MCP, @@ -235,3 +236,66 @@ def test_self_selected_public_key_cannot_replace_the_pinned_collector(tmp_path) (trusted / "collector.pem").write_text(trusted_signer.public_key_pem, encoding="ascii") with pytest.raises(ValueError, match="untrusted_signing_key"): load_signed_attestation(envelope_path, trusted) + + +def test_directory_provider_resolves_only_the_exact_target_and_environment(tmp_path) -> None: + signer = Ed25519VerdictSigner.generate() + payload = _payload(signing_key_id=signer.key_id) + envelope = signer.sign_payload(payload) + attestations = tmp_path / "attestations" + trusted = tmp_path / "trusted" + attestations.mkdir() + trusted.mkdir() + (trusted / "collector.pem").write_text(signer.public_key_pem, encoding="ascii") + expected_path = attestations / ( + f"{_target().identity_digest}.{_environment().identity_digest}.json" + ) + expected_path.write_text( + json.dumps( + { + "payload": envelope.payload, + "signature_b64": envelope.signature_b64, + "key_id": envelope.key_id, + "public_key_pem": envelope.public_key_pem, + } + ), + encoding="utf-8", + ) + + provider = DirectoryProductionE2EProvider(attestations, trusted) + verified = provider(_target(), _environment()) + assert verified is not None + assert verified.payload == payload + + +def test_directory_provider_fails_closed_for_missing_or_untrusted_envelopes(tmp_path) -> None: + trusted_signer = Ed25519VerdictSigner.generate() + attacker = Ed25519VerdictSigner.generate() + attestations = tmp_path / "attestations" + trusted = tmp_path / "trusted" + attestations.mkdir() + trusted.mkdir() + (trusted / "collector.pem").write_text( + trusted_signer.public_key_pem, + encoding="ascii", + ) + provider = DirectoryProductionE2EProvider(attestations, trusted) + assert provider(_target(), _environment()) is None + + payload = _payload(signing_key_id=attacker.key_id) + envelope = attacker.sign_payload(payload) + expected_path = attestations / ( + f"{_target().identity_digest}.{_environment().identity_digest}.json" + ) + expected_path.write_text( + json.dumps( + { + "payload": envelope.payload, + "signature_b64": envelope.signature_b64, + "key_id": envelope.key_id, + "public_key_pem": envelope.public_key_pem, + } + ), + encoding="utf-8", + ) + assert provider(_target(), _environment()) is None From 10fb82fb521dabec80548b25e87d6cb99d052307 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Tue, 1 Sep 2026 09:19:20 -0700 Subject: [PATCH 10/39] fix: bound large repository acquisition timeout --- deploy/deploy_forge.sh | 10 ++++++ docs/OPERATIONS.md | 7 ++++ src/echo_certification_forge/acquisition.py | 37 ++++++++++++++++++++- tests/test_t4_live_run.py | 37 +++++++++++++++++++++ 4 files changed, 90 insertions(+), 1 deletion(-) diff --git a/deploy/deploy_forge.sh b/deploy/deploy_forge.sh index d1abdaf..c736ac5 100644 --- a/deploy/deploy_forge.sh +++ b/deploy/deploy_forge.sh @@ -7,6 +7,7 @@ set -euo pipefail SOURCE_REPO="${CERTFORGE_SOURCE_REPO:-$(cd "$(dirname "$0")/.." && pwd)}" STAGING_PORT="${CERTFORGE_STAGING_PORT:-8311}" PROD_PORT="${CERTFORGE_PROD_PORT:-8309}" +GIT_ACQUISITION_TIMEOUT_SECONDS="${ECHO_CERTFORGE_GIT_ACQUISITION_TIMEOUT_SECONDS:-300}" SERVICE="echo-certforge" DISPATCH_SERVICE="echo-certforge-dispatcher" BRANCH="${CERTFORGE_BRANCH:-main}" @@ -94,6 +95,14 @@ case "$ADAPTER_MODE" in required|pending) ;; *) echo "!! CERTFORGE_ADAPTER_MODE must be required or pending"; exit 1 ;; esac +[[ "$GIT_ACQUISITION_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] || { + echo "!! ECHO_CERTFORGE_GIT_ACQUISITION_TIMEOUT_SECONDS must be an integer" + exit 1 +} +if (( GIT_ACQUISITION_TIMEOUT_SECONDS < 30 || GIT_ACQUISITION_TIMEOUT_SECONDS > 1800 )); then + echo "!! ECHO_CERTFORGE_GIT_ACQUISITION_TIMEOUT_SECONDS must be between 30 and 1800" + exit 1 +fi ADAPTER_RESPONSE="" ADAPTER_POLICY="" ADAPTER_REGISTRY="" @@ -587,6 +596,7 @@ Environment=ECHO_CERTFORGE_ADAPTER_RUNNER_SIGNING_KEY=$ADAPTER_SIGNING_KEY Environment=ECHO_CERTFORGE_ADAPTER_ACCEPTANCE_REPORT=$ADAPTER_ACCEPTANCE_REPORT Environment=ECHO_CERTFORGE_PRODUCTION_E2E_ATTESTATION_DIR=$PRODUCTION_E2E_ATTESTATION_DIR Environment=ECHO_CERTFORGE_TRUSTED_PRODUCTION_E2E_KEYS=$PRODUCTION_E2E_TRUSTED_KEYS +Environment=ECHO_CERTFORGE_GIT_ACQUISITION_TIMEOUT_SECONDS=$GIT_ACQUISITION_TIMEOUT_SECONDS Environment=ECHO_CERTFORGE_TRUSTED_MANIFEST_SHA256=$TRUSTED_MANIFEST_SHA256 EnvironmentFile=$ENV_FILE ExecStart=$CURRENT_LINK/.venv/bin/python -m echo_certification_forge.dispatch_worker --sandbox $DISPATCH_COMPAT_FLAG diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index f5194a7..84723df 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -30,6 +30,13 @@ rename), never place a private key in either directory, and never accept a target-selected path or target-supplied public key. Missing, untrusted, stale, or identity-mismatched envelopes remain `NOT_READY`. +Exact Git acquisition uses the bounded dispatcher setting +`ECHO_CERTFORGE_GIT_ACQUISITION_TIMEOUT_SECONDS` (production default: 300; +allowed range: 30–1800 seconds). This bounds network and checkout stalls while +allowing large repositories to materialize under normal disk contention. An +invalid value fails the acquisition before creating the target destination; +never remove the timeout to make a run pass. + 1. Confirm hosted CI succeeded for the exact source SHA. 2. Execute the declared production-shaped journey with the pinned runner image. 3. Verify evidence custody, receipt chains, signatures, expiry, and revocation state. diff --git a/src/echo_certification_forge/acquisition.py b/src/echo_certification_forge/acquisition.py index c11ef29..b34ac65 100644 --- a/src/echo_certification_forge/acquisition.py +++ b/src/echo_certification_forge/acquisition.py @@ -15,6 +15,8 @@ import io import json +import math +import os import re import shutil import subprocess @@ -32,6 +34,32 @@ class AcquisitionError(RuntimeError): """Fail-closed acquisition failure.""" +_GIT_ACQUISITION_TIMEOUT_ENV = "ECHO_CERTFORGE_GIT_ACQUISITION_TIMEOUT_SECONDS" +_DEFAULT_GIT_ACQUISITION_TIMEOUT_SECONDS = 300.0 +_MIN_GIT_ACQUISITION_TIMEOUT_SECONDS = 30.0 +_MAX_GIT_ACQUISITION_TIMEOUT_SECONDS = 1800.0 + + +def configured_git_acquisition_timeout_seconds() -> float: + """Return the bounded operator-controlled Git acquisition timeout.""" + + raw = os.getenv( + _GIT_ACQUISITION_TIMEOUT_ENV, + str(_DEFAULT_GIT_ACQUISITION_TIMEOUT_SECONDS), + ).strip() + try: + value = float(raw) + except ValueError as exc: + raise AcquisitionError("git acquisition timeout configuration is invalid") from exc + if ( + not math.isfinite(value) + or value < _MIN_GIT_ACQUISITION_TIMEOUT_SECONDS + or value > _MAX_GIT_ACQUISITION_TIMEOUT_SECONDS + ): + raise AcquisitionError("git acquisition timeout configuration is out of bounds") + return value + + @dataclass(frozen=True, slots=True) class AcquiredTarget: source_root: Path @@ -348,7 +376,12 @@ def _acquire_oci(spec: dict, dest: Path, timeout_s: float) -> AcquiredTarget: return AcquiredTarget(rootfs, "oci", canonical, raw_digest.split(":", 1)[1]) -def acquire_target(spec: dict, dest: Path, *, clone_timeout_s: float = 120.0) -> AcquiredTarget: +def acquire_target( + spec: dict, + dest: Path, + *, + clone_timeout_s: float | None = None, +) -> AcquiredTarget: """Acquire a target into (or referencing) an isolated tree and compute its immutable identity. spec: @@ -369,6 +402,8 @@ def acquire_target(spec: dict, dest: Path, *, clone_timeout_s: float = 120.0) -> return AcquiredTarget(root, "local", root.as_posix(), _tree_digest(root)) if target_type == "git": + if clone_timeout_s is None: + clone_timeout_s = configured_git_acquisition_timeout_seconds() url = str(spec.get("url", "")).strip() if not url: raise AcquisitionError("git target requires 'url'") diff --git a/tests/test_t4_live_run.py b/tests/test_t4_live_run.py index 02d6784..07580b5 100644 --- a/tests/test_t4_live_run.py +++ b/tests/test_t4_live_run.py @@ -77,6 +77,43 @@ def fake_run(command, **_kwargs): assert not any("clone" in command and "--branch" in command for command in commands) +def test_acquire_git_uses_bounded_configured_timeout(tmp_path, monkeypatch): + source_commit = "a" * 40 + observed_timeouts: list[float] = [] + + def fake_run(command, **kwargs): + observed_timeouts.append(kwargs["timeout"]) + destination = tmp_path / "configured-timeout" + if "init" in command: + (destination / ".git").mkdir(parents=True) + if "checkout" in command: + (destination / "README.md").write_text("exact revision\n", encoding="utf-8") + stdout = f"{source_commit}\n" if "rev-parse" in command else "" + return subprocess.CompletedProcess(command, 0, stdout=stdout, stderr="") + + monkeypatch.setenv("ECHO_CERTFORGE_GIT_ACQUISITION_TIMEOUT_SECONDS", "321") + monkeypatch.setattr("echo_certification_forge.acquisition.subprocess.run", fake_run) + acquired = acquire_target( + {"type": "git", "url": "https://github.com/example/project.git", "ref": source_commit}, + tmp_path / "configured-timeout", + ) + + assert acquired.source_commit == source_commit + assert observed_timeouts + assert set(observed_timeouts) == {321.0} + + +@pytest.mark.parametrize("configured", ["invalid", "29", "1801", "nan", "inf"]) +def test_acquire_git_rejects_invalid_configured_timeout(tmp_path, monkeypatch, configured): + monkeypatch.setenv("ECHO_CERTFORGE_GIT_ACQUISITION_TIMEOUT_SECONDS", configured) + with pytest.raises(AcquisitionError, match="timeout configuration"): + acquire_target( + {"type": "git", "url": "https://github.com/example/project.git", "ref": "a" * 40}, + tmp_path / "invalid-timeout", + ) + assert not (tmp_path / "invalid-timeout").exists() + + def test_acquire_git_exact_commit_fails_closed_on_revision_mismatch(tmp_path, monkeypatch): source_commit = "a" * 40 From a5cae3b7af43cb0e5176b05297d8d42f36d768a7 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Tue, 1 Sep 2026 11:24:02 -0700 Subject: [PATCH 11/39] fix: authenticate private GitHub target acquisition --- src/echo_certification_forge/acquisition.py | 220 +++++++++++++++++--- tests/test_t4_live_run.py | 82 ++++++++ 2 files changed, 278 insertions(+), 24 deletions(-) diff --git a/src/echo_certification_forge/acquisition.py b/src/echo_certification_forge/acquisition.py index b34ac65..9460694 100644 --- a/src/echo_certification_forge/acquisition.py +++ b/src/echo_certification_forge/acquisition.py @@ -13,6 +13,7 @@ """ from __future__ import annotations +import base64 import io import json import math @@ -21,11 +22,16 @@ import shutil import subprocess import tarfile +import time import urllib.error +import urllib.parse import urllib.request from dataclasses import dataclass from pathlib import Path, PurePosixPath +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import padding, rsa + from .canonical import sha256_bytes from .models import is_exact_git_commit @@ -38,6 +44,14 @@ class AcquisitionError(RuntimeError): _DEFAULT_GIT_ACQUISITION_TIMEOUT_SECONDS = 300.0 _MIN_GIT_ACQUISITION_TIMEOUT_SECONDS = 30.0 _MAX_GIT_ACQUISITION_TIMEOUT_SECONDS = 1800.0 +_GITHUB_APP_ID_ENV = "ECHO_CERTFORGE_GITHUB_APP_ID" +_GITHUB_PRIVATE_KEY_FILE_ENV = "ECHO_CERTFORGE_GITHUB_PRIVATE_KEY_FILE" +_GITHUB_API = "https://api.github.com" +_GITHUB_REPOSITORY_URL = re.compile( + r"^https://github\.com/(?P[A-Za-z0-9_.-]+)/" + r"(?P[A-Za-z0-9_.-]+?)(?:\.git)?$" +) +_GITHUB_RESPONSE_LIMIT = 1024 * 1024 def configured_git_acquisition_timeout_seconds() -> float: @@ -60,6 +74,142 @@ def configured_git_acquisition_timeout_seconds() -> float: return value +def _base64url(value: bytes) -> str: + return base64.urlsafe_b64encode(value).rstrip(b"=").decode("ascii") + + +def _github_app_jwt(app_id: int, private_key: bytes) -> str: + now = int(time.time()) + header = _base64url(json.dumps({"alg": "RS256", "typ": "JWT"}, separators=(",", ":")).encode()) + payload = _base64url( + json.dumps( + {"iat": now - 30, "exp": now + 540, "iss": str(app_id)}, + separators=(",", ":"), + ).encode() + ) + signing_input = f"{header}.{payload}".encode("ascii") + try: + loaded = serialization.load_pem_private_key(private_key, password=None) + except (TypeError, ValueError) as exc: + raise AcquisitionError("GitHub App private key is invalid") from exc + if not isinstance(loaded, rsa.RSAPrivateKey): + raise AcquisitionError("GitHub App private key must be RSA") + signature = loaded.sign(signing_input, padding.PKCS1v15(), hashes.SHA256()) + return f"{header}.{payload}.{_base64url(signature)}" + + +def _github_json( + method: str, + path: str, + token: str, + *, + body: dict | None = None, +) -> dict: + encoded = None if body is None else json.dumps(body, separators=(",", ":")).encode() + request = urllib.request.Request( + f"{_GITHUB_API}{path}", + data=encoded, + method=method, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + "User-Agent": "echo-certification-forge/1.2", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310 + raw = response.read(_GITHUB_RESPONSE_LIMIT + 1) + except urllib.error.HTTPError as exc: + raise AcquisitionError( + f"GitHub App credential request failed: {method} HTTP {exc.code}" + ) from None + except (OSError, TimeoutError, urllib.error.URLError) as exc: + raise AcquisitionError( + f"GitHub App credential request failed: {method} {type(exc).__name__}" + ) from None + if len(raw) > _GITHUB_RESPONSE_LIMIT: + raise AcquisitionError("GitHub App credential response exceeded size limit") + try: + document = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError): + raise AcquisitionError("GitHub App credential response was not JSON") from None + if not isinstance(document, dict): + raise AcquisitionError("GitHub App credential response was not an object") + return document + + +def _github_app_installation_token(url: str) -> str | None: + """Resolve a short-lived, repository-scoped token for a canonical GitHub URL. + + The token is used only through ``GIT_ASKPASS`` and is never placed in the URL, + command line, target specification, evidence, or exception text. + """ + + match = _GITHUB_REPOSITORY_URL.fullmatch(url) + if match is None: + return None + raw_app_id = os.getenv(_GITHUB_APP_ID_ENV, "").strip() + raw_key_file = os.getenv(_GITHUB_PRIVATE_KEY_FILE_ENV, "").strip() + if not raw_app_id and not raw_key_file: + return None + if not raw_app_id or not raw_key_file: + raise AcquisitionError("GitHub App acquisition credential configuration is incomplete") + try: + app_id = int(raw_app_id) + except ValueError as exc: + raise AcquisitionError("GitHub App id is invalid") from exc + if app_id <= 0: + raise AcquisitionError("GitHub App id is invalid") + key_file = Path(raw_key_file).resolve(strict=False) + try: + private_key = key_file.read_bytes() + except OSError as exc: + raise AcquisitionError("GitHub App private key file is unavailable") from exc + app_jwt = _github_app_jwt(app_id, private_key) + owner = match.group("owner") + repository = match.group("repository") + encoded_owner = urllib.parse.quote(owner, safe="") + encoded_repository = urllib.parse.quote(repository, safe="") + installation = _github_json( + "GET", + f"/repos/{encoded_owner}/{encoded_repository}/installation", + app_jwt, + ) + installation_id = installation.get("id") + if not isinstance(installation_id, int) or installation_id <= 0: + raise AcquisitionError("GitHub App repository installation is unavailable") + credential = _github_json( + "POST", + f"/app/installations/{installation_id}/access_tokens", + app_jwt, + body={ + "repositories": [repository], + "permissions": {"contents": "read"}, + }, + ) + token = credential.get("token") + if not isinstance(token, str) or not token: + raise AcquisitionError("GitHub App installation token was not returned") + return token + + +def _write_git_askpass(parent: Path) -> Path: + path = parent / ".certforge-git-askpass" + path.write_text( + "#!/bin/sh\n" + "case \"$1\" in\n" + " *Username*) printf '%s\\n' 'x-access-token' ;;\n" + " *Password*) printf '%s\\n' \"$ECHO_CERTFORGE_GIT_TOKEN\" ;;\n" + " *) exit 1 ;;\n" + "esac\n", + encoding="utf-8", + ) + path.chmod(0o700) + return path + + @dataclass(frozen=True, slots=True) class AcquiredTarget: source_root: Path @@ -412,6 +562,18 @@ def acquire_target( if dest.exists(): shutil.rmtree(dest) dest.parent.mkdir(parents=True, exist_ok=True) + installation_token = _github_app_installation_token(url) + askpass_path: Path | None = None + git_environment = os.environ.copy() + if installation_token is not None: + askpass_path = _write_git_askpass(dest.parent) + git_environment.update( + { + "ECHO_CERTFORGE_GIT_TOKEN": installation_token, + "GIT_ASKPASS": str(askpass_path), + "GIT_TERMINAL_PROMPT": "0", + } + ) git_base = [ "git", "-c", "core.hooksPath=/dev/null", # never run target-supplied hooks @@ -424,40 +586,50 @@ def run_git(arguments: list[str], operation: str) -> subprocess.CompletedProcess process = subprocess.run( [*git_base, *arguments], capture_output=True, text=True, timeout=clone_timeout_s, shell=False, check=False, + env=git_environment, ) except (OSError, subprocess.TimeoutExpired) as exc: raise AcquisitionError(f"git {operation} failed: {type(exc).__name__}") from exc if process.returncode != 0: detail = process.stderr.strip()[-300:] + if installation_token is not None: + detail = detail.replace(installation_token, "[REDACTED]") raise AcquisitionError(f"git {operation} exited {process.returncode}: {detail}") return process actual_ref: str | None = None - if ref and is_exact_git_commit(ref): - # `git clone --branch ` does not reliably accept raw commit IDs. Fetch the - # immutable object explicitly, detach at FETCH_HEAD, and then prove HEAD is exact. - run_git(["init", "--", str(dest)], "init") - run_git(["-C", str(dest), "remote", "add", "origin", url], "remote-add") - run_git( - ["-C", str(dest), "fetch", "--depth", "1", "--no-tags", "origin", ref], - "fetch-exact-commit", - ) - run_git(["-C", str(dest), "checkout", "--detach", "FETCH_HEAD"], "checkout") - actual_ref = run_git(["-C", str(dest), "rev-parse", "HEAD"], "rev-parse").stdout.strip() - if actual_ref.lower() != ref.lower(): - shutil.rmtree(dest, ignore_errors=True) - raise AcquisitionError( - f"git revision mismatch: requested {ref.lower()}, acquired {actual_ref.lower()}" + try: + if ref and is_exact_git_commit(ref): + # `git clone --branch ` does not reliably accept raw commit IDs. Fetch the + # immutable object explicitly, detach at FETCH_HEAD, and then prove HEAD is exact. + run_git(["init", "--", str(dest)], "init") + run_git(["-C", str(dest), "remote", "add", "origin", url], "remote-add") + run_git( + ["-C", str(dest), "fetch", "--depth", "1", "--no-tags", "origin", ref], + "fetch-exact-commit", ) - else: - clone_args = ["clone", "--depth", "1", "--no-tags", "--single-branch"] - if ref: - clone_args += ["--branch", ref] - clone_args += ["--", url, str(dest)] - run_git(clone_args, "clone") - actual_ref = run_git( - ["-C", str(dest), "rev-parse", "HEAD"], "rev-parse" - ).stdout.strip() + run_git(["-C", str(dest), "checkout", "--detach", "FETCH_HEAD"], "checkout") + actual_ref = run_git( + ["-C", str(dest), "rev-parse", "HEAD"], "rev-parse" + ).stdout.strip() + if actual_ref.lower() != ref.lower(): + shutil.rmtree(dest, ignore_errors=True) + raise AcquisitionError( + f"git revision mismatch: requested {ref.lower()}, acquired {actual_ref.lower()}" + ) + else: + clone_args = ["clone", "--depth", "1", "--no-tags", "--single-branch"] + if ref: + clone_args += ["--branch", ref] + clone_args += ["--", url, str(dest)] + run_git(clone_args, "clone") + actual_ref = run_git( + ["-C", str(dest), "rev-parse", "HEAD"], "rev-parse" + ).stdout.strip() + finally: + git_environment.pop("ECHO_CERTFORGE_GIT_TOKEN", None) + if askpass_path is not None: + askpass_path.unlink(missing_ok=True) # remove the .git dir so no repo metadata/hooks enter the scanned build context git_dir = dest / ".git" if git_dir.exists(): diff --git a/tests/test_t4_live_run.py b/tests/test_t4_live_run.py index 07580b5..be4d6e8 100644 --- a/tests/test_t4_live_run.py +++ b/tests/test_t4_live_run.py @@ -77,6 +77,88 @@ def fake_run(command, **_kwargs): assert not any("clone" in command and "--branch" in command for command in commands) +def test_acquire_private_github_uses_ephemeral_askpass_without_secret_in_command( + tmp_path, monkeypatch +): + source_commit = "a" * 40 + installation_token = "installation-token-must-not-leak" + commands: list[list[str]] = [] + observed_environments: list[dict[str, str]] = [] + askpass_bodies: list[str] = [] + + monkeypatch.setattr( + "echo_certification_forge.acquisition._github_app_installation_token", + lambda _url: installation_token, + ) + + def fake_run(command, **kwargs): + commands.append(command) + environment = kwargs["env"] + observed_environments.append(dict(environment)) + askpass = Path(environment["GIT_ASKPASS"]) + askpass_bodies.append(askpass.read_text(encoding="utf-8")) + destination = tmp_path / "private" + if "init" in command: + (destination / ".git").mkdir(parents=True) + if "checkout" in command: + (destination / "README.md").write_text("private exact revision\n", encoding="utf-8") + stdout = f"{source_commit}\n" if "rev-parse" in command else "" + return subprocess.CompletedProcess(command, 0, stdout=stdout, stderr="") + + monkeypatch.setattr("echo_certification_forge.acquisition.subprocess.run", fake_run) + acquired = acquire_target( + { + "type": "git", + "url": "https://github.com/echoomegaprime/private-project.git", + "ref": source_commit, + }, + tmp_path / "private", + ) + + assert acquired.source_commit == source_commit + assert observed_environments + assert all(item["ECHO_CERTFORGE_GIT_TOKEN"] == installation_token for item in observed_environments) + assert all(item["GIT_TERMINAL_PROMPT"] == "0" for item in observed_environments) + assert installation_token not in json.dumps(commands) + assert all(installation_token not in body for body in askpass_bodies) + assert not (tmp_path / ".certforge-git-askpass").exists() + + +def test_acquire_private_github_redacts_token_from_git_failure(tmp_path, monkeypatch): + installation_token = "installation-token-must-not-leak" + monkeypatch.setattr( + "echo_certification_forge.acquisition._github_app_installation_token", + lambda _url: installation_token, + ) + + def fake_run(command, **_kwargs): + if "fetch" in command: + return subprocess.CompletedProcess( + command, + 128, + stdout="", + stderr=f"fatal: rejected {installation_token}", + ) + destination = tmp_path / "private-failure" + if "init" in command: + (destination / ".git").mkdir(parents=True) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr("echo_certification_forge.acquisition.subprocess.run", fake_run) + with pytest.raises(AcquisitionError) as error: + acquire_target( + { + "type": "git", + "url": "https://github.com/echoomegaprime/private-project.git", + "ref": "a" * 40, + }, + tmp_path / "private-failure", + ) + assert installation_token not in str(error.value) + assert "[REDACTED]" in str(error.value) + assert not (tmp_path / ".certforge-git-askpass").exists() + + def test_acquire_git_uses_bounded_configured_timeout(tmp_path, monkeypatch): source_commit = "a" * 40 observed_timeouts: list[float] = [] From 795ca1a927b8af649c91bfe74b84f55dc2e5c852 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Tue, 1 Sep 2026 11:28:26 -0700 Subject: [PATCH 12/39] docs: document private GitHub acquisition boundary --- docs/OPERATIONS.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 84723df..a41385e 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -37,6 +37,18 @@ allowing large repositories to materialize under normal disk contention. An invalid value fails the acquisition before creating the target destination; never remove the timeout to make a run pass. +Private GitHub targets use the dispatcher-owned GitHub App identity configured by +`ECHO_CERTFORGE_GITHUB_APP_ID` and +`ECHO_CERTFORGE_GITHUB_PRIVATE_KEY_FILE`. The worker mints a short-lived +installation token restricted to the requested repository and `contents:read`. +Git receives that token only through an ephemeral `GIT_ASKPASS` process +environment; the token is never placed in the repository URL, command line, +target record, evidence, or exception text. The askpass helper is removed after +acquisition on both success and failure. A partially configured App identity, +missing installation, invalid key, or token-mint failure blocks acquisition. +Do not replace this path with a PAT-bearing URL or a persistent credential +helper. + 1. Confirm hosted CI succeeded for the exact source SHA. 2. Execute the declared production-shaped journey with the pinned runner image. 3. Verify evidence custody, receipt chains, signatures, expiry, and revocation state. @@ -55,6 +67,12 @@ Capture the run ID, target SHA, environment digest, policy version, service heal and sanitized logs. A failed gate is not repaired by changing a verdict field; repair the failing evidence-producing control, rerun from a new idempotency key, and retain the failed run for audit. +For `acquisition_failed` with GitHub reporting `Repository not found`, first +confirm that both GitHub App settings are present, their files are readable by +the dispatcher, and that the App is installed on the named repository. Never +print the private key, minted token, process environment, or credential-helper +output while diagnosing the failure. + ## Recovery Production uses a current-release pointer and retained prior release. If post-promotion health or From dbd47e15a4d06234c1d485a173039be5cb126326 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Wed, 2 Sep 2026 02:11:14 -0700 Subject: [PATCH 13/39] fix(certforge): allow large exact-SHA checkouts --- deploy/deploy_forge.sh | 5 ++++- docs/OPERATIONS.md | 7 ++++++- tests/test_deploy_gate.py | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/deploy/deploy_forge.sh b/deploy/deploy_forge.sh index c736ac5..fd134ac 100644 --- a/deploy/deploy_forge.sh +++ b/deploy/deploy_forge.sh @@ -7,7 +7,10 @@ set -euo pipefail SOURCE_REPO="${CERTFORGE_SOURCE_REPO:-$(cd "$(dirname "$0")/.." && pwd)}" STAGING_PORT="${CERTFORGE_STAGING_PORT:-8311}" PROD_PORT="${CERTFORGE_PROD_PORT:-8309}" -GIT_ACQUISITION_TIMEOUT_SECONDS="${ECHO_CERTFORGE_GIT_ACQUISITION_TIMEOUT_SECONDS:-300}" +# The governed Echo monorepo is roughly 1 GiB and can require more than five +# minutes to materialize under normal FORGE disk contention. Keep a finite, +# supported ceiling while avoiding false infrastructure failures at 300 seconds. +GIT_ACQUISITION_TIMEOUT_SECONDS="${ECHO_CERTFORGE_GIT_ACQUISITION_TIMEOUT_SECONDS:-900}" SERVICE="echo-certforge" DISPATCH_SERVICE="echo-certforge-dispatcher" BRANCH="${CERTFORGE_BRANCH:-main}" diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index a41385e..4290af1 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -31,12 +31,17 @@ target-selected path or target-supplied public key. Missing, untrusted, stale, or identity-mismatched envelopes remain `NOT_READY`. Exact Git acquisition uses the bounded dispatcher setting -`ECHO_CERTFORGE_GIT_ACQUISITION_TIMEOUT_SECONDS` (production default: 300; +`ECHO_CERTFORGE_GIT_ACQUISITION_TIMEOUT_SECONDS` (production default: 900; allowed range: 30–1800 seconds). This bounds network and checkout stalls while allowing large repositories to materialize under normal disk contention. An invalid value fails the acquisition before creating the target destination; never remove the timeout to make a run pass. +The 900-second default is sized for the governed Echo monorepo (approximately +1 GiB and more than 23,000 files). The previous 300-second ceiling produced a +reproducible false infrastructure failure during exact-SHA checkout while the +disk remained healthy and the checkout continued making I/O progress. + Private GitHub targets use the dispatcher-owned GitHub App identity configured by `ECHO_CERTFORGE_GITHUB_APP_ID` and `ECHO_CERTFORGE_GITHUB_PRIVATE_KEY_FILE`. The worker mints a short-lived diff --git a/tests/test_deploy_gate.py b/tests/test_deploy_gate.py index 9f83072..2b9009e 100644 --- a/tests/test_deploy_gate.py +++ b/tests/test_deploy_gate.py @@ -109,6 +109,20 @@ def test_deploy_gate_uses_the_effective_production_database_everywhere() -> None assert 'ECHO_CERTFORGE_DB="$STATE_ROOT/certforge.sqlite3"' not in DEPLOY_SCRIPT +def test_deploy_gate_allows_large_exact_sha_checkout_without_removing_bound() -> None: + assert ( + 'GIT_ACQUISITION_TIMEOUT_SECONDS="${ECHO_CERTFORGE_GIT_ACQUISITION_TIMEOUT_SECONDS:-900}"' + in DEPLOY_SCRIPT + ) + assert "GIT_ACQUISITION_TIMEOUT_SECONDS < 30" in DEPLOY_SCRIPT + assert "GIT_ACQUISITION_TIMEOUT_SECONDS > 1800" in DEPLOY_SCRIPT + assert ( + "Environment=ECHO_CERTFORGE_GIT_ACQUISITION_TIMEOUT_SECONDS=" + "$GIT_ACQUISITION_TIMEOUT_SECONDS" + in DEPLOY_SCRIPT + ) + + def test_deploy_gate_restores_prior_service_lifecycle() -> None: assert ( 'PREV_ENABLED="$(systemctl is-enabled "$SERVICE.service"' From b30cf970bd801d30038135d637bb372474d8e22c Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Tue, 22 Sep 2026 02:03:04 -0500 Subject: [PATCH 14/39] fix: publish P4 completion after verified cleanup and enrichment Successful hostile acceptance reports omitted the completed_phase_gate field required by the master collector. Publish P4 only after every core check and cleanup succeeds, and keep exact-wrapper reports pending until enrichment returns. Failed verification, malformed capture, interruption and nonzero exit cannot leave a newly completed report for the master consumer. Keep master validation, exact-source and signing rules, release verdicts and historical acceptance reports unchanged. Add producer/wrapper/consumer integration coverage including pending publication and failure paths. Validation: 24 focused and adjacent tests pass; full suite 637 pass, 2 skip, 1 existing symlink test blocked by Windows privilege 1314. P1 acceptance passes all 3 scenarios using a byte-identical source copy outside historical artifacts. Hosted CI, dual certification and deployment remain pending. --- scripts/p4_hostile_acceptance.py | 21 +++- scripts/p4_hostile_acceptance_exact.py | 18 +++ tests/test_p4_master_report_contract.py | 141 ++++++++++++++++++++++++ 3 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 tests/test_p4_master_report_contract.py diff --git a/scripts/p4_hostile_acceptance.py b/scripts/p4_hostile_acceptance.py index b17dd0e..3ab0f36 100644 --- a/scripts/p4_hostile_acceptance.py +++ b/scripts/p4_hostile_acceptance.py @@ -136,6 +136,25 @@ def write_json(path: Path, value: Any) -> None: path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") +def write_acceptance_report(path: Path, report: dict[str, Any]) -> None: + """Publish phase completion only after checks and final cleanup succeed.""" + report.pop("completed_phase_gate", None) + checks = report.get("checks") + cleanup = report.get("cleanup") or {} + if ( + report.get("phase") == "P4" + and report.get("passed") is True + and report.get("run_outcome") == "COMPLETE" + and isinstance(checks, dict) + and checks + and all(value is True for value in checks.values()) + and cleanup.get("unrelated_container_ids_preserved") is True + and cleanup.get("ephemeral_private_files_removed") is True + ): + report["completed_phase_gate"] = "P4" + write_json(path, report) + + def docker_json(*arguments: str) -> Any: return json.loads(run(["docker", *arguments]).stdout) @@ -1859,7 +1878,7 @@ def main() -> int: report["error"] = {"type": "CleanupError", "message": "unrelated container identity changed"} return_code = 1 report["completed_at_utc"] = to_utc_iso(datetime.now(UTC)) - write_json(output, report) + write_acceptance_report(output, report) print( json.dumps( { diff --git a/scripts/p4_hostile_acceptance_exact.py b/scripts/p4_hostile_acceptance_exact.py index 0d6d61c..732ad99 100644 --- a/scripts/p4_hostile_acceptance_exact.py +++ b/scripts/p4_hostile_acceptance_exact.py @@ -147,6 +147,10 @@ def enrich_report( if not output_path.is_file(): raise RuntimeError(f"P4 harness did not produce an output report: {output_path}") report = json.loads(output_path.read_text(encoding="utf-8")) + # Enrichment is a further acceptance boundary; invalidate any earlier marker + # before parsing captured evidence, which can raise or be interrupted. + report.pop("completed_phase_gate", None) + output_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") report["harness"] = { "path": str(HERE), "sha256": sha256_file(HERE), @@ -205,6 +209,10 @@ def enrich_report( ) return_code = 1 report["wrapper_return_code"] = return_code + if return_code != 0: + report["passed"] = False + report["run_outcome"] = "INFRA_FAILED" + report["release_verdict"] = "NOT_READY" output_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") return report @@ -213,6 +221,15 @@ def main(arguments: list[str] | None = None) -> int: argv = list(sys.argv[1:] if arguments is None else arguments) sealed_manifest_path, output_path = parse_wrapper_paths(argv) harness = load_harness() + publish_report = harness.write_acceptance_report + + def write_pending_report(path: Path, report: dict[str, Any]) -> None: + report.pop("completed_phase_gate", None) + harness.write_json(path, report) + + # The core can finish before the exact verifier evidence is enriched. + # Keep its on-disk result pending until every wrapper layer has returned. + harness.write_acceptance_report = write_pending_report artifacts = resolve_runner_artifacts(harness, sealed_manifest_path) capture: dict[str, Any] = {} harness.execute_container = make_execute_container_override( @@ -234,6 +251,7 @@ def main(arguments: list[str] | None = None) -> int: return_code=return_code, ) final_code = 0 if report.get("passed") is True and report.get("run_outcome") == "COMPLETE" else 1 + publish_report(output_path, report) print( json.dumps( { diff --git a/tests/test_p4_master_report_contract.py b/tests/test_p4_master_report_contract.py new file mode 100644 index 0000000..4026343 --- /dev/null +++ b/tests/test_p4_master_report_contract.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import json +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +def load_script(name: str): + spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / f"{name}.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def completed_report() -> dict: + # Contract fixture only: no real P4 execution or certification is claimed. + return { + "phase": "P4", + "passed": True, + "run_outcome": "COMPLETE", + "release_verdict": "NOT_READY", + "source_commit": "a" * 40, + "checks": {"sealed_twelve_image_manifest_verified": True, "public_verifier_passed": True}, + "cleanup": { + "unrelated_container_ids_preserved": True, + "ephemeral_private_files_removed": True, + }, + } + + +def test_successful_producer_report_is_consumable_without_changing_master_gate(tmp_path: Path) -> None: + producer = load_script("p4_hostile_acceptance") + consumer = load_script("master_acceptance") + report = completed_report() + output = tmp_path / "p4.json" + + producer.write_acceptance_report(output, report) + + gate = consumer._gate("P4", output, consumer._p4) + assert gate == {"state": "COMPLETE", "evidence_sha256": hashlib.sha256(output.read_bytes()).hexdigest()} + emitted = json.loads(output.read_text()) + assert emitted["source_commit"] == "a" * 40 + assert emitted["release_verdict"] == "NOT_READY" + + +@pytest.mark.parametrize("failure", ["checks", "empty_checks", "outcome", "passed", "containers", "private_files"]) +def test_incomplete_producer_report_cannot_satisfy_master_gate(tmp_path: Path, failure: str) -> None: + producer = load_script("p4_hostile_acceptance") + consumer = load_script("master_acceptance") + report = completed_report() + # A stale marker must be removed if a later verification or cleanup fails. + report["completed_phase_gate"] = "P4" + if failure == "checks": + report["checks"]["public_verifier_passed"] = False + elif failure == "empty_checks": + report["checks"] = {} + elif failure == "outcome": + report["run_outcome"] = "INFRA_FAILED" + elif failure == "passed": + report["passed"] = False + elif failure == "containers": + report["cleanup"]["unrelated_container_ids_preserved"] = False + else: + report["cleanup"]["ephemeral_private_files_removed"] = False + output = tmp_path / "p4.json" + + producer.write_acceptance_report(output, report) + + with pytest.raises(ValueError, match="P4 gate is incomplete"): + consumer._gate("P4", output, consumer._p4) + assert "completed_phase_gate" not in json.loads(output.read_text()) + + +def test_wrapper_failure_removes_previously_completed_marker(tmp_path: Path) -> None: + producer = load_script("p4_hostile_acceptance") + wrapper = load_script("p4_hostile_acceptance_exact") + consumer = load_script("master_acceptance") + output = tmp_path / "p4.json" + producer.write_acceptance_report(output, completed_report()) + + result = wrapper.enrich_report(output_path=output, harness=producer, capture={}, return_code=1) + + assert result["passed"] is False + assert "completed_phase_gate" not in result + with pytest.raises(ValueError, match="P4 gate is incomplete"): + consumer._gate("P4", output, consumer._p4) + + +@pytest.mark.parametrize("scenario", ["success", "nonzero", "malformed", "interrupted"]) +def test_exact_entrypoint_publishes_only_after_complete_enrichment(tmp_path: Path, monkeypatch, scenario: str) -> None: + producer = load_script("p4_hostile_acceptance") + wrapper = load_script("p4_hostile_acceptance_exact") + consumer = load_script("master_acceptance") + output = tmp_path / "p4.json" + capture = { + "result": { + "exit_code": 0, "timed_out": False, "oom_killed": False, + "containment": {"network_none": True}, + "logs": json.dumps({"passed": True, "private_key_loaded": False, + "sbom_valid": True, "admission": {"allowed": True}}), + }, + "artifacts": {}, "execution_limits": {}, + } + if scenario == "malformed": + capture["result"] = None + + def core_main(): + producer.write_acceptance_report(output, completed_report()) + # Even a successful core result is pending while the wrapper is running. + with pytest.raises(ValueError, match="P4 gate is incomplete"): + consumer._gate("P4", output, consumer._p4) + if scenario == "interrupted": + raise RuntimeError("simulated interruption after core serialization") + return 1 if scenario == "nonzero" else 0 + + def capture_override(original, *, artifacts, capture: dict): + capture.update(captured) + return original + + captured = capture + monkeypatch.setattr(producer, "main", core_main) + monkeypatch.setattr(wrapper, "load_harness", lambda: producer) + monkeypatch.setattr(wrapper, "parse_wrapper_paths", lambda argv: (tmp_path / "manifest.json", output)) + monkeypatch.setattr(wrapper, "resolve_runner_artifacts", lambda *args: {}) + monkeypatch.setattr(wrapper, "make_execute_container_override", capture_override) + if scenario in {"malformed", "interrupted"}: + with pytest.raises((AttributeError, RuntimeError)): + wrapper.main([]) + else: + assert wrapper.main([]) == (0 if scenario == "success" else 1) + if scenario == "success": + assert consumer._gate("P4", output, consumer._p4)["state"] == "COMPLETE" + else: + with pytest.raises(ValueError, match="P4 gate is incomplete"): + consumer._gate("P4", output, consumer._p4) From bc1b8ad9535ad6c736b3d03037298f6307e727fc Mon Sep 17 00:00:00 2001 From: echo-ops-certforge Date: Thu, 24 Sep 2026 15:16:59 -0500 Subject: [PATCH 15/39] fix(certforge): patched base image and cryptography 50 for P4 sealing (#22) Release hardening so the P4 hostile acceptance gate can seal again: - Bump the pinned python:3.12-alpine base for all six role images, the sandbox default image and the P4 gate to sha256:4c47124a8391cb7a9f571164147d154777cf012a4ece5f86097130d7a4478111 (Trivy: 0 HIGH/CRITICAL findings on the new base). - Bump the hash-locked cryptography pin to 50.0.0 (all PyPI wheel and sdist hashes locked). - deploy_forge.sh: the dispatcher unit uses Wants= instead of Requires= on the API service so an API restart no longer tears down in-flight runs; the source-fetch git credential helper is configurable (CERTFORGE_GIT_CREDENTIAL_HELPER); the trusted manifest default is the canonical-JSON digest that run_worker pins (08ba068c...), not the raw file-bytes digest. - Opt the repository into its own CertForge gate with a stdlib-only journey (scripts/certforge_journey.py) and pin tests (tests/test_release_pins.py). The worker environment identity digest is label-derived and does not change with this release (8e6466d2...). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0152mN1wMwj9vE2YmV2xZF4F --- .echo/certification.json | 4 + deploy/deploy_forge.sh | 14 +- images/anchor/Dockerfile | 4 +- images/custody/Dockerfile | 4 +- images/requirements.in | 2 +- images/requirements.lock | 94 ++-- images/runner/Dockerfile | 4 +- images/signer/Dockerfile | 4 +- images/verifier/Dockerfile | 4 +- images/worker/Dockerfile | 4 +- scripts/certforge_journey.py | 76 ++++ scripts/certforge_testkit.py | 562 ++++++++++++++++++++++++ scripts/run_p4_gate.sh | 2 +- src/echo_certification_forge/sandbox.py | 2 +- tests/test_release_pins.py | 69 +++ 15 files changed, 784 insertions(+), 65 deletions(-) create mode 100644 .echo/certification.json create mode 100644 scripts/certforge_journey.py create mode 100644 scripts/certforge_testkit.py create mode 100644 tests/test_release_pins.py diff --git a/.echo/certification.json b/.echo/certification.json new file mode 100644 index 0000000..b06c33e --- /dev/null +++ b/.echo/certification.json @@ -0,0 +1,4 @@ +{ + "version": 1, + "journey": ["python3", "-B", "scripts/certforge_journey.py"] +} diff --git a/deploy/deploy_forge.sh b/deploy/deploy_forge.sh index fd134ac..7301e49 100644 --- a/deploy/deploy_forge.sh +++ b/deploy/deploy_forge.sh @@ -22,13 +22,19 @@ PRODUCTION_E2E_ATTESTATION_DIR="${ECHO_CERTFORGE_PRODUCTION_E2E_ATTESTATION_DIR: PRODUCTION_E2E_TRUSTED_KEYS="${ECHO_CERTFORGE_TRUSTED_PRODUCTION_E2E_KEYS:-$STATE_ROOT/production-e2e/trusted-public-keys}" ADAPTER_DIR="${ECHO_CERTFORGE_PROD_ADAPTER_DIR:-$STATE_ROOT/p5}" ADAPTER_MODE="${CERTFORGE_ADAPTER_MODE:-required}" -TRUSTED_MANIFEST_SHA256="${ECHO_CERTFORGE_TRUSTED_MANIFEST_SHA256:-965106b00917268d556b325719f26f5096e6c3746551658ffecb9fd4a95ec342}" +# Canonical-JSON digest of policies/mandatory-rules.v2.json (== run_worker._PRODUCTION_MANIFEST_SHA256). +# The old default (965106b0...) was the raw file-bytes digest, which run_worker rejects. +TRUSTED_MANIFEST_SHA256="${ECHO_CERTFORGE_TRUSTED_MANIFEST_SHA256:-08ba068ceb3e14bfed2690337edbb94c546e3e0a1a89b1321f7657653d8eea43}" UNIT_PATH="/etc/systemd/system/$SERVICE.service" DISPATCH_UNIT_PATH="/etc/systemd/system/$DISPATCH_SERVICE.service" RELEASE_DROPIN="/etc/systemd/system/$SERVICE.service.d/10-release.conf" DISPATCH_RELEASE_DROPIN="/etc/systemd/system/$DISPATCH_SERVICE.service.d/10-release.conf" ENV_FILE="${CERTFORGE_ENV_FILE:-/home/forge/.config/echo/certforge.env}" -GITC=(-c credential.helper= -c credential.helper="store --file=/home/forge/.config/echo/omega_git_creds") +# Source fetch credentials. Default: the FORGE credential store. Repositories on the +# echoomegaprime account are fetched with a repo-scoped GitHub App helper instead +# (CERTFORGE_GIT_CREDENTIAL_HELPER), because that store cannot see them. +GIT_CREDENTIAL_HELPER="${CERTFORGE_GIT_CREDENTIAL_HELPER:-store --file=/home/forge/.config/echo/omega_git_creds}" +GITC=(-c credential.helper= -c credential.helper="$GIT_CREDENTIAL_HELPER") LOCK_FILE="${CERTFORGE_DEPLOY_LOCK:-/run/lock/echo-certforge-deploy.lock}" exec 9>"$LOCK_FILE" @@ -577,7 +583,9 @@ sudo tee "$DISPATCH_UNIT_PATH" >/dev/null < int: + print(f"CERTFORGE_SELF_JOURNEY_FAILED: {message}", file=sys.stderr) + return 1 + + +def canonical_digest(path: Path) -> str: + value = json.loads(path.read_text(encoding="utf-8")) + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), + ensure_ascii=False).encode("utf-8")).hexdigest() + + +def main() -> int: + cert = json.loads((ROOT / ".echo/certification.json").read_text(encoding="utf-8")) + apps = json.loads((ROOT / ".echo/apps.json").read_text(encoding="utf-8")) + if cert.get("version") != 1 or apps.get("apps", {}).get("certification-forge", {}).get("enabled") is not True: + return fail("opt-in contract invalid") + + manifest_digest = canonical_digest(ROOT / "policies/mandatory-rules.v2.json") + run_worker = (ROOT / "src/echo_certification_forge/run_worker.py").read_text(encoding="utf-8") + pinned = re.search(r'_PRODUCTION_MANIFEST_SHA256 = \(\s*"([0-9a-f]{64})"', run_worker) + deploy = (ROOT / "deploy/deploy_forge.sh").read_text(encoding="utf-8") + deploy_default = re.search(r'TRUSTED_MANIFEST_SHA256="\$\{ECHO_CERTFORGE_TRUSTED_MANIFEST_SHA256:-([0-9a-f]{64})\}"', + deploy) + if not pinned or pinned.group(1) != manifest_digest: + return fail(f"run_worker manifest pin != canonical policy digest {manifest_digest}") + if not deploy_default or deploy_default.group(1) != manifest_digest: + return fail("deploy default trusted manifest digest != canonical policy digest") + + compiled = 0 + for path in sorted((ROOT / "src").rglob("*.py")): + try: + compile(path.read_text(encoding="utf-8"), str(path), "exec") + except SyntaxError as exc: + return fail(f"syntax error: {path.relative_to(ROOT)}: {exc}") + compiled += 1 + print(json.dumps({"manifest_sha256": manifest_digest, "modules_compiled": compiled})) + # The two release suites are self-contained text/contract checks; tests/conftest.py needs + # pydantic/cryptography, which the stdlib-only sandbox does not have. + return certforge_testkit.run(ROOT, SUITES, sys_paths=("tests", "."), budget_s=60.0, + min_passed=15, conftest=False) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/certforge_testkit.py b/scripts/certforge_testkit.py new file mode 100644 index 0000000..ce78108 --- /dev/null +++ b/scripts/certforge_testkit.py @@ -0,0 +1,562 @@ +"""Stdlib-only test runner for Certification Forge journeys (ops-20260924). + +The CertForge sandbox runs a journey in ``python:3.12-alpine`` with no network, no +third-party packages, a read-only checkout and a 90 s wall clock. Most ECHO suites are +written for pytest, so this module provides the small pytest surface they use: +``raises``, ``mark.parametrize`` / ``skip`` / ``skipif`` (other marks are no-ops), ``param``, +``skip()``, ``fail()``, ``importorskip()``, ``approx``, and ``@fixture`` functions (plain or +``yield``) from the suite module and its ``conftest.py`` files, plus the built-in fixtures +``tmp_path``, ``monkeypatch`` and ``capsys``. It also runs ``unittest.TestCase`` classes and +pytest-style ``Test*`` classes. + +Third-party modules that a suite imports but never exercises can be replaced with inert +stubs (``stub_modules``). Stubs are installed only when the real module is missing, and the +report names them. Every failure, error, import error, empty run or blown time budget fails +the journey (fail closed). +""" +from __future__ import annotations + +import contextlib +import importlib +import importlib.util +import inspect +import io +import itertools +import os +import re +import sys +import tempfile +import time +import traceback +import types +import unittest +from pathlib import Path +from typing import Any, Callable, Iterable + + +class Skipped(Exception): + """Raised by ``pytest.skip`` / ``importorskip``.""" + + +class Failed(AssertionError): + """Raised by ``pytest.fail``.""" + + +class BudgetExceeded(Exception): + """The journey ran out of its wall-clock budget (fails the journey).""" + + +# ── pytest compatibility surface ───────────────────────────────────────────────────────── +class _Raises: + def __init__(self, expected: Any, match: str | None = None) -> None: + self.expected = expected + self.match = match + self.value: BaseException | None = None + + def __enter__(self) -> "_Raises": + return self + + def __exit__(self, exc_type, exc, tb) -> bool: + if exc_type is None: + raise Failed(f"DID NOT RAISE {self.expected}") + if not issubclass(exc_type, self.expected): + return False + if self.match is not None and not re.search(self.match, str(exc)): + raise Failed(f"regex {self.match!r} did not match {str(exc)!r}") + self.value = exc + return True + + +class _Param: + def __init__(self, *values: Any, id: str | None = None, marks: Any = ()) -> None: # noqa: A002 + self.values = values + self.id = id + self.marks = marks + + +class _NoopMark: + def __call__(self, *args: Any, **kwargs: Any) -> Any: + if len(args) == 1 and not kwargs and callable(args[0]): + return args[0] + return lambda fn: fn + + +class _Mark: + @staticmethod + def parametrize(names: Any, values: Iterable[Any], **_: Any) -> Callable: + if isinstance(names, str): + names = [n.strip() for n in names.split(",") if n.strip()] + names = list(names) + rows = [] + for value in values: + if isinstance(value, _Param): + value = value.values if len(names) > 1 else value.values[0] + rows.append(tuple(value) if len(names) > 1 else (value,)) + + def deco(fn: Callable) -> Callable: + fn.__dict__.setdefault("_cf_params", []).insert(0, (names, rows)) + return fn + return deco + + @staticmethod + def skip(reason: str = "") -> Callable: + def deco(fn: Callable) -> Callable: + fn._cf_skip = reason or "skip" + return fn + return deco + + @staticmethod + def skipif(condition: Any, reason: str = "") -> Callable: + def deco(fn: Callable) -> Callable: + if condition: + fn._cf_skip = reason or "skipif" + return fn + return deco + + def __getattr__(self, name: str) -> _NoopMark: + return _NoopMark() + + +class _Approx: + def __init__(self, expected: Any, rel: float | None = None, abs: float | None = None) -> None: # noqa: A002 + self.expected, self.rel, self.abs = expected, rel, abs + + def _close(self, a: float, b: float) -> bool: + rel = 1e-6 if self.rel is None else self.rel + tol = max(rel * abs(b), 1e-12 if self.abs is None else self.abs) + return abs(a - b) <= tol + + def __eq__(self, other: Any) -> bool: + if isinstance(self.expected, (list, tuple)): + return len(other) == len(self.expected) and all( + self._close(a, b) for a, b in zip(other, self.expected)) + if isinstance(self.expected, dict): + return other.keys() == self.expected.keys() and all( + self._close(other[k], v) for k, v in self.expected.items()) + return self._close(other, self.expected) + + def __repr__(self) -> str: + return f"approx({self.expected!r})" + + +def _fixture(*args: Any, **kwargs: Any) -> Any: + def mark(fn: Callable) -> Callable: + fn._cf_fixture = True + return fn + if len(args) == 1 and callable(args[0]) and not kwargs: + return mark(args[0]) + return mark + + +def _skip(reason: str = "", **_: Any) -> None: + raise Skipped(reason) + + +def _fail(reason: str = "", **_: Any) -> None: + raise Failed(reason) + + +def _importorskip(name: str, *_: Any, **__: Any) -> Any: + try: + return importlib.import_module(name) + except ImportError as exc: + raise Skipped(f"{name} unavailable: {exc}") from None + + +def pytest_module() -> types.ModuleType: + mod = types.ModuleType("pytest") + mod.__dict__.update( + raises=_Raises, mark=_Mark(), param=_Param, fixture=_fixture, skip=_skip, + fail=_fail, importorskip=_importorskip, approx=_Approx, + __certforge_shim__=True, + ) + mod.skip.Exception = Skipped # type: ignore[attr-defined] + return mod + + +# ── built-in fixtures ──────────────────────────────────────────────────────────────────── +def _resolve_dotted(path: str) -> Any: + """Import the longest importable prefix of ``path``, then walk the remaining attributes.""" + parts = path.split(".") + for cut in range(len(parts), 0, -1): + try: + obj = importlib.import_module(".".join(parts[:cut])) + except ImportError: + continue + for attr in parts[cut:]: + obj = getattr(obj, attr) + return obj + raise ImportError(f"cannot resolve {path!r}") + + +class MonkeyPatch: + _MISSING = object() + + def __init__(self) -> None: + self._undo: list[Callable[[], None]] = [] + + def setattr(self, target: Any, name: Any, value: Any = _MISSING, raising: bool = True) -> None: + if value is self._MISSING: # "pkg.mod.attr" form + value = name + owner_path, _, name = str(target).rpartition(".") + target = _resolve_dotted(owner_path) + old = getattr(target, name, self._MISSING) + if old is self._MISSING and raising: + raise AttributeError(f"{target!r} has no attribute {name!r}") + setattr(target, name, value) + self._undo.append(lambda: delattr(target, name) if old is self._MISSING + else setattr(target, name, old)) + + def delattr(self, target: Any, name: str, raising: bool = True) -> None: + if not hasattr(target, name): + if raising: + raise AttributeError(name) + return + old = getattr(target, name) + delattr(target, name) + self._undo.append(lambda: setattr(target, name, old)) + + def setitem(self, mapping: Any, key: Any, value: Any) -> None: + old = mapping.get(key, self._MISSING) + mapping[key] = value + self._undo.append(lambda: mapping.pop(key, None) if old is self._MISSING + else mapping.__setitem__(key, old)) + + def delitem(self, mapping: Any, key: Any, raising: bool = True) -> None: + if key not in mapping: + if raising: + raise KeyError(key) + return + old = mapping.pop(key) + self._undo.append(lambda: mapping.__setitem__(key, old)) + + def setenv(self, name: str, value: Any, prepend: str | None = None) -> None: + value = str(value) + if prepend and name in os.environ: + value = value + prepend + os.environ[name] + self.setitem(os.environ, name, value) + + def delenv(self, name: str, raising: bool = True) -> None: + self.delitem(os.environ, name, raising) + + def syspath_prepend(self, path: Any) -> None: + sys.path.insert(0, str(path)) + self._undo.append(lambda: sys.path.remove(str(path))) + + def chdir(self, path: Any) -> None: + old = os.getcwd() + os.chdir(path) + self._undo.append(lambda: os.chdir(old)) + + def undo(self) -> None: + while self._undo: + self._undo.pop()() + + +class CapSys: + def __init__(self) -> None: + self._out, self._err = io.StringIO(), io.StringIO() + self._saved = (sys.stdout, sys.stderr) + sys.stdout, sys.stderr = self._out, self._err + + def readouterr(self) -> Any: + out, err = self._out.getvalue(), self._err.getvalue() + self._out.seek(0), self._out.truncate(), self._err.seek(0), self._err.truncate() + return types.SimpleNamespace(out=out, err=err) + + def close(self) -> None: + sys.stdout, sys.stderr = self._saved + + +# ── inert stubs for unexercised third-party imports ────────────────────────────────────── +class _StubObject: + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + if len(args) == 1 and callable(args[0]) and not kwargs: + return args[0] # used as a decorator + return _StubObject() + + def __getattr__(self, name: str) -> Any: + return _StubObject() + + +def _stub_module(name: str) -> types.ModuleType: + mod = types.ModuleType(name) + mod.__path__ = [] # allow "import pkg.sub" + mod.__certforge_stub__ = True + + def __getattr__(attr: str) -> Any: + if attr.startswith("__"): + raise AttributeError(attr) + base = Exception if attr.endswith(("Error", "Exception")) else _StubObject + value = type(attr, (base,), {"__module__": name}) + setattr(mod, attr, value) + return value + mod.__getattr__ = __getattr__ # type: ignore[attr-defined] + return mod + + +def install_stubs(names: Iterable[str]) -> list[str]: + installed = [] + for name in names: + if name in sys.modules: + continue + try: + importlib.import_module(name) + continue + except ImportError: + pass + sys.modules[name] = _stub_module(name) + parent, _, child = name.rpartition(".") + if parent and parent in sys.modules: + setattr(sys.modules[parent], child, sys.modules[name]) + installed.append(name) + return installed + + +# ── collection and execution ───────────────────────────────────────────────────────────── +class _Session: + def __init__(self, root: Path, deadline: float) -> None: + self.root = root + self.deadline = deadline + self.passed = self.failed = self.skipped = 0 + self.failures: list[str] = [] + + # fixtures ---------------------------------------------------------------------------- + def _resolve(self, name: str, fixtures: dict, cache: dict, teardown: list) -> Any: + if name in cache: + return cache[name] + if name == "tmp_path": + value = Path(tempfile.mkdtemp(prefix="cf-")) + elif name == "monkeypatch": + value = MonkeyPatch() + teardown.append(value.undo) + elif name == "capsys": + value = CapSys() + teardown.append(value.close) + elif name == "request": + value = types.SimpleNamespace(param=None, node=None, config=None) + elif name in fixtures: + fn = fixtures[name] + kwargs = {p: self._resolve(p, fixtures, cache, teardown) + for p in inspect.signature(fn).parameters} + value = fn(**kwargs) + if inspect.isgenerator(value): + gen = value + value = next(gen) + teardown.append(lambda g=gen: next(g, None)) + else: + raise LookupError(f"fixture {name!r} not found") + cache[name] = value + return value + + def _call(self, label: str, fn: Callable, fixtures: dict, bound: dict) -> None: + if time.monotonic() > self.deadline: + raise BudgetExceeded(f"journey time budget exhausted before {label}") + skip = getattr(fn, "_cf_skip", None) + if skip: + self.skipped += 1 + return + cache: dict = dict(bound) + teardown: list = [] + try: + params = [p for p in inspect.signature(fn).parameters if p != "self"] + kwargs = {p: self._resolve(p, fixtures, cache, teardown) for p in params} + fn(**kwargs) + self.passed += 1 + except Skipped: + self.skipped += 1 + except (BudgetExceeded, KeyboardInterrupt): + raise + except BaseException as exc: # noqa: BLE001 - every error (incl. SystemExit) is a failure + self.failed += 1 + self.failures.append(f"{label}: {type(exc).__name__}: {exc}\n" + + "".join(traceback.format_exc(limit=6))[-1500:]) + finally: + for fin in reversed(teardown): + with contextlib.suppress(Exception): + fin() + + def _expand(self, label: str, fn: Callable, fixtures: dict) -> None: + grids = getattr(fn, "_cf_params", []) + if not grids: + self._call(label, fn, fixtures, {}) + return + for i, combo in enumerate(itertools.product(*[rows for _, rows in grids])): + bound: dict = {} + for (names, _), row in zip(grids, combo): + bound.update(zip(names, row)) + self._call(f"{label}[{i}]", fn, fixtures, bound) + + # suites ------------------------------------------------------------------------------ + def _conftest_fixtures(self, suite: Path) -> dict: + fixtures: dict = {} + if not getattr(self, "use_conftest", True): + return fixtures # self-contained suites; conftest needs packages the sandbox lacks + chain = [d for d in [suite.parent, *suite.parent.parents] if self.root in (d, *d.parents)] + for directory in reversed(chain): + conftest = directory / "conftest.py" + if conftest.is_file(): + mod = _load(conftest, "cf_conftest_" + _slug(conftest.relative_to(self.root))) + fixtures.update(_fixtures_of(mod)) + return fixtures + + def run_suite(self, suite: Path) -> None: + fixtures = self._conftest_fixtures(suite) + module = _load(suite, "cf_suite_" + _slug(suite.relative_to(self.root))) + fixtures.update(_fixtures_of(module)) + rel = str(suite.relative_to(self.root)) + for name, obj in list(vars(module).items()): + if isinstance(obj, type) and issubclass(obj, unittest.TestCase): + result = unittest.TestResult() + unittest.defaultTestLoader.loadTestsFromTestCase(obj).run(result) + self.passed += result.testsRun - len(result.failures) - len(result.errors) - len(result.skipped) + self.skipped += len(result.skipped) + for case, tb in result.failures + result.errors: + self.failed += 1 + self.failures.append(f"{rel}::{case.id()}\n{tb[-1500:]}") + elif isinstance(obj, type) and name.startswith("Test") and "__init__" not in vars(obj): + for meth in [m for m in dir(obj) if m.startswith("test")]: + inst = obj() + if hasattr(inst, "setup_method"): + inst.setup_method(getattr(inst, meth)) + try: + self._expand(f"{rel}::{name}::{meth}", getattr(inst, meth), fixtures) + finally: + if hasattr(inst, "teardown_method"): + inst.teardown_method(getattr(inst, meth)) + elif (name.startswith("test") and inspect.isfunction(obj) + and obj.__module__ == module.__name__): + self._expand(f"{rel}::{name}", obj, fixtures) + + +def _slug(path: Any) -> str: + return re.sub(r"[^0-9A-Za-z]+", "_", str(path)) + + +def _load(path: Path, name: str) -> types.ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise ImportError(f"cannot load {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def _fixtures_of(module: types.ModuleType) -> dict: + return {n: f for n, f in vars(module).items() if callable(f) and getattr(f, "_cf_fixture", False)} + + +def run(root: Path, suites: Iterable[str], *, sys_paths: Iterable[str] = (), + stub_modules: Iterable[str] = (), budget_s: float = 75.0, min_passed: int = 1, + conftest: bool = True) -> int: + """Run ``suites`` (paths relative to ``root``); return a process exit code (0 = PASS).""" + import json + + start = time.monotonic() + sys.dont_write_bytecode = True # read-only checkout + for entry in reversed(list(sys_paths)): + sys.path.insert(0, str(root / entry)) + shim = importlib.util.find_spec("pytest") is None + if shim: + sys.modules["pytest"] = pytest_module() + stubs = install_stubs(stub_modules) + suites = list(suites) + missing = [s for s in suites if not (root / s).is_file()] + if missing: + print("CERTFORGE_TESTKIT_FAILED: missing suites: " + ", ".join(missing), file=sys.stderr) + return 1 + session = _Session(root, start + budget_s) + session.use_conftest = conftest + aborted = None + for suite in suites: + try: + session.run_suite(root / suite) + except BudgetExceeded as exc: + aborted = str(exc) + break + except BaseException as exc: # noqa: BLE001 - an import/collection error fails the suite + session.failed += 1 + session.failures.append(f"{suite}: collection error {type(exc).__name__}: {exc}\n" + + traceback.format_exc(limit=6)[-1500:]) + for failure in session.failures[:20]: + print("FAIL " + failure, file=sys.stderr) + report = { + "suites": len(suites), "passed": session.passed, "failed": session.failed, + "skipped": session.skipped, "pytest_shim": shim, "stubbed_modules": stubs, + "elapsed_s": round(time.monotonic() - start, 2), "aborted": aborted, + } + ok = aborted is None and session.failed == 0 and session.passed >= min_passed + print(("CERTFORGE_TESTKIT_OK " if ok else "CERTFORGE_TESTKIT_FAILED ") + + json.dumps(report, sort_keys=True)) + return 0 if ok else 1 + + +def run_isolated(root: Path, suites: Iterable[dict], *, budget_s: float = 60.0, + per_suite_timeout_s: float = 30.0, min_passed: int = 1) -> int: + """Run each suite in its own interpreter (no module-name collisions between suites). + + ``suites``: dicts with ``path`` plus optional ``sys_paths`` / ``stub_modules``. + """ + import json + import subprocess + + start = time.monotonic() + suites = list(suites) + totals = {"passed": 0, "failed": 0, "skipped": 0} + failed_suites: list[str] = [] + for spec in suites: + remaining = budget_s - (time.monotonic() - start) + if remaining <= 1: + failed_suites.append(f"{spec['path']}: journey time budget exhausted") + break + argv = [sys.executable, "-B", str(Path(__file__).resolve()), "--root", str(root), + "--suite", spec["path"]] + for entry in spec.get("sys_paths", []): + argv += ["--sys-path", entry] + for name in spec.get("stub_modules", []): + argv += ["--stub", name] + try: + proc = subprocess.run(argv, cwd=str(root), capture_output=True, text=True, + timeout=min(per_suite_timeout_s, remaining), check=False) + except subprocess.TimeoutExpired: + failed_suites.append(f"{spec['path']}: timed out") + continue + line = next((ln for ln in reversed(proc.stdout.splitlines()) + if ln.startswith("CERTFORGE_TESTKIT_")), "") + try: + report = json.loads(line.split(" ", 1)[1]) + except (IndexError, ValueError): + report = {} + for key in totals: + totals[key] += int(report.get(key, 0) or 0) + status = "ok" if proc.returncode == 0 else "FAILED" + print(f"suite {status} {spec['path']} passed={report.get('passed')} " + f"failed={report.get('failed')} skipped={report.get('skipped')} " + f"elapsed_s={report.get('elapsed_s')}") + if proc.returncode != 0: + failed_suites.append(spec["path"]) + sys.stderr.write(proc.stderr[-3000:]) + ok = not failed_suites and totals["failed"] == 0 and totals["passed"] >= min_passed + summary = dict(totals, suites=len(suites), failed_suites=failed_suites, + elapsed_s=round(time.monotonic() - start, 2)) + print(("CERTFORGE_SUITES_OK " if ok else "CERTFORGE_SUITES_FAILED ") + + json.dumps(summary, sort_keys=True)) + return 0 if ok else 1 + + +if __name__ == "__main__": # single-suite worker used by run_isolated + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--root", required=True) + parser.add_argument("--suite", required=True) + parser.add_argument("--sys-path", action="append", default=[]) + parser.add_argument("--stub", action="append", default=[]) + parser.add_argument("--budget", type=float, default=60.0) + ns = parser.parse_args() + raise SystemExit(run(Path(ns.root), [ns.suite], sys_paths=ns.sys_path, + stub_modules=ns.stub, budget_s=ns.budget)) diff --git a/scripts/run_p4_gate.sh b/scripts/run_p4_gate.sh index 046b33b..21bce96 100644 --- a/scripts/run_p4_gate.sh +++ b/scripts/run_p4_gate.sh @@ -25,7 +25,7 @@ set -Eeuo pipefail REPO_URL="${REPO_URL:-https://github.com/ECHO-OMEGA-PRIME/echo-certification-forge}" BRANCH="${BRANCH:-feat/certforge-r5-negative-controls}" GIT_CRED_FILE="${GIT_CRED_FILE:-/home/forge/.config/echo/omega_git_creds}" -BASE_DIGEST="${BASE_DIGEST:-sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df}" +BASE_DIGEST="${BASE_DIGEST:-sha256:4c47124a8391cb7a9f571164147d154777cf012a4ece5f86097130d7a4478111}" CLAMAV_IMAGE="${CLAMAV_IMAGE:-clamav/clamav@sha256:7f5389ccaa2368c383fa80e167ccfe44348d71e685f926fce4755eed1757673a}" COSIGN="${COSIGN:-/home/forge/.cache/echo-certforge/p4-9c07eb7/tools/cosign/cosign}" TRIVY="${TRIVY:-/home/forge/.cache/echo-certforge/p4-9c07eb7/tools/trivy/trivy}" diff --git a/src/echo_certification_forge/sandbox.py b/src/echo_certification_forge/sandbox.py index a6ed098..96ed7f9 100644 --- a/src/echo_certification_forge/sandbox.py +++ b/src/echo_certification_forge/sandbox.py @@ -25,7 +25,7 @@ from typing import Callable # Pinned minimal Python base (same digest the P4 supply-chain pipeline pins). Override per policy. -DEFAULT_IMAGE = "python:3.12-alpine@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df" +DEFAULT_IMAGE = "python:3.12-alpine@sha256:4c47124a8391cb7a9f571164147d154777cf012a4ece5f86097130d7a4478111" class SandboxError(RuntimeError): diff --git a/tests/test_release_pins.py b/tests/test_release_pins.py new file mode 100644 index 0000000..152dc36 --- /dev/null +++ b/tests/test_release_pins.py @@ -0,0 +1,69 @@ +"""ops-20260924: the supply-chain pins stay consistent and patched (issue #22 security release). + +P4 image sealing fails closed on HIGH vulnerabilities with an available fix. This guards the +fix: one patched base image digest everywhere, the patched cryptography release hash-locked, +and the dispatcher unit decoupled from API restarts. +""" +from __future__ import annotations + +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +PATCHED_BASE = "sha256:4c47124a8391cb7a9f571164147d154777cf012a4ece5f86097130d7a4478111" +VULNERABLE_BASE = "sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df" +ROLES = ("runner", "custody", "anchor", "signer", "worker", "verifier") + + +def _text(rel: str) -> str: + return (ROOT / rel).read_text(encoding="utf-8") + + +def test_every_role_image_uses_the_patched_base() -> None: + for role in ROLES: + dockerfile = _text(f"images/{role}/Dockerfile") + assert dockerfile.startswith(f"FROM python:3.12-alpine@{PATCHED_BASE}\n"), role + assert f'org.opencontainers.image.base.digest="{PATCHED_BASE}"' in dockerfile, role + assert VULNERABLE_BASE not in dockerfile, role + + +def test_sandbox_and_p4_gate_share_the_patched_base() -> None: + assert f'DEFAULT_IMAGE = "python:3.12-alpine@{PATCHED_BASE}"' in _text( + "src/echo_certification_forge/sandbox.py") + assert f'BASE_DIGEST="${{BASE_DIGEST:-{PATCHED_BASE}}}"' in _text("scripts/run_p4_gate.sh") + + +def test_cryptography_is_the_patched_hash_locked_release() -> None: + assert "cryptography==50.0.0" in _text("images/requirements.in") + lock = _text("images/requirements.lock").replace("\r\n", "\n") + block = re.search(r"^cryptography==(\S+) \\\n((?: --hash=sha256:[0-9a-f]{64}(?: \\)?\n)+)", lock, re.M) + assert block is not None + major = int(block.group(1).split(".")[0]) + assert major >= 50 + assert len(re.findall(r"sha256:[0-9a-f]{64}", block.group(2))) >= 20 + assert "cryptography==49" not in lock + + +def test_dispatcher_wants_but_does_not_require_the_api_service() -> None: + script = _text("deploy/deploy_forge.sh") + dispatcher = script[script.index('sudo tee "$DISPATCH_UNIT_PATH"'):] + unit = dispatcher[: dispatcher.index("[Service]")] + assert "Wants=$SERVICE.service" in unit + assert "Requires=$SERVICE.service" not in unit + assert "After=network.target $SERVICE.service" in unit + + +def test_source_fetch_credential_helper_is_configurable_with_safe_default() -> None: + script = _text("deploy/deploy_forge.sh") + assert ('GIT_CREDENTIAL_HELPER="${CERTFORGE_GIT_CREDENTIAL_HELPER:-store --file=' + '/home/forge/.config/echo/omega_git_creds}"') in script + assert 'GITC=(-c credential.helper= -c credential.helper="$GIT_CREDENTIAL_HELPER")' in script + + +def test_deploy_default_manifest_digest_matches_the_enforced_pin() -> None: + script = _text("deploy/deploy_forge.sh") + run_worker = _text("src/echo_certification_forge/run_worker.py") + pinned = re.search(r'_PRODUCTION_MANIFEST_SHA256 = \(\s*"([0-9a-f]{64})"', run_worker) + default = re.search(r'TRUSTED_MANIFEST_SHA256="\$\{ECHO_CERTFORGE_TRUSTED_MANIFEST_SHA256:-([0-9a-f]{64})\}"', + script) + assert pinned and default and default.group(1) == pinned.group(1) From d8ad9fd881365801354b78a758632d2a31ab6828 Mon Sep 17 00:00:00 2001 From: echo-ops-certforge Date: Thu, 24 Sep 2026 16:04:54 -0500 Subject: [PATCH 16/39] fix(p4): keep the worker health probe's deployment ledger on the workspace (#22) The fresh P4 hostile-acceptance run for bc1b8ad built and sealed all 12 images and passed the scans, the static and runtime attack matrices and the custody and anchor service probes. It then stopped as INFRA_FAILED: the worker container exited before /healthz answered. Cause: since P6 (bfc2159) the worker app opens its deployment ledger at import time. Its default path, /var/deployments.sqlite3, resolves to /opt/var on the read-only image root, so uvicorn died with "Read-only file system: '/opt/var'". The last green P4 (9c07eb7) predates P6, so no P6-era source could pass P4 until now. Fix: the hardened probe sets ECHO_CERTFORGE_DEPLOYMENT_LEDGER to /workspace/state/deployments.sqlite3, next to the other state paths. Only the harness changes; production and the image stay as they are. Reproduced with the sealed worker image: exit 1 without the variable, healthy with it. A new test pins every worker state path to the writable workspace; it fails without this change. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0152mN1wMwj9vE2YmV2xZF4F --- scripts/p4_hostile_acceptance.py | 4 +++ tests/test_p4_worker_service_env.py | 51 +++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 tests/test_p4_worker_service_env.py diff --git a/scripts/p4_hostile_acceptance.py b/scripts/p4_hostile_acceptance.py index 3ab0f36..f05e19c 100644 --- a/scripts/p4_hostile_acceptance.py +++ b/scripts/p4_hostile_acceptance.py @@ -1394,6 +1394,10 @@ def service_health_probe(image: str, role: ImageRole, ownership_token: str, work "ECHO_CERTFORGE_DB": "/workspace/state/certforge.sqlite3", "ECHO_CERTFORGE_EVIDENCE_ROOT": "/workspace/state/evidence", "ECHO_CERTFORGE_TRUSTED_KEYS": "/workspace/state/trusted-public-keys", + # The worker app opens its P6 deployment ledger at import time. The default + # (/var/deployments.sqlite3) is on the read-only image root, so + # the ledger joins the other state paths on the writable workspace. + "ECHO_CERTFORGE_DEPLOYMENT_LEDGER": "/workspace/state/deployments.sqlite3", } ) command = ["-lc", "mkdir -p /workspace/state/trusted-public-keys /workspace/state/evidence && exec uvicorn echo_certification_forge.app:app --host 127.0.0.1 --port 8080 --no-access-log"] diff --git a/tests/test_p4_worker_service_env.py b/tests/test_p4_worker_service_env.py new file mode 100644 index 0000000..db4d071 --- /dev/null +++ b/tests/test_p4_worker_service_env.py @@ -0,0 +1,51 @@ +"""The P4 worker health probe keeps every state path on the writable workspace. + +The hardened probe runs the worker image with a read-only root. Since P6 the worker app opens its +deployment ledger at import time, and its default path lives on the image root, so the probe died with +"Read-only file system: '/opt/var'" before /healthz answered (P4 INFRA_FAILED, ops-20260924). +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "p4_hostile_acceptance.py" +STATE_PATHS = ( + "ECHO_CERTFORGE_DB", + "ECHO_CERTFORGE_EVIDENCE_ROOT", + "ECHO_CERTFORGE_TRUSTED_KEYS", + "ECHO_CERTFORGE_DEPLOYMENT_LEDGER", +) + + +class _Captured(Exception): + pass + + +def _load_harness(): + spec = importlib.util.spec_from_file_location("p4_hostile_acceptance_worker_env_test", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_worker_health_probe_keeps_state_on_the_writable_workspace(monkeypatch, tmp_path: Path) -> None: + harness = _load_harness() + captured: dict = {} + + def fake_create_container(**kwargs): + captured.update(kwargs) + raise _Captured() + + monkeypatch.setattr(harness, "create_container", fake_create_container) + with pytest.raises(_Captured): + harness.service_health_probe("image@sha256:" + "0" * 64, harness.ImageRole.WORKER, "token-12345678", tmp_path) + environment = captured["environment"] + assert environment["ECHO_CERTFORGE_DEPLOYMENT_LEDGER"] == "/workspace/state/deployments.sqlite3" + for name in STATE_PATHS: + assert environment[name].startswith("/workspace/state/"), name From 0ab617ab6ed97e1a40e5cb3d160215dc93dc86d5 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Wed, 11 Nov 2026 00:45:18 +0000 Subject: [PATCH 17/39] fix(subscriber): integrate guarded rollback and orphan dispatch recovery (#40 #41) --- src/echo_certification_forge/subscriber.py | 132 +++++++++++++++++++- tests/test_dispatch_orphaned_runs.py | 117 +++++++++++++++++ tests/test_subscriber_migration_rollback.py | 76 +++++++++++ 3 files changed, 323 insertions(+), 2 deletions(-) create mode 100644 tests/test_dispatch_orphaned_runs.py create mode 100644 tests/test_subscriber_migration_rollback.py diff --git a/src/echo_certification_forge/subscriber.py b/src/echo_certification_forge/subscriber.py index 3c69fb4..6a6df86 100644 --- a/src/echo_certification_forge/subscriber.py +++ b/src/echo_certification_forge/subscriber.py @@ -353,7 +353,11 @@ def _migrate_legacy_platform_tables(self) -> None: connection.execute(f'ALTER TABLE "{table}" RENAME TO "{archive}"') connection.execute("COMMIT") except BaseException: - connection.execute("ROLLBACK") + # Roll back only an open transaction. If BEGIN IMMEDIATE itself failed (for example + # "database is locked" after the busy timeout), a bare ROLLBACK raised "cannot rollback - + # no transaction is active" and hid the real cause (Echo fleet #46912). + if connection.in_transaction: + connection.execute("ROLLBACK") raise finally: connection.close() @@ -3955,6 +3959,61 @@ def bind_run( outcome="allowed", ) + def _terminalize_unstarted_run( + self, + connection: sqlite3.Connection, + *, + run_id: str, + organization_id: str, + idempotency_key: str, + metered_at: str, + now_text: str, + actor: str, + reason: str, + ) -> None: + """Finish a QUEUED run whose dispatch gave up before execution started (fleet #49604). + + Same outcome as lease-expiry exhaustion in _recover_expired_dispatches: the run becomes + INFRASTRUCTURE_FAILURE / INFRA_FAILED, its BOUND reservation is released and the metered + run is compensated. The compensation uses the same 'dispatch-crash' key, so a run is + never compensated twice whichever path reaches it first. + """ + self._transition_run_with_connection( + connection, + run_id, + organization_id, + "QUEUED", + "INFRASTRUCTURE_FAILURE", + actor, + reason, + "p7.subscriber", + now_text, + ) + connection.execute( + """ + UPDATE runs SET run_outcome = 'INFRA_FAILED', updated_at = ? + WHERE run_id = ? AND tenant_id = ? + """, + (now_text, run_id, organization_id), + ) + connection.execute( + """ + UPDATE subscriber_run_reservations + SET state = 'RELEASED', updated_at = ? + WHERE run_id = ? AND organization_id = ? AND state = 'BOUND' + """, + (now_text, run_id, organization_id), + ) + self._compensate_crash_meter( + connection, + organization_id, + idempotency_key, + run_id, + metered_at, + key_prefix="dispatch-crash", + reason="dispatch_attempts_exhausted_before_execution_start", + ) + def _recover_expired_dispatches( self, connection: sqlite3.Connection, now: datetime ) -> int: @@ -4094,7 +4153,61 @@ def _recover_expired_dispatches( "meter_compensated": next_state == "FAILED", }, ) - return terminalized + len(rows) + # Runs orphaned before fleet #49604: dispatch FAILED, run still QUEUED, reservation BOUND. + # Once terminalized they no longer match, so this pass is idempotent. + orphans = connection.execute( + """ + SELECT d.run_id, d.organization_id, d.idempotency_key, d.attempts, d.last_error, + COALESCE( + ( + SELECT u.occurred_at + FROM subscriber_usage_events u + WHERE u.organization_id = d.organization_id + AND u.meter = 'certification_runs' + AND u.idempotency_key = d.idempotency_key + AND u.quantity > 0 + LIMIT 1 + ), + d.created_at + ) AS metered_at + FROM subscriber_run_dispatches AS d + JOIN subscriber_run_reservations AS r + ON r.run_id = d.run_id + AND r.organization_id = d.organization_id + JOIN runs + ON runs.run_id = d.run_id + AND runs.tenant_id = d.organization_id + WHERE d.state = 'FAILED' + AND r.state = 'BOUND' + AND runs.state = 'QUEUED' + """ + ).fetchall() + for row in orphans: + self._terminalize_unstarted_run( + connection, + run_id=str(row["run_id"]), + organization_id=str(row["organization_id"]), + idempotency_key=str(row["idempotency_key"]), + metered_at=str(row["metered_at"]), + now_text=now_text, + actor="certforge.dispatch-recovery", + reason="dispatch_failed_run_orphaned", + ) + self._append_audit( + connection, + organization_id=str(row["organization_id"]), + actor_ref="certforge.dispatch-recovery", + action="certification.dispatch_orphan_reconciled", + resource_type="certification", + resource_id=str(row["run_id"]), + outcome="allowed", + details={ + "attempts": int(row["attempts"]), + "last_error": row["last_error"], + "meter_compensated": True, + }, + ) + return terminalized + len(rows) + len(orphans) def claim_dispatch( self, @@ -4291,6 +4404,21 @@ def finish_dispatch( state = "FAILED" elif error is not None and int(row["attempts"]) < self.policy.max_dispatch_attempts: state = "PENDING" + elif error is not None and run_state == "QUEUED" and row["reservation_state"] == "BOUND": + # Last attempt failed before execution started (for example + # subscriber_worker_claim_denied). Marking only the dispatch FAILED left the run + # QUEUED forever and uncompensated (fleet #49604). + self._terminalize_unstarted_run( + connection, + run_id=dispatch.run_id, + organization_id=dispatch.organization_id, + idempotency_key=str(row["idempotency_key"]), + metered_at=str(row["metered_at"]), + now_text=now, + actor="certforge.dispatcher", + reason="dispatch_attempts_exhausted", + ) + state = "FAILED" else: state = "FAILED" connection.execute( diff --git a/tests/test_dispatch_orphaned_runs.py b/tests/test_dispatch_orphaned_runs.py new file mode 100644 index 0000000..a80578e --- /dev/null +++ b/tests/test_dispatch_orphaned_runs.py @@ -0,0 +1,117 @@ +"""A dispatch that gives up must not leave its run QUEUED forever (Echo fleet #49604). + +On FORGE (2026-11-03) seven runs from 2026-10-09..10-30 were QUEUED while their dispatch rows were +FAILED after 3 attempts with `subscriber_worker_claim_denied`, their reservations still BOUND and +execution never started. finish_dispatch() marked only the dispatch FAILED, so the runs never +reached a terminal state, the customer was never compensated for the metered run, and every +"is the queue idle?" check (for example the swarm re-certification) stayed blocked. + +The lease-expiry path already does the right thing (INFRASTRUCTURE_FAILURE, run_outcome INFRA_FAILED, +reservation RELEASED, meter compensated). These tests pin the same outcome for finish_dispatch, and +for runs already orphaned by the old code. +""" +from __future__ import annotations + +import sqlite3 +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from echo_certification_forge.models import RunState # noqa: E402 +from echo_certification_forge.subscriber import Permission # noqa: E402 +from test_p7_subscriber_governance import ( # noqa: E402 + _owner, + _policy, + _provision, + _stack, + _submit_local_run, +) + + +def _run(tmp_path, manifest, name, *, max_attempts): + policy = _policy().model_copy(update={"max_dispatch_attempts": max_attempts}) + store, governance, client = _stack(tmp_path, manifest, policy=policy) + org = _provision(governance, name) + source = tmp_path / f"{name}-source" + source.mkdir() + (source / "hello.py").write_text("print('dispatch')\n", encoding="utf-8") + run_id, _ = _submit_local_run(client, manifest, org.organization_id, org.bootstrap_api_key, source, + key=f"{name}-0001") + return store, governance, org, run_id + + +def _usage(governance, org): + return governance.usage_summary(_owner(governance, org.organization_id, org.bootstrap_api_key, + Permission.USAGE_READ)) + + +def test_worker_error_on_the_last_attempt_terminalizes_and_compensates(tmp_path, manifest) -> None: + store, governance, org, run_id = _run(tmp_path, manifest, "exhausted-claim-denied", max_attempts=1) + dispatch = governance.claim_dispatch("dispatcher-a", run_id=run_id, organization_id=org.organization_id) + assert dispatch is not None + + assert governance.finish_dispatch(dispatch, error="subscriber_worker_claim_denied") == "FAILED" + + run = store.get_run(run_id, org.organization_id) + assert run["state"] == RunState.INFRASTRUCTURE_FAILURE.value + assert run["run_outcome"] == "INFRA_FAILED" + assert governance.dispatch_status(run_id, org.organization_id)["state"] == "FAILED" + usage = _usage(governance, org) + assert usage["meters"]["certification_runs"] == 0 + assert usage["active_run_reservations"] == 0 + + +def test_worker_error_with_attempts_left_still_retries(tmp_path, manifest) -> None: + store, governance, org, run_id = _run(tmp_path, manifest, "retry-left", max_attempts=3) + dispatch = governance.claim_dispatch("dispatcher-a", run_id=run_id, organization_id=org.organization_id) + + assert governance.finish_dispatch(dispatch, error="subscriber_worker_claim_denied") == "PENDING" + + assert store.get_run(run_id, org.organization_id)["state"] == RunState.QUEUED.value + assert _usage(governance, org)["meters"]["certification_runs"] == 1 + + +def _orphan_the_old_way(governance, org, run_id) -> None: + """Reproduce what the old finish_dispatch left behind: dispatch FAILED, run QUEUED, reservation BOUND.""" + dispatch = governance.claim_dispatch("dispatcher-a", run_id=run_id, organization_id=org.organization_id) + assert dispatch is not None + with sqlite3.connect(governance.db_path) as connection: + connection.execute( + """ + UPDATE subscriber_run_dispatches + SET state = 'FAILED', attempts = 3, claim_token = NULL, lease_owner = NULL, + lease_expires_at = NULL, last_error = 'subscriber_worker_claim_denied' + WHERE run_id = ? AND organization_id = ? + """, + (run_id, org.organization_id), + ) + connection.commit() + + +def test_runs_orphaned_by_the_old_code_are_reconciled_once(tmp_path, manifest) -> None: + store, governance, org, run_id = _run(tmp_path, manifest, "orphan-reconcile", max_attempts=3) + _orphan_the_old_way(governance, org, run_id) + assert store.get_run(run_id, org.organization_id)["state"] == RunState.QUEUED.value + assert _usage(governance, org)["meters"]["certification_runs"] == 1 + + assert governance.claim_dispatch("dispatcher-b", organization_id=org.organization_id) is None + + run = store.get_run(run_id, org.organization_id) + assert run["state"] == RunState.INFRASTRUCTURE_FAILURE.value + assert run["run_outcome"] == "INFRA_FAILED" + usage = _usage(governance, org) + assert usage["meters"]["certification_runs"] == 0 + assert usage["active_run_reservations"] == 0 + # Idempotent: a later pass neither re-transitions nor compensates twice. + assert governance.claim_dispatch("dispatcher-c", organization_id=org.organization_id) is None + assert _usage(governance, org)["meters"]["certification_runs"] == 0 + + +def test_a_failed_dispatch_whose_run_already_finished_is_left_alone(tmp_path, manifest) -> None: + store, governance, org, run_id = _run(tmp_path, manifest, "orphan-terminal", max_attempts=1) + dispatch = governance.claim_dispatch("dispatcher-a", run_id=run_id, organization_id=org.organization_id) + governance.finish_dispatch(dispatch, error="subscriber_worker_claim_denied") + before = _usage(governance, org)["meters"]["certification_runs"] + assert governance.claim_dispatch("dispatcher-b", organization_id=org.organization_id) is None + assert _usage(governance, org)["meters"]["certification_runs"] == before == 0 diff --git a/tests/test_subscriber_migration_rollback.py b/tests/test_subscriber_migration_rollback.py new file mode 100644 index 0000000..1fa2d41 --- /dev/null +++ b/tests/test_subscriber_migration_rollback.py @@ -0,0 +1,76 @@ +"""Legacy-table migration must not mask its real failure (Echo fleet #46912). + +On 2026-10-19 echo-certforge-dispatcher flapped at start-up with +``sqlite3.OperationalError: cannot rollback - no transaction is active``. When ``BEGIN IMMEDIATE`` +itself fails (for example ``database is locked`` once the busy timeout expires while another +process holds the write lock), ``_migrate_legacy_platform_tables`` ran ``ROLLBACK`` with no +transaction open. That raised a second error which hid the real cause. +""" +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest + +from echo_certification_forge.subscriber import SubscriberGovernance + + +class _LockedOnBegin: + """A real sqlite3 connection whose ``BEGIN IMMEDIATE`` fails the way a held write lock does.""" + + def __init__(self, inner: sqlite3.Connection) -> None: + object.__setattr__(self, "_inner", inner) + + def execute(self, sql, *args): + if str(sql).strip().upper().startswith("BEGIN IMMEDIATE"): + raise sqlite3.OperationalError("database is locked") + return self._inner.execute(sql, *args) + + def __getattr__(self, name): + return getattr(self._inner, name) + + def __setattr__(self, name, value): + setattr(self._inner, name, value) + + +def _governance_without_init(db: Path) -> SubscriberGovernance: + gov = object.__new__(SubscriberGovernance) + gov.db_path = db + return gov + + +def test_a_failed_begin_surfaces_the_real_error_not_cannot_rollback(tmp_path, monkeypatch) -> None: + db = tmp_path / "certforge.sqlite3" + gov = _governance_without_init(db) + real_connect = SubscriberGovernance._connect + monkeypatch.setattr(SubscriberGovernance, "_connect", lambda self: _LockedOnBegin(real_connect(self))) + with pytest.raises(sqlite3.OperationalError) as exc: + gov._migrate_legacy_platform_tables() + assert "database is locked" in str(exc.value) + assert "cannot rollback" not in str(exc.value) + + +def test_an_error_inside_the_transaction_still_rolls_back(tmp_path) -> None: + db = tmp_path / "certforge.sqlite3" + with sqlite3.connect(db) as c: + # A legacy-looking subscriber_users table that matches neither layout: the migration must + # refuse it and leave the database exactly as it was. + c.execute("CREATE TABLE subscriber_users (user_id TEXT, unexpected TEXT)") + gov = _governance_without_init(db) + with pytest.raises(RuntimeError, match="incompatible subscriber schema"): + gov._migrate_legacy_platform_tables() + with sqlite3.connect(db) as c: + tables = {r[0] for r in c.execute("SELECT name FROM sqlite_master WHERE type='table'")} + in_tx = c.in_transaction + assert tables == {"subscriber_users"} and not in_tx + + +def test_a_needed_rename_is_committed(tmp_path) -> None: + db = tmp_path / "certforge.sqlite3" + with sqlite3.connect(db) as c: + c.execute("CREATE TABLE subscriber_users (user_id TEXT, email TEXT, created_at TEXT)") + _governance_without_init(db)._migrate_legacy_platform_tables() + with sqlite3.connect(db) as c: + tables = {r[0] for r in c.execute("SELECT name FROM sqlite_master WHERE type='table'")} + assert tables == {"legacy_platform_subscriber_users"} From 3dcd40bd1db8694b5843e034c0333f1d1e9b0924 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Tue, 10 Nov 2026 20:23:03 -0600 Subject: [PATCH 18/39] test(subscriber): prove two concurrent orphan recoverers compensate once --- tests/test_dispatch_orphan_concurrency.py | 43 +++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 tests/test_dispatch_orphan_concurrency.py diff --git a/tests/test_dispatch_orphan_concurrency.py b/tests/test_dispatch_orphan_concurrency.py new file mode 100644 index 0000000..13fa345 --- /dev/null +++ b/tests/test_dispatch_orphan_concurrency.py @@ -0,0 +1,43 @@ +"""A concurrent orphan sweep must compensate a metered run exactly once. + +This regression supplements the sequential orphan recovery coverage from PR #49. +It uses an isolated SQLite fixture; no production service or Telegram/OAuth integration. +""" +from concurrent.futures import ThreadPoolExecutor +from threading import Barrier +import sqlite3 + +from echo_certification_forge.models import RunState +from test_dispatch_orphaned_runs import _run, _orphan_the_old_way, _usage + + +def test_two_simultaneous_recoverers_compensate_once(tmp_path, manifest): + store, governance, org, run_id = _run( + tmp_path, manifest, "probe-concurrent-recovery", max_attempts=3 + ) + _orphan_the_old_way(governance, org, run_id) + barrier = Barrier(3) + + def recover(label): + barrier.wait(timeout=8) + return governance.claim_dispatch(label, organization_id=org.organization_id) + + with ThreadPoolExecutor(max_workers=2) as pool: + one = pool.submit(recover, "probe-recoverer-a") + two = pool.submit(recover, "probe-recoverer-b") + barrier.wait(timeout=8) + results = [one.result(timeout=20), two.result(timeout=20)] + + assert results == [None, None] + assert store.get_run(run_id, org.organization_id)["state"] == RunState.INFRASTRUCTURE_FAILURE.value + usage = _usage(governance, org) + assert usage["meters"]["certification_runs"] == 0 + assert usage["active_run_reservations"] == 0 + with sqlite3.connect(governance.db_path) as connection: + compensations = connection.execute( + "SELECT count(*) FROM subscriber_usage_events " + "WHERE organization_id = ? AND meter = 'certification_runs' " + "AND quantity = -1 AND idempotency_key LIKE 'dispatch-crash:%'", + (org.organization_id,), + ).fetchone()[0] + assert compensations == 1, f"expected one compensation, got {compensations}" From 6cb0c9b0dfca33357cf7f8e7c6caa6671a383b78 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Tue, 10 Nov 2026 21:17:24 -0600 Subject: [PATCH 19/39] test(subscriber): fault-inject dispatch compensation and assert transactional rollback --- tests/test_dispatch_compensation_atomicity.py | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 tests/test_dispatch_compensation_atomicity.py diff --git a/tests/test_dispatch_compensation_atomicity.py b/tests/test_dispatch_compensation_atomicity.py new file mode 100644 index 0000000..2ab1b25 --- /dev/null +++ b/tests/test_dispatch_compensation_atomicity.py @@ -0,0 +1,91 @@ +"""Failure-injection acceptance for dispatch compensation transaction atomicity. + +Test-only follow-up to combined recovery PR #49 and concurrency PR #53. +All writes are confined to pytest's temporary SQLite fixture. +""" +from __future__ import annotations + +import sqlite3 +from unittest.mock import patch + +import pytest + +from echo_certification_forge.models import RunState +from test_dispatch_orphaned_runs import _orphan_the_old_way, _run, _usage + + +def _database_snapshot(governance, org, run_id): + with sqlite3.connect(governance.db_path) as conn: + reservation = conn.execute( + "SELECT state FROM subscriber_run_reservations " + "WHERE run_id = ? AND organization_id = ?", + (run_id, org.organization_id), + ).fetchone() + dispatch = conn.execute( + "SELECT state FROM subscriber_run_dispatches " + "WHERE run_id = ? AND organization_id = ?", + (run_id, org.organization_id), + ).fetchone() + compensations = conn.execute( + "SELECT COUNT(*) FROM subscriber_usage_events " + "WHERE organization_id = ? AND meter = 'certification_runs' " + "AND quantity = -1 AND idempotency_key LIKE 'dispatch-crash:%'", + (org.organization_id,), + ).fetchone()[0] + assert reservation is not None and dispatch is not None + return reservation[0], dispatch[0], compensations + + +def _compensation_unavailable(*_args, **_kwargs): + raise RuntimeError("injected compensation failure") + + +def test_orphan_recovery_compensation_exception_rolls_back(tmp_path, manifest): + store, governance, org, run_id = _run( + tmp_path, manifest, "orphan-compensation-rollback", max_attempts=3 + ) + _orphan_the_old_way(governance, org, run_id) + + with patch.object( + type(governance), "_compensate_crash_meter", + side_effect=_compensation_unavailable, + ): + with pytest.raises(RuntimeError, match="injected compensation failure"): + governance.claim_dispatch("failure-injection", organization_id=org.organization_id) + + # A crash between state transition and metering must not leave a half-finished run. + assert store.get_run(run_id, org.organization_id)["state"] == RunState.QUEUED.value + assert _database_snapshot(governance, org, run_id) == ("BOUND", "FAILED", 0) + assert _usage(governance, org)["meters"]["certification_runs"] == 1 + + # Retrying after restoring the real compensator must complete the entire transaction once. + assert governance.claim_dispatch("recovery-retry", organization_id=org.organization_id) is None + assert store.get_run(run_id, org.organization_id)["state"] == RunState.INFRASTRUCTURE_FAILURE.value + assert _database_snapshot(governance, org, run_id) == ("RELEASED", "FAILED", 1) + assert _usage(governance, org)["meters"]["certification_runs"] == 0 + assert governance.claim_dispatch("recovery-repeat", organization_id=org.organization_id) is None + assert _database_snapshot(governance, org, run_id)[2] == 1 + + +def test_last_attempt_compensation_exception_preserves_claim(tmp_path, manifest): + store, governance, org, run_id = _run( + tmp_path, manifest, "finish-compensation-rollback", max_attempts=1 + ) + dispatch = governance.claim_dispatch("failure-injection", organization_id=org.organization_id) + assert dispatch is not None + + with patch.object( + type(governance), "_compensate_crash_meter", + side_effect=_compensation_unavailable, + ): + with pytest.raises(RuntimeError, match="injected compensation failure"): + governance.finish_dispatch(dispatch, error="subscriber_worker_claim_denied") + + assert store.get_run(run_id, org.organization_id)["state"] == RunState.QUEUED.value + assert _database_snapshot(governance, org, run_id) == ("BOUND", "CLAIMED", 0) + assert _usage(governance, org)["meters"]["certification_runs"] == 1 + + assert governance.finish_dispatch(dispatch, error="subscriber_worker_claim_denied") == "FAILED" + assert store.get_run(run_id, org.organization_id)["state"] == RunState.INFRASTRUCTURE_FAILURE.value + assert _database_snapshot(governance, org, run_id) == ("RELEASED", "FAILED", 1) + assert _usage(governance, org)["meters"]["certification_runs"] == 0 From 6aacebfa298265da42e3a99f43010e37380250d0 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Tue, 10 Nov 2026 22:21:02 -0600 Subject: [PATCH 20/39] fix(subscriber): redact untrusted dispatch errors from immutable orphan audit --- src/echo_certification_forge/subscriber.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/echo_certification_forge/subscriber.py b/src/echo_certification_forge/subscriber.py index 6a6df86..afa263a 100644 --- a/src/echo_certification_forge/subscriber.py +++ b/src/echo_certification_forge/subscriber.py @@ -4203,7 +4203,18 @@ def _recover_expired_dispatches( outcome="allowed", details={ "attempts": int(row["attempts"]), - "last_error": row["last_error"], + # Dispatch error text may originate from a worker or remote failure. + # Audit entries are immutable: retain only known machine reason codes, + # never an unbounded/raw exception or message that might include secrets. + "last_error": ( + row["last_error"] + if row["last_error"] in ( + "subscriber_worker_claim_denied", + "dispatch_attempts_exhausted", + "dispatch_lease_expired", + ) + else "redacted" if row["last_error"] else None + ), "meter_compensated": True, }, ) From 24277990381d376ab427e42851c62a03a3110a34 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Tue, 10 Nov 2026 22:21:28 -0600 Subject: [PATCH 21/39] test(subscriber): prove orphan audits cannot persist untrusted worker error text --- tests/test_dispatch_orphan_audit_redaction.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 tests/test_dispatch_orphan_audit_redaction.py diff --git a/tests/test_dispatch_orphan_audit_redaction.py b/tests/test_dispatch_orphan_audit_redaction.py new file mode 100644 index 0000000..db925e9 --- /dev/null +++ b/tests/test_dispatch_orphan_audit_redaction.py @@ -0,0 +1,58 @@ +"""The immutable orphan-reconciliation audit must not persist untrusted error text. + +All state changes use pytest's private temporary SQLite database. +This test covers the recovery path only, without a production dispatcher. +""" +from __future__ import annotations + +import json +import sqlite3 + +import pytest + +from echo_certification_forge.models import RunState +from test_dispatch_orphaned_runs import _orphan_the_old_way, _run, _usage + + +@pytest.mark.parametrize( + ("incoming_error", "audited_error"), + [ + ("subscriber_worker_claim_denied", "subscriber_worker_claim_denied"), + ("Authorization: Bearer synthetic-do-not-store-000", "redacted"), + (None, None), + ], +) +def test_orphan_recovery_audit_redacts_untrusted_last_error( + tmp_path, manifest, incoming_error, audited_error +): + store, governance, org, run_id = _run( + tmp_path, manifest, "audit-redaction", max_attempts=3 + ) + _orphan_the_old_way(governance, org, run_id) + with sqlite3.connect(governance.db_path) as connection: + connection.execute( + "UPDATE subscriber_run_dispatches SET last_error = ? " + "WHERE run_id = ? AND organization_id = ?", + (incoming_error, run_id, org.organization_id), + ) + connection.commit() + + assert governance.claim_dispatch( + "audit-recovery", organization_id=org.organization_id + ) is None + assert store.get_run(run_id, org.organization_id)["state"] == ( + RunState.INFRASTRUCTURE_FAILURE.value + ) + assert _usage(governance, org)["meters"]["certification_runs"] == 0 + + with sqlite3.connect(governance.db_path) as connection: + audit_rows = connection.execute( + "SELECT details_json FROM subscriber_audit_events " + "WHERE organization_id = ? AND resource_id = ? " + "AND action = 'certification.dispatch_orphan_reconciled'", + (org.organization_id, run_id), + ).fetchall() + assert len(audit_rows) == 1 + recorded = audit_rows[0][0] + assert json.loads(recorded)["last_error"] == audited_error + assert "synthetic-do-not-store-000" not in recorded From 9056dcae36a110e23e6b9721cc3524a65312620d Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Wed, 11 Nov 2026 02:19:15 -0600 Subject: [PATCH 22/39] test(subscriber): verify cross-tenant orphan batch reconciliation idempotency --- tests/test_dispatch_orphan_batch_recovery.py | 67 ++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 tests/test_dispatch_orphan_batch_recovery.py diff --git a/tests/test_dispatch_orphan_batch_recovery.py b/tests/test_dispatch_orphan_batch_recovery.py new file mode 100644 index 0000000..7706f21 --- /dev/null +++ b/tests/test_dispatch_orphan_batch_recovery.py @@ -0,0 +1,67 @@ +"""Reconcile multiple legacy dispatch orphans across tenants exactly once. + +All runs, principals, and accounting events use disposable local fixtures. +""" +from __future__ import annotations + +import sqlite3 + +from echo_certification_forge.models import RunState +from test_dispatch_orphaned_runs import _usage +from test_p7_subscriber_governance import _policy, _provision, _stack, _submit_local_run + + +def test_multiple_tenant_orphans_reconciled_once(tmp_path, manifest): + policy = _policy().model_copy(update={"max_dispatch_attempts": 3}) + store, governance, client = _stack(tmp_path, manifest, policy=policy) + pairs = [] + for index in range(5): + org = _provision(governance, f"batch-orphan-org-{index}") + source = tmp_path / f"orphan-source-{index}" + source.mkdir() + (source / "hello.py").write_text("print('ok')\n", encoding="utf-8") + run_id, _ = _submit_local_run( + client, manifest, org.organization_id, org.bootstrap_api_key, + source, key=f"batch-orphan-{index:04d}", + ) + pairs.append((org, run_id)) + + # Recreate the legacy state without invoking claim_dispatch while staging; + # that call would run the new reconciler before the whole batch exists. + with sqlite3.connect(governance.db_path) as db: + for org, run_id in pairs: + db.execute( + "UPDATE subscriber_run_dispatches " + "SET state='FAILED', attempts=3, claim_token=NULL, " + "lease_owner=NULL, lease_expires_at=NULL, " + "last_error='subscriber_worker_claim_denied' " + "WHERE run_id=? AND organization_id=?", + (run_id, org.organization_id), + ) + db.commit() + + assert all(_usage(governance, org)["meters"]["certification_runs"] == 1 for org, _ in pairs) + assert governance.claim_dispatch("batch-recovery") is None + for org, run_id in pairs: + assert store.get_run(run_id, org.organization_id)["state"] == RunState.INFRASTRUCTURE_FAILURE.value + usage = _usage(governance, org) + assert usage["meters"]["certification_runs"] == 0 + assert usage["active_run_reservations"] == 0 + + def readback(): + with sqlite3.connect(governance.db_path) as db: + audits = db.execute( + "SELECT COUNT(*) FROM subscriber_audit_events " + "WHERE action='certification.dispatch_orphan_reconciled'" + ).fetchone()[0] + refunds = dict(db.execute( + "SELECT organization_id, COUNT(*) FROM subscriber_usage_events " + "WHERE meter='certification_runs' AND quantity=-1 " + "GROUP BY organization_id" + ).fetchall()) + return audits, refunds + + expected_refunds = {org.organization_id: 1 for org, _ in pairs} + assert readback() == (len(pairs), expected_refunds) + assert governance.claim_dispatch("batch-recovery-repeat") is None + assert readback() == (len(pairs), expected_refunds) From 1fa77c4a5875b8a64f0390eab64de5bd650229f8 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Wed, 11 Nov 2026 20:31:36 -0600 Subject: [PATCH 23/39] fix: bound persisted dispatch worker errors and legacy status responses --- src/echo_certification_forge/subscriber.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/echo_certification_forge/subscriber.py b/src/echo_certification_forge/subscriber.py index afa263a..f2de215 100644 --- a/src/echo_certification_forge/subscriber.py +++ b/src/echo_certification_forge/subscriber.py @@ -31,6 +31,20 @@ _ZERO_HASH = "0" * 64 _TERMINAL_RUN_STATES = ("COMPLETED", "CANCELLED", "INFRASTRUCTURE_FAILURE") +# Worker exception text is untrusted; retain only documented machine reasons. +# Also redact API responses from legacy rows with previously stored raw errors. +def _bounded_dispatch_error(error: str | None) -> str | None: + if error is None: + return None + if error in ( + "subscriber_worker_claim_denied", + "dispatch_attempts_exhausted", + "dispatch_lease_expired", + ): + return error + return "redacted" + + class OrganizationStatus(StrEnum): TRIALING = "TRIALING" @@ -4441,7 +4455,7 @@ def finish_dispatch( """, ( state, - error, + _bounded_dispatch_error(error), now, dispatch.run_id, dispatch.organization_id, @@ -4465,7 +4479,9 @@ def dispatch_status(self, run_id: str, organization_id: str) -> dict[str, Any]: ).fetchone() if row is None: raise SubscriberError(404, "dispatch_not_found") - return dict(row) + result = dict(row) + result["last_error"] = _bounded_dispatch_error(result["last_error"]) + return result def _resolve_worker_governance( self, From 86a11e0b2ad0a72c7b9ecb0dde008b83240ffbb0 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Wed, 11 Nov 2026 20:31:38 -0600 Subject: [PATCH 24/39] test: preserve fail-closed dispatch error retention across fresh and legacy rows --- tests/test_dispatch_error_retention.py | 60 ++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 tests/test_dispatch_error_retention.py diff --git a/tests/test_dispatch_error_retention.py b/tests/test_dispatch_error_retention.py new file mode 100644 index 0000000..00d0883 --- /dev/null +++ b/tests/test_dispatch_error_retention.py @@ -0,0 +1,60 @@ +"""Untrusted dispatch worker exceptions must not persist or leak on readback.""" +from __future__ import annotations + +import sqlite3 + +import pytest + +from echo_certification_forge.models import RunState +from test_dispatch_orphaned_runs import _orphan_the_old_way, _run, _usage + + +@pytest.mark.parametrize( + ("incoming", "expected"), + [ + ("subscriber_worker_claim_denied", "subscriber_worker_claim_denied"), + ("Authorization: Bearer synthetic-do-not-retain-000", "redacted"), + ("arbitrary remote stack trace", "redacted"), + ], +) +def test_new_worker_error_storage_and_status_redacted( + tmp_path, manifest, incoming, expected +): + store, gov, org, run_id = _run(tmp_path, manifest, "error-retention", max_attempts=1) + dispatch = gov.claim_dispatch( + "dispatcher", run_id=run_id, organization_id=org.organization_id + ) + assert dispatch is not None + assert gov.finish_dispatch(dispatch, error=incoming) == "FAILED" + assert store.get_run(run_id, org.organization_id)["state"] == ( + RunState.INFRASTRUCTURE_FAILURE.value + ) + usage = _usage(gov, org) + assert usage["meters"]["certification_runs"] == 0 + assert usage["active_run_reservations"] == 0 + assert gov.dispatch_status(run_id, org.organization_id)["last_error"] == expected + with sqlite3.connect(gov.db_path) as conn: + stored = conn.execute( + "SELECT last_error FROM subscriber_run_dispatches WHERE run_id = ?", (run_id,) + ).fetchone()[0] + assert stored == expected + assert "synthetic-do-not-retain-000" not in (stored or "") + + +def test_legacy_error_read_redacted_without_mutating_db(tmp_path, manifest): + store, gov, org, run_id = _run(tmp_path, manifest, "legacy-error", max_attempts=3) + _orphan_the_old_way(gov, org, run_id) + raw = "Authorization: Bearer synthetic-legacy-no-display-000" + with sqlite3.connect(gov.db_path) as conn: + conn.execute( + "UPDATE subscriber_run_dispatches SET last_error = ? WHERE run_id = ?", + (raw, run_id), + ) + conn.commit() + assert gov.dispatch_status(run_id, org.organization_id)["last_error"] == "redacted" + with sqlite3.connect(gov.db_path) as conn: + assert conn.execute( + "SELECT last_error FROM subscriber_run_dispatches WHERE run_id = ?", + (run_id,), + ).fetchone()[0] == raw + assert store.get_run(run_id, org.organization_id)["state"] == RunState.QUEUED.value From 941cebb97f8ea008bba8c0e3d18b6ffc422d801e Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Wed, 11 Nov 2026 22:31:39 -0600 Subject: [PATCH 25/39] fix(subscriber): bound persisted worker failure reasons across audit sinks --- src/echo_certification_forge/subscriber.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/echo_certification_forge/subscriber.py b/src/echo_certification_forge/subscriber.py index f2de215..1c68de2 100644 --- a/src/echo_certification_forge/subscriber.py +++ b/src/echo_certification_forge/subscriber.py @@ -30,6 +30,14 @@ _ZERO_HASH = "0" * 64 _TERMINAL_RUN_STATES = ("COMPLETED", "CANCELLED", "INFRASTRUCTURE_FAILURE") +# Known machine codes observed at worker failure call sites. Unknown identifier-form +# strings must not enter durable state, audit chains, or tenant-visible responses. +_SAFE_WORKER_FAILURE_REASONS = frozenset({ + "oci_journey_requires_sandbox", + "target_reconciliation_failed", + "windows_package_finalization_failed", + "worker_run_claim_inactive", +}) # Worker exception text is untrusted; retain only documented machine reasons. # Also redact API responses from legacy rows with previously stored raw errors. @@ -5063,6 +5071,9 @@ def assert_worker_claim_active(self, claim: WorkerRunClaim) -> None: def fail_worker_execution(self, claim: WorkerRunClaim, *, reason: str) -> None: require_identifier(reason, "reason") + # Even identifier-shaped inputs may contain sensitive untrusted data. + # Use one sanitized reason for dispatch state, run history and immutable audit. + reason = reason if reason in _SAFE_WORKER_FAILURE_REASONS else "redacted" now_value = self._now() now = to_utc_iso(now_value) with self._connection(immediate=True) as connection: From 5b639aec0db099bee3af77ccf40e940b024a7cea Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Wed, 11 Nov 2026 22:31:41 -0600 Subject: [PATCH 26/39] test(subscriber): prevent identifier-form worker reasons entering durable evidence --- tests/test_worker_failure_reason_redaction.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 tests/test_worker_failure_reason_redaction.py diff --git a/tests/test_worker_failure_reason_redaction.py b/tests/test_worker_failure_reason_redaction.py new file mode 100644 index 0000000..22f11fc --- /dev/null +++ b/tests/test_worker_failure_reason_redaction.py @@ -0,0 +1,69 @@ +"""The worker failure path must redact untrusted identifier-form reasons at every durable sink.""" +from __future__ import annotations + +import json +import sqlite3 + +import pytest + +from echo_certification_forge.models import RunState +from echo_certification_forge.signing import Ed25519VerdictSigner +from test_p7_subscriber_governance import _stack, _provision, _submit_local_run + + +@pytest.mark.parametrize( + ("incoming", "expected"), + [ + ("synthetic_private_token_12345", "redacted"), + ("oci_journey_requires_sandbox", "oci_journey_requires_sandbox"), + ], +) +def test_worker_failure_reason_is_bounded_across_state_and_immutable_audit( + tmp_path, manifest, incoming, expected +): + store, governance, client = _stack(tmp_path, manifest) + org = _provision(governance, "worker-reason") + source = tmp_path / "source" + source.mkdir() + (source / "run.py").write_text("print('safe')\n", encoding="utf-8") + run_id, target = _submit_local_run( + client, manifest, org.organization_id, org.bootstrap_api_key, + source, key="worker-reason-001", + ) + signer = Ed25519VerdictSigner.generate() + claim = governance.claim_worker_run( + run_id=run_id, + organization_id=org.organization_id, + policy_version=manifest.manifest_id, + target_type="local", + target_reference=target.canonical_ref, + worker_id=None, + worker_attestation_sha256=None, + execution_location="local", + signing_authority="platform", + signing_key_id=signer.key_id, + ) + governance.fail_worker_execution(claim, reason=incoming) + assert store.get_run(run_id, org.organization_id)["state"] == ( + RunState.INFRASTRUCTURE_FAILURE.value + ) + assert governance.dispatch_status(run_id, org.organization_id)["last_error"] == expected + with sqlite3.connect(governance.db_path) as conn: + dispatch = conn.execute( + "SELECT last_error FROM subscriber_run_dispatches WHERE run_id = ?", + (run_id,), + ).fetchone()[0] + history = conn.execute( + "SELECT reason FROM state_events WHERE run_id = ? ORDER BY event_id DESC LIMIT 1", + (run_id,), + ).fetchone()[0] + audit = conn.execute( + """SELECT details_json FROM subscriber_audit_events + WHERE organization_id = ? AND resource_id = ? + AND action = 'certification.worker_fail' + ORDER BY event_id DESC LIMIT 1""", + (org.organization_id, run_id), + ).fetchone()[0] + assert dispatch == history == json.loads(audit)["reason"] == expected + if expected == "redacted": + assert incoming not in (dispatch + history + audit) From 76b85df4185addbba0f8b700750c38c019e083ec Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Wed, 11 Nov 2026 22:35:23 -0600 Subject: [PATCH 27/39] fix(subscriber): keep safe worker machine codes readable through dispatch status --- src/echo_certification_forge/subscriber.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/echo_certification_forge/subscriber.py b/src/echo_certification_forge/subscriber.py index 1c68de2..e358df2 100644 --- a/src/echo_certification_forge/subscriber.py +++ b/src/echo_certification_forge/subscriber.py @@ -44,7 +44,7 @@ def _bounded_dispatch_error(error: str | None) -> str | None: if error is None: return None - if error in ( + if error in _SAFE_WORKER_FAILURE_REASONS or error in ( "subscriber_worker_claim_denied", "dispatch_attempts_exhausted", "dispatch_lease_expired", From b96325ef3a06afed0deb68935e8564f7ef0b6f2b Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Thu, 12 Nov 2026 04:25:07 -0600 Subject: [PATCH 28/39] fix(deploy): enforce bounded git acquisition before release stages --- deploy/deploy_forge.sh | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/deploy/deploy_forge.sh b/deploy/deploy_forge.sh index db9c2c2..40b31c1 100644 --- a/deploy/deploy_forge.sh +++ b/deploy/deploy_forge.sh @@ -55,9 +55,34 @@ service_owns_port() { ss -H -ltnp "sport = :$port" | grep -q "pid=$service_pid," } +[[ "$GIT_ACQUISITION_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] || { + echo "!! ECHO_CERTFORGE_GIT_ACQUISITION_TIMEOUT_SECONDS must be an integer" + exit 1 +} +if (( GIT_ACQUISITION_TIMEOUT_SECONDS < 30 || GIT_ACQUISITION_TIMEOUT_SECONDS > 1800 )); then + echo "!! ECHO_CERTFORGE_GIT_ACQUISITION_TIMEOUT_SECONDS must be between 30 and 1800" + exit 1 +fi +command -v timeout >/dev/null 2>&1 || { + echo "!! GNU timeout is required for bounded source acquisition" + exit 1 +} cd "$SOURCE_REPO" echo "== [1/9] fetch immutable source ($BRANCH) ==" -git "${GITC[@]}" fetch --quiet origin "$BRANCH" +# timeout controls the git process group, including credential and transport helpers. +# An unresponsive fetch cannot enter the release build or promotion stages. +if timeout --signal=TERM --kill-after=5s "${GIT_ACQUISITION_TIMEOUT_SECONDS}s" \ + git "${GITC[@]}" fetch --quiet origin "$BRANCH"; then + : +else + fetch_status=$? + if [ "$fetch_status" -eq 124 ] || [ "$fetch_status" -eq 137 ]; then + echo "!! source fetch exceeded the configured acquisition deadline" + else + echo "!! source fetch failed (status $fetch_status)" + fi + exit "$fetch_status" +fi NEW_SHA="$(git rev-parse "origin/$BRANCH^{commit}")" if [ -n "$EXPECTED_COMMIT_SHA" ] && [ "$NEW_SHA" != "$EXPECTED_COMMIT_SHA" ]; then echo "!! fetched commit does not match the hosted-CI-approved commit" @@ -107,14 +132,6 @@ case "$ADAPTER_MODE" in required|pending) ;; *) echo "!! CERTFORGE_ADAPTER_MODE must be required or pending"; exit 1 ;; esac -[[ "$GIT_ACQUISITION_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] || { - echo "!! ECHO_CERTFORGE_GIT_ACQUISITION_TIMEOUT_SECONDS must be an integer" - exit 1 -} -if (( GIT_ACQUISITION_TIMEOUT_SECONDS < 30 || GIT_ACQUISITION_TIMEOUT_SECONDS > 1800 )); then - echo "!! ECHO_CERTFORGE_GIT_ACQUISITION_TIMEOUT_SECONDS must be between 30 and 1800" - exit 1 -fi ADAPTER_RESPONSE="" ADAPTER_POLICY="" ADAPTER_REGISTRY="" From 3648116c15d67616f0296a376696fc114b401708 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Thu, 12 Nov 2026 04:27:40 -0600 Subject: [PATCH 29/39] test(deploy): fail closed on stalled and invalid source fetch --- tests/test_deploy_git_fetch_deadline.py | 62 +++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 tests/test_deploy_git_fetch_deadline.py diff --git a/tests/test_deploy_git_fetch_deadline.py b/tests/test_deploy_git_fetch_deadline.py new file mode 100644 index 0000000..7c5a6f5 --- /dev/null +++ b/tests/test_deploy_git_fetch_deadline.py @@ -0,0 +1,62 @@ +"""Isolated, no-network regression checks for deploy source-fetch deadlines.""" +from pathlib import Path +import os +import subprocess + +import pytest + +SCRIPT = (Path(__file__).resolve().parents[1] / "deploy" / "deploy_forge.sh").read_text() +START = '[[ "$GIT_ACQUISITION_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]]' +END = 'NEW_SHA="$(git rev-parse' +FETCH_BLOCK = SCRIPT[SCRIPT.index(START) : SCRIPT.index(END)] + + +def exercise(tmp_path: Path, seconds: str, code: int) -> subprocess.CompletedProcess[str]: + mock = 'timeout() { printf "%s\n" "$@" > "$FETCH_ARGS_LOG"; return ' + str(code) + '; }' + source = ( + 'set -euo pipefail\n' + f'GIT_ACQUISITION_TIMEOUT_SECONDS={seconds}\n' + 'SOURCE_REPO="$PWD"\n' + 'BRANCH=main\n' + 'GITC=()\n' + + mock + '\n' + FETCH_BLOCK + ) + return subprocess.run( + ["bash", "-c", source], + cwd=tmp_path, + env={**os.environ, "FETCH_ARGS_LOG": str(tmp_path / "timeout-args.txt")}, + capture_output=True, + text=True, + check=False, + timeout=5, + ) + + +def test_bounded_fetch_happens_after_validation_before_sha_resolution(tmp_path: Path) -> None: + assert SCRIPT.index(START) < SCRIPT.index('fetch --quiet origin "$BRANCH"') < SCRIPT.index(END) + result = exercise(tmp_path, "30", 0) + assert result.returncode == 0, result.stderr + assert (tmp_path / "timeout-args.txt").read_text().splitlines() == [ + "--signal=TERM", "--kill-after=5s", "30s", "git", + "fetch", "--quiet", "origin", "main", + ] + + +@pytest.mark.parametrize("status", [124, 137]) +def test_deadline_failure_is_fail_closed(tmp_path: Path, status: int) -> None: + result = exercise(tmp_path, "30", status) + assert result.returncode == status + assert "exceeded the configured acquisition deadline" in result.stdout + + +def test_other_fetch_failure_is_fail_closed(tmp_path: Path) -> None: + result = exercise(tmp_path, "30", 23) + assert result.returncode == 23 + assert "source fetch failed (status 23)" in result.stdout + + +@pytest.mark.parametrize("seconds", ["invalid", "0", "29", "1801"]) +def test_invalid_deadlines_reject_before_fetch(tmp_path: Path, seconds: str) -> None: + result = exercise(tmp_path, seconds, 0) + assert result.returncode != 0 + assert not (tmp_path / "timeout-args.txt").exists() From 272bea3674f6cdecd4ca0423b30b745cbf755cab Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Thu, 12 Nov 2026 06:40:55 -0600 Subject: [PATCH 30/39] fix(subscriber): fail closed on orphaned dispatch with execution or ownership evidence --- src/echo_certification_forge/subscriber.py | 14 ++++++ tests/test_dispatch_orphaned_runs.py | 51 ++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/src/echo_certification_forge/subscriber.py b/src/echo_certification_forge/subscriber.py index e358df2..66ff83d 100644 --- a/src/echo_certification_forge/subscriber.py +++ b/src/echo_certification_forge/subscriber.py @@ -4202,6 +4202,20 @@ def _recover_expired_dispatches( WHERE d.state = 'FAILED' AND r.state = 'BOUND' AND runs.state = 'QUEUED' + -- Legacy recovery may only compensate work with no execution or + -- ownership evidence. Inconsistent state is held for human review. + AND r.execution_started_at IS NULL + AND r.worker_id IS NULL + AND r.worker_attestation_sha256 IS NULL + AND r.execution_location IS NULL + AND r.signing_authority IS NULL + AND r.signing_key_id IS NULL + AND r.claim_token IS NULL + AND r.lease_expires_at IS NULL + AND r.last_heartbeat_at IS NULL + AND d.claim_token IS NULL + AND d.lease_owner IS NULL + AND d.lease_expires_at IS NULL """ ).fetchall() for row in orphans: diff --git a/tests/test_dispatch_orphaned_runs.py b/tests/test_dispatch_orphaned_runs.py index a80578e..7286c2b 100644 --- a/tests/test_dispatch_orphaned_runs.py +++ b/tests/test_dispatch_orphaned_runs.py @@ -115,3 +115,54 @@ def test_a_failed_dispatch_whose_run_already_finished_is_left_alone(tmp_path, ma before = _usage(governance, org)["meters"]["certification_runs"] assert governance.claim_dispatch("dispatcher-b", organization_id=org.organization_id) is None assert _usage(governance, org)["meters"]["certification_runs"] == before == 0 + + +# Synthetic contradictory records must not lose live execution ownership, +# immutable evidence or metered reservations during historical recovery. +import pytest + + +@pytest.mark.parametrize( + ("table", "column", "value"), + [ + ("subscriber_run_reservations", "execution_started_at", "2026-11-12T12:00:00Z"), + ("subscriber_run_reservations", "worker_id", "synthetic-test-worker"), + ("subscriber_run_reservations", "worker_attestation_sha256", "1" * 64), + ("subscriber_run_reservations", "execution_location", "synthetic-test-location"), + ("subscriber_run_reservations", "signing_authority", "synthetic-test-authority"), + ("subscriber_run_reservations", "signing_key_id", "synthetic-test-key"), + ("subscriber_run_reservations", "claim_token", "synthetic-test-claim"), + ("subscriber_run_reservations", "lease_expires_at", "2026-11-12T13:00:00Z"), + ("subscriber_run_reservations", "last_heartbeat_at", "2026-11-12T12:00:00Z"), + ("subscriber_run_dispatches", "claim_token", "synthetic-test-dispatch-claim"), + ("subscriber_run_dispatches", "lease_owner", "synthetic-test-dispatch-owner"), + ("subscriber_run_dispatches", "lease_expires_at", "2026-11-12T13:00:00Z"), + ], +) +def test_orphan_recovery_preserves_conflicting_ownership( + tmp_path, manifest, table, column, value +) -> None: + store, governance, org, run_id = _run( + tmp_path, manifest, "contradictory-owner", max_attempts=3 + ) + _orphan_the_old_way(governance, org, run_id) + # Only disposable test SQLite, never production. Parameters are test constants. + with sqlite3.connect(governance.db_path) as connection: + connection.execute( + f"UPDATE {table} SET {column} = ? WHERE run_id = ? AND organization_id = ?", + (value, run_id, org.organization_id), + ) + assert governance.claim_dispatch( + "dispatcher-reconciler", organization_id=org.organization_id + ) is None + assert store.get_run(run_id, org.organization_id)["state"] == RunState.QUEUED.value + usage = _usage(governance, org) + assert usage["meters"]["certification_runs"] == 1 + assert usage["active_run_reservations"] == 1 + with sqlite3.connect(governance.db_path) as connection: + state = connection.execute( + "SELECT state FROM subscriber_run_reservations " + "WHERE run_id = ? AND organization_id = ?", + (run_id, org.organization_id), + ).fetchone()[0] + assert state == "BOUND" From 0a9bbc041bb3ded5f1bde5aba1e41591e755ef88 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Thu, 12 Nov 2026 08:28:50 -0600 Subject: [PATCH 31/39] fix(subscriber): exclude consumed reservations from orphan compensation --- src/echo_certification_forge/subscriber.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/echo_certification_forge/subscriber.py b/src/echo_certification_forge/subscriber.py index 66ff83d..f3b2196 100644 --- a/src/echo_certification_forge/subscriber.py +++ b/src/echo_certification_forge/subscriber.py @@ -4205,6 +4205,8 @@ def _recover_expired_dispatches( -- Legacy recovery may only compensate work with no execution or -- ownership evidence. Inconsistent state is held for human review. AND r.execution_started_at IS NULL + -- A previously consumed reservation is not proven unstarted. + AND r.consumed_at IS NULL AND r.worker_id IS NULL AND r.worker_attestation_sha256 IS NULL AND r.execution_location IS NULL From 0af5f6fe3047394aeb11f1c921940a6dfef7055d Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Thu, 12 Nov 2026 08:28:56 -0600 Subject: [PATCH 32/39] test(subscriber): preserve consumed orphan reservation and meter --- tests/test_dispatch_orphaned_runs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_dispatch_orphaned_runs.py b/tests/test_dispatch_orphaned_runs.py index 7286c2b..8dfef4f 100644 --- a/tests/test_dispatch_orphaned_runs.py +++ b/tests/test_dispatch_orphaned_runs.py @@ -126,6 +126,7 @@ def test_a_failed_dispatch_whose_run_already_finished_is_left_alone(tmp_path, ma ("table", "column", "value"), [ ("subscriber_run_reservations", "execution_started_at", "2026-11-12T12:00:00Z"), + ("subscriber_run_reservations", "consumed_at", "2026-11-12T12:01:00Z"), ("subscriber_run_reservations", "worker_id", "synthetic-test-worker"), ("subscriber_run_reservations", "worker_attestation_sha256", "1" * 64), ("subscriber_run_reservations", "execution_location", "synthetic-test-location"), From 7119fe77808634effba8aa5607bdb1422a152ed3 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Thu, 12 Nov 2026 08:33:02 -0600 Subject: [PATCH 33/39] fix(subscriber): fail closed on contradictory ownership at final dispatch --- src/echo_certification_forge/subscriber.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/echo_certification_forge/subscriber.py b/src/echo_certification_forge/subscriber.py index f3b2196..4cca4bf 100644 --- a/src/echo_certification_forge/subscriber.py +++ b/src/echo_certification_forge/subscriber.py @@ -4387,6 +4387,17 @@ def finish_dispatch( """ SELECT d.state, d.attempts, d.idempotency_key, runs.state AS run_state, r.state AS reservation_state, + CASE WHEN r.execution_started_at IS NULL + AND r.consumed_at IS NULL + AND r.worker_id IS NULL + AND r.worker_attestation_sha256 IS NULL + AND r.execution_location IS NULL + AND r.signing_authority IS NULL + AND r.signing_key_id IS NULL + AND r.claim_token IS NULL + AND r.lease_expires_at IS NULL + AND r.last_heartbeat_at IS NULL + THEN 1 ELSE 0 END AS unstarted_without_ownership, COALESCE( ( SELECT u.occurred_at @@ -4453,7 +4464,12 @@ def finish_dispatch( state = "FAILED" elif error is not None and int(row["attempts"]) < self.policy.max_dispatch_attempts: state = "PENDING" - elif error is not None and run_state == "QUEUED" and row["reservation_state"] == "BOUND": + elif ( + error is not None + and run_state == "QUEUED" + and row["reservation_state"] == "BOUND" + and int(row["unstarted_without_ownership"]) == 1 + ): # Last attempt failed before execution started (for example # subscriber_worker_claim_denied). Marking only the dispatch FAILED left the run # QUEUED forever and uncompensated (fleet #49604). From b98d1cedd6532ac29fd7dfb33adb340a953975fc Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Thu, 12 Nov 2026 08:33:10 -0600 Subject: [PATCH 34/39] test(subscriber): block final-attempt compensation with active evidence --- tests/test_dispatch_orphaned_runs.py | 39 ++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/test_dispatch_orphaned_runs.py b/tests/test_dispatch_orphaned_runs.py index 8dfef4f..5dc8298 100644 --- a/tests/test_dispatch_orphaned_runs.py +++ b/tests/test_dispatch_orphaned_runs.py @@ -167,3 +167,42 @@ def test_orphan_recovery_preserves_conflicting_ownership( (run_id, org.organization_id), ).fetchone()[0] assert state == "BOUND" + + +@pytest.mark.parametrize( + ("column", "value"), + [ + ("consumed_at", "2026-11-12T14:00:00Z"), + ("execution_started_at", "2026-11-12T14:00:00Z"), + ("worker_id", "synthetic-live-worker"), + ], +) +def test_final_dispatch_error_preserves_conflicting_execution_evidence( + tmp_path, manifest, column, value +) -> None: + """An exhausted claimed dispatch cannot erase contradictory ownership.""" + store, governance, org, run_id = _run( + tmp_path, manifest, "final-attempt-owner", max_attempts=1 + ) + dispatch = governance.claim_dispatch( + "dispatcher-a", run_id=run_id, organization_id=org.organization_id + ) + assert dispatch is not None + # Synthetic state mutation is confined to the disposable test fixture DB. + with sqlite3.connect(governance.db_path) as connection: + connection.execute( + f"UPDATE subscriber_run_reservations SET {column} = ? " + "WHERE run_id = ? AND organization_id = ?", + (value, run_id, org.organization_id), + ) + assert governance.finish_dispatch( + dispatch, error="subscriber_worker_claim_denied" + ) == "FAILED" + assert store.get_run(run_id, org.organization_id)["state"] == RunState.QUEUED.value + assert governance.dispatch_status(run_id, org.organization_id)["state"] == "FAILED" + usage = _usage(governance, org) + assert usage["meters"]["certification_runs"] == 1 + assert usage["active_run_reservations"] == 1 + # No historical reconcile may reinterpret the contradictory record as unstarted. + assert governance.claim_dispatch("dispatcher-b", organization_id=org.organization_id) is None + assert store.get_run(run_id, org.organization_id)["state"] == RunState.QUEUED.value From c95dd7d2a971ea26d827382a4f1d5ec96fa327d6 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Thu, 12 Nov 2026 09:36:21 -0600 Subject: [PATCH 35/39] fix(subscriber): hold expired dispatch leases with execution evidence Fail closed on lease expiry and retry errors when worker ownership, reservation consumption or execution markers exist. Preserve run/reservation/metering evidence and add synthetic negative and positive recovery controls. Reviewed locally: 30 focused pytest cases passed; Git diff check clean. --- src/echo_certification_forge/subscriber.py | 43 +++++++++- tests/test_dispatch_orphaned_runs.py | 92 ++++++++++++++++++++++ 2 files changed, 132 insertions(+), 3 deletions(-) diff --git a/src/echo_certification_forge/subscriber.py b/src/echo_certification_forge/subscriber.py index 4cca4bf..57b0a45 100644 --- a/src/echo_certification_forge/subscriber.py +++ b/src/echo_certification_forge/subscriber.py @@ -4083,6 +4083,17 @@ def _recover_expired_dispatches( rows = connection.execute( """ SELECT d.run_id, d.organization_id, d.idempotency_key, d.attempts, + CASE WHEN r.execution_started_at IS NULL + AND r.consumed_at IS NULL + AND r.worker_id IS NULL + AND r.worker_attestation_sha256 IS NULL + AND r.execution_location IS NULL + AND r.signing_authority IS NULL + AND r.signing_key_id IS NULL + AND r.claim_token IS NULL + AND r.lease_expires_at IS NULL + AND r.last_heartbeat_at IS NULL + THEN 1 ELSE 0 END AS unstarted_without_ownership, COALESCE( ( SELECT u.occurred_at @@ -4111,7 +4122,14 @@ def _recover_expired_dispatches( (now_text,), ).fetchall() for row in rows: - if int(row["attempts"]) >= self.policy.max_dispatch_attempts: + ownership_review_required = int(row["unstarted_without_ownership"]) != 1 + meter_compensated = False + if ownership_review_required: + # Expired dispatcher lease cannot prove that live worker work stopped. + # Hold the run and reservation for independent ownership review. + next_state = "FAILED" + error = "dispatch_lease_expired_ownership_review" + elif int(row["attempts"]) >= self.policy.max_dispatch_attempts: self._transition_run_with_connection( connection, str(row["run_id"]), @@ -4149,6 +4167,7 @@ def _recover_expired_dispatches( ) next_state = "FAILED" error = "dispatch_attempts_exhausted" + meter_compensated = True else: next_state = "PENDING" error = "dispatch_lease_expired" @@ -4172,7 +4191,8 @@ def _recover_expired_dispatches( details={ "recovery": next_state.lower(), "attempts": int(row["attempts"]), - "meter_compensated": next_state == "FAILED", + "meter_compensated": meter_compensated, + "ownership_review_required": ownership_review_required, }, ) # Runs orphaned before fleet #49604: dispatch FAILED, run still QUEUED, reservation BOUND. @@ -4463,7 +4483,8 @@ def finish_dispatch( ) state = "FAILED" elif error is not None and int(row["attempts"]) < self.policy.max_dispatch_attempts: - state = "PENDING" + # Never requeue if the worker may already have started execution. + state = "PENDING" if int(row["unstarted_without_ownership"]) == 1 else "FAILED" elif ( error is not None and run_state == "QUEUED" @@ -4486,6 +4507,22 @@ def finish_dispatch( state = "FAILED" else: state = "FAILED" + if ( + error is not None + and run_state == "QUEUED" + and row["reservation_state"] == "BOUND" + and int(row["unstarted_without_ownership"]) != 1 + ): + self._append_audit( + connection, + organization_id=dispatch.organization_id, + actor_ref="certforge.dispatcher", + action="certification.dispatch_ownership_review_required", + resource_type="certification", + resource_id=dispatch.run_id, + outcome="denied", + details={"attempts": int(row["attempts"]), "meter_compensated": False}, + ) connection.execute( """ UPDATE subscriber_run_dispatches diff --git a/tests/test_dispatch_orphaned_runs.py b/tests/test_dispatch_orphaned_runs.py index 5dc8298..fee0364 100644 --- a/tests/test_dispatch_orphaned_runs.py +++ b/tests/test_dispatch_orphaned_runs.py @@ -206,3 +206,95 @@ def test_final_dispatch_error_preserves_conflicting_execution_evidence( # No historical reconcile may reinterpret the contradictory record as unstarted. assert governance.claim_dispatch("dispatcher-b", organization_id=org.organization_id) is None assert store.get_run(run_id, org.organization_id)["state"] == RunState.QUEUED.value + + +@pytest.mark.parametrize("max_attempts", [1, 3]) +@pytest.mark.parametrize(("column", "value"), [ + ("consumed_at", "2026-11-12T12:00:00Z"), + ("execution_started_at", "2026-11-12T12:00:00Z"), + ("worker_id", "synthetic-worker"), +]) +def test_expired_claim_with_ownership_requires_review( + tmp_path, manifest, max_attempts, column, value +): + store, governance, org, run_id = _run( + tmp_path, manifest, "expired-dispatch-owner", max_attempts=max_attempts + ) + dispatch = governance.claim_dispatch( + "dispatcher-a", run_id=run_id, organization_id=org.organization_id + ) + assert dispatch is not None + # Deliberate contradictory state exists only in this disposable test database. + with sqlite3.connect(governance.db_path) as connection: + connection.execute( + f"UPDATE subscriber_run_reservations SET {column} = ? WHERE run_id = ?", + (value, run_id), + ) + connection.execute( + "UPDATE subscriber_run_dispatches " + "SET lease_expires_at = '2000-01-01T00:00:00Z', attempts = ? " + "WHERE run_id = ?", (max_attempts, run_id) + ) + assert governance.claim_dispatch( + "dispatcher-b", organization_id=org.organization_id + ) is None + assert store.get_run(run_id, org.organization_id)["state"] == RunState.QUEUED.value + assert governance.dispatch_status(run_id, org.organization_id)["state"] == "FAILED" + usage = _usage(governance, org) + assert usage["meters"]["certification_runs"] == 1 + assert usage["active_run_reservations"] == 1 + +@pytest.mark.parametrize(("column", "value"), [ + ("consumed_at", "2026-11-12T12:00:00Z"), + ("worker_id", "synthetic-worker"), +]) +def test_retryable_dispatch_with_ownership_is_not_requeued( + tmp_path, manifest, column, value +): + store, governance, org, run_id = _run( + tmp_path, manifest, "retryable-dispatch-owner", max_attempts=3 + ) + dispatch = governance.claim_dispatch( + "dispatcher-a", run_id=run_id, organization_id=org.organization_id + ) + assert dispatch is not None + with sqlite3.connect(governance.db_path) as connection: + connection.execute( + f"UPDATE subscriber_run_reservations SET {column} = ? WHERE run_id = ?", + (value, run_id), + ) + assert governance.finish_dispatch( + dispatch, error="subscriber_worker_claim_denied" + ) == "FAILED" + assert store.get_run(run_id, org.organization_id)["state"] == RunState.QUEUED.value + usage = _usage(governance, org) + assert usage["meters"]["certification_runs"] == 1 + assert usage["active_run_reservations"] == 1 + assert governance.claim_dispatch( + "dispatcher-b", organization_id=org.organization_id + ) is None + + +@pytest.mark.parametrize("max_attempts,expected_state,expected_meter", [ + (1, RunState.INFRASTRUCTURE_FAILURE.value, 0), + (3, RunState.QUEUED.value, 1), +]) +def test_expired_claim_without_ownership_preserves_existing_policy( + tmp_path, manifest, max_attempts, expected_state, expected_meter +): + store, governance, org, run_id = _run( + tmp_path, manifest, "expired-unstarted-proof", max_attempts=max_attempts + ) + dispatch = governance.claim_dispatch( + "dispatcher-a", run_id=run_id, organization_id=org.organization_id + ) + assert dispatch is not None + with sqlite3.connect(governance.db_path) as connection: + connection.execute( + "UPDATE subscriber_run_dispatches " + "SET lease_expires_at = '2000-01-01T00:00:00Z' WHERE run_id = ?", + (run_id,) + ) + governance.claim_dispatch("dispatcher-b", organization_id=org.organization_id) + assert store.get_run(run_id, org.organization_id)["state"] == expected_state + assert _usage(governance, org)["meters"]["certification_runs"] == expected_meter From 9417dc5a06159291384d11f569a2ffdb5b15c108 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Thu, 12 Nov 2026 11:32:49 -0600 Subject: [PATCH 36/39] ci: integrate exact-source CodeQL and immutable actions for PR61 gate review --- .github/workflows/ci.yml | 52 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01578a4..dadb8ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,15 +11,28 @@ name: certification-forge-ci permissions: contents: read +env: + ECHO_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + ECHO_SCAN_REF: ${{ github.event_name == 'pull_request' && format('refs/pull/{0}/head', github.event.pull_request.number) || github.ref }} + jobs: test: + name: CI runs-on: ubuntu-latest timeout-minutes: 30 steps: - name: Check out exact source - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ env.ECHO_SOURCE_SHA }} + persist-credentials: false + - name: Verify source identity + run: | + echo_actual_sha=$(git rev-parse HEAD) + test "$echo_actual_sha" = "$ECHO_SOURCE_SHA" + printf 'Verified source: %s\n' "$echo_actual_sha" - name: Configure Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.13" cache: pip @@ -37,9 +50,40 @@ jobs: run: python scripts/p7_acceptance.py - name: Upload verification evidence if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: certification-forge-evidence-${{ github.sha }} + name: certification-forge-evidence-${{ env.ECHO_SOURCE_SHA }} path: artifacts/ if-no-files-found: error retention-days: 30 + + codeql: + name: CodeQL + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + actions: read + security-events: write + steps: + - name: Check out exact source for static analysis + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ env.ECHO_SOURCE_SHA }} + persist-credentials: false + - name: Verify source identity + run: | + echo_actual_sha=$(git rev-parse HEAD) + test "$echo_actual_sha" = "$ECHO_SOURCE_SHA" + printf 'Verified source: %s\n' "$echo_actual_sha" + - name: Initialize Python analysis + uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + with: + languages: python + build-mode: none + - name: Analyze and upload results + uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + with: + category: /language:python + ref: ${{ env.ECHO_SCAN_REF }} + sha: ${{ env.ECHO_SOURCE_SHA }} From c78783c8b8a5bf609ec7fc98dc63281ceae505c2 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Thu, 12 Nov 2026 13:34:13 -0600 Subject: [PATCH 37/39] fix(subscriber): retain redacted dispatch claim custody in chained audit --- src/echo_certification_forge/subscriber.py | 28 +++++++++++++++++----- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/echo_certification_forge/subscriber.py b/src/echo_certification_forge/subscriber.py index 57b0a45..a6f0de9 100644 --- a/src/echo_certification_forge/subscriber.py +++ b/src/echo_certification_forge/subscriber.py @@ -4083,6 +4083,9 @@ def _recover_expired_dispatches( rows = connection.execute( """ SELECT d.run_id, d.organization_id, d.idempotency_key, d.attempts, + d.lease_owner AS previous_lease_owner, + d.lease_expires_at AS previous_lease_expires_at, + d.claim_token AS previous_claim_token, CASE WHEN r.execution_started_at IS NULL AND r.consumed_at IS NULL AND r.worker_id IS NULL @@ -4180,6 +4183,24 @@ def _recover_expired_dispatches( """, (next_state, error, now_text, row["run_id"]), ) + recovery_details = { + "recovery": next_state.lower(), + "attempts": int(row["attempts"]), + "meter_compensated": meter_compensated, + "ownership_review_required": ownership_review_required, + } + if ownership_review_required: + # Preserve non-secret lease provenance in the append-only audit + # before mutable claim fields are cleared. Never persist a replayable token. + previous_claim = row["previous_claim_token"] + recovery_details.update({ + "previous_lease_owner": row["previous_lease_owner"], + "previous_lease_expires_at": row["previous_lease_expires_at"], + "previous_claim_token_sha256": ( + hashlib.sha256(str(previous_claim).encode("utf-8")).hexdigest() + if previous_claim is not None else None + ), + }) self._append_audit( connection, organization_id=str(row["organization_id"]), @@ -4188,12 +4209,7 @@ def _recover_expired_dispatches( resource_type="certification", resource_id=str(row["run_id"]), outcome="allowed", - details={ - "recovery": next_state.lower(), - "attempts": int(row["attempts"]), - "meter_compensated": meter_compensated, - "ownership_review_required": ownership_review_required, - }, + details=recovery_details, ) # Runs orphaned before fleet #49604: dispatch FAILED, run still QUEUED, reservation BOUND. # Once terminalized they no longer match, so this pass is idempotent. From a0515d875c2a384965229eb5716331a776ab8190 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Thu, 12 Nov 2026 13:34:16 -0600 Subject: [PATCH 38/39] test(subscriber): prove expired claim custody and token secrecy --- tests/test_dispatch_orphaned_runs.py | 90 ++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/tests/test_dispatch_orphaned_runs.py b/tests/test_dispatch_orphaned_runs.py index fee0364..388687a 100644 --- a/tests/test_dispatch_orphaned_runs.py +++ b/tests/test_dispatch_orphaned_runs.py @@ -298,3 +298,93 @@ def test_expired_claim_without_ownership_preserves_existing_policy( governance.claim_dispatch("dispatcher-b", organization_id=org.organization_id) assert store.get_run(run_id, org.organization_id)["state"] == expected_state assert _usage(governance, org)["meters"]["certification_runs"] == expected_meter + + +def test_expired_owned_dispatch_preserves_nonsecret_claim_custody(tmp_path, manifest): + # Disposable SQLite fixture only: no production mutation or artificial heartbeat. + import hashlib + import json + + store, governance, org, run_id = _run( + tmp_path, manifest, "expired-custody", max_attempts=3 + ) + dispatch = governance.claim_dispatch( + "dispatcher-a", run_id=run_id, organization_id=org.organization_id + ) + assert dispatch is not None + with sqlite3.connect(governance.db_path) as connection: + connection.execute( + "UPDATE subscriber_run_reservations SET consumed_at = ? WHERE run_id = ?", + ("2026-11-12T12:00:00Z", run_id), + ) + connection.execute( + "UPDATE subscriber_run_dispatches SET lease_expires_at = ? WHERE run_id = ?", + ("2000-01-01T00:00:00Z", run_id), + ) + assert governance.claim_dispatch( + "dispatcher-b", organization_id=org.organization_id + ) is None + with sqlite3.connect(governance.db_path) as connection: + row = connection.execute( + "SELECT details_json FROM subscriber_audit_events " + "WHERE organization_id = ? AND resource_id = ? " + "AND action = 'certification.dispatch_recovered' " + "ORDER BY event_id DESC LIMIT 1", + (org.organization_id, run_id), + ).fetchone() + state = connection.execute( + "SELECT state, claim_token, lease_owner, lease_expires_at " + "FROM subscriber_run_dispatches WHERE run_id = ?", (run_id,) + ).fetchone() + assert row is not None + details = json.loads(row[0]) + assert details["ownership_review_required"] is True + assert details["meter_compensated"] is False + assert details["previous_lease_owner"] == "dispatcher-a" + assert details["previous_lease_expires_at"] == "2000-01-01T00:00:00Z" + assert details["previous_claim_token_sha256"] == hashlib.sha256( + dispatch.claim_token.encode("utf-8") + ).hexdigest() + assert dispatch.claim_token not in row[0] + assert state == ("FAILED", None, None, None) + assert store.get_run(run_id, org.organization_id)["state"] == RunState.QUEUED.value + assert _usage(governance, org)["meters"]["certification_runs"] == 1 + principal = _owner( + governance, org.organization_id, org.bootstrap_api_key, Permission.AUDIT_READ + ) + verification = governance.verify_audit(principal) + assert verification["valid"] is True + assert verification["invalid_event_ids"] == [] + + +def test_expired_unstarted_dispatch_does_not_expose_claim_token_in_audit(tmp_path, manifest): + import json + + store, governance, org, run_id = _run( + tmp_path, manifest, "unstarted-custody", max_attempts=1 + ) + dispatch = governance.claim_dispatch( + "dispatcher-a", run_id=run_id, organization_id=org.organization_id + ) + assert dispatch is not None + with sqlite3.connect(governance.db_path) as connection: + connection.execute( + "UPDATE subscriber_run_dispatches SET lease_expires_at = ? WHERE run_id = ?", + ("2000-01-01T00:00:00Z", run_id), + ) + governance.claim_dispatch("dispatcher-b", organization_id=org.organization_id) + with sqlite3.connect(governance.db_path) as connection: + row = connection.execute( + "SELECT details_json FROM subscriber_audit_events " + "WHERE organization_id = ? AND resource_id = ? " + "AND action = 'certification.dispatch_recovered' " + "ORDER BY event_id DESC LIMIT 1", + (org.organization_id, run_id), + ).fetchone() + assert row is not None + details = json.loads(row[0]) + assert details["ownership_review_required"] is False + assert "previous_claim_token_sha256" not in details + assert dispatch.claim_token not in row[0] + assert store.get_run(run_id, org.organization_id)["state"] == RunState.INFRASTRUCTURE_FAILURE.value + assert _usage(governance, org)["meters"]["certification_runs"] == 0 From e6b51c683c0027cab3ed446a26fdd614476a32c6 Mon Sep 17 00:00:00 2001 From: ECHO OMEGA PRIME Date: Thu, 12 Nov 2026 15:33:53 -0600 Subject: [PATCH 39/39] fix(security): peppered HMAC for prior dispatch claim custody Replace bare SHA-256 on the replayable dispatch claim token with existing server-peppered HMAC-SHA256, and assert the audit record never contains the raw token or its unkeyed digest. Addresses high CodeQL finding py/weak-sensitive-data-hashing at prior integrated head. Exact focused FORGE test: 2 passed, 30 deselected; immutable source/test blobs verified. --- src/echo_certification_forge/subscriber.py | 4 ++-- tests/test_dispatch_orphaned_runs.py | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/echo_certification_forge/subscriber.py b/src/echo_certification_forge/subscriber.py index a6f0de9..fef7c65 100644 --- a/src/echo_certification_forge/subscriber.py +++ b/src/echo_certification_forge/subscriber.py @@ -4196,8 +4196,8 @@ def _recover_expired_dispatches( recovery_details.update({ "previous_lease_owner": row["previous_lease_owner"], "previous_lease_expires_at": row["previous_lease_expires_at"], - "previous_claim_token_sha256": ( - hashlib.sha256(str(previous_claim).encode("utf-8")).hexdigest() + "previous_claim_token_hmac_sha256": ( + self._token_digest(str(previous_claim)) if previous_claim is not None else None ), }) diff --git a/tests/test_dispatch_orphaned_runs.py b/tests/test_dispatch_orphaned_runs.py index 388687a..3baea28 100644 --- a/tests/test_dispatch_orphaned_runs.py +++ b/tests/test_dispatch_orphaned_runs.py @@ -342,7 +342,10 @@ def test_expired_owned_dispatch_preserves_nonsecret_claim_custody(tmp_path, mani assert details["meter_compensated"] is False assert details["previous_lease_owner"] == "dispatcher-a" assert details["previous_lease_expires_at"] == "2000-01-01T00:00:00Z" - assert details["previous_claim_token_sha256"] == hashlib.sha256( + assert details["previous_claim_token_hmac_sha256"] == governance._token_digest( + dispatch.claim_token + ) + assert details["previous_claim_token_hmac_sha256"] != hashlib.sha256( dispatch.claim_token.encode("utf-8") ).hexdigest() assert dispatch.claim_token not in row[0] @@ -384,7 +387,7 @@ def test_expired_unstarted_dispatch_does_not_expose_claim_token_in_audit(tmp_pat assert row is not None details = json.loads(row[0]) assert details["ownership_review_required"] is False - assert "previous_claim_token_sha256" not in details + assert "previous_claim_token_hmac_sha256" not in details assert dispatch.claim_token not in row[0] assert store.get_run(run_id, org.organization_id)["state"] == RunState.INFRASTRUCTURE_FAILURE.value assert _usage(governance, org)["meters"]["certification_runs"] == 0