From 50a0f869cab073e241d5047641235e021dcf2ef7 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:49:38 -0600 Subject: [PATCH 1/7] Verify native OCI provenance in failover artifact inspector --- .github/workflows/phpunit-feature.yml | 3 + .github/workflows/single-region-failover.yml | 2 + docs/ha-failover-validation.md | 15 ++ ...gle-region-failover-published-artifacts.py | 122 ++++++++++-- .../single_region_failover_identity_test.py | 186 ++++++++++++++++++ 5 files changed, 311 insertions(+), 17 deletions(-) create mode 100644 tests/Unit/Support/single_region_failover_identity_test.py diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index 75a665f54..e92cfb3e6 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -152,6 +152,9 @@ jobs: - name: Prove the image installs and checks the fixed Brotli decoder run: python3 -m unittest tests/Unit/Support/brotli_trust_test.py + - name: Check published failover image identity and recovery result gates + run: python3 -m unittest discover -s tests/Unit/Support -p 'single_region_failover*test.py' + feature: name: PHPUnit feature suite needs: action-policy diff --git a/.github/workflows/single-region-failover.yml b/.github/workflows/single-region-failover.yml index 5e16e3f68..6451cdcf8 100644 --- a/.github/workflows/single-region-failover.yml +++ b/.github/workflows/single-region-failover.yml @@ -11,6 +11,7 @@ on: permissions: contents: read + attestations: read concurrency: group: single-region-failover-${{ github.event_name }}-${{ github.ref }} @@ -37,6 +38,7 @@ jobs: - name: Run bounded published-artifact rehearsal env: + GH_TOKEN: ${{ github.token }} DW_FAILOVER_MODE: bounded DW_FAILOVER_PROJECT: single-region-failover-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} DW_FAILOVER_SERVER_A_PORT: "0" diff --git a/docs/ha-failover-validation.md b/docs/ha-failover-validation.md index 0610716a5..17197a89b 100644 --- a/docs/ha-failover-validation.md +++ b/docs/ha-failover-validation.md @@ -285,6 +285,21 @@ DW_SERVER_IMAGE=durableworkflow/server: \ --result-dir ./failover-result ``` +Native images also require Docker Buildx and a GitHub CLI that supports `gh attestation verify` +and `--deny-self-hosted-runners`; `GH` can select its executable. The host +inspector checks the exact OCI index, platform manifest and configuration digest, native entrypoint, +source/release/build labels, and signed provenance for that exact image, +repository, workflow, tag, source commit and build run. It records +`artifacts.native_server_identity`. Its `release_version` comes from the +verified release tag; serving-version and recovery observations remain part of +the later runtime phases. It neither invokes PHP nor invents Composer package +records for a native image. + +PHP images keep the existing `embedded_workflow_package` version/reference +inspection. Passing either artifact inspector supplies only the provenance +phase: every topology and recovery cell is still required before the complete +rehearsal can pass. An inspector-only check does not qualify native failover. + The runner connects to all three published endpoints through `127.0.0.1` by default. When the runner itself is in a container that controls a Docker daemon through a mounted socket, set `DW_FAILOVER_CONNECT_HOST` to the Docker host diff --git a/scripts/conformance/single-region-failover-published-artifacts.py b/scripts/conformance/single-region-failover-published-artifacts.py index e0bd569f8..b65b961ad 100755 --- a/scripts/conformance/single-region-failover-published-artifacts.py +++ b/scripts/conformance/single-region-failover-published-artifacts.py @@ -971,21 +971,7 @@ def require_passing_result() -> None: require_recovery_bounds(BOUNDS) -def provenance_phase() -> dict[str, Any]: - require(CONFIGURATION_ERROR is None, CONFIGURATION_ERROR or "invalid runner configuration") - raw = compose("config", "--format", "json").stdout - config = json.loads(raw) - services = config.get("services", {}) - require(set(services) == {"bootstrap", "server-a", "server-b", "queue-worker", "scheduler", "load-balancer", "mysql", "redis"}, "unexpected Compose topology") - require(all("build" not in service for service in services.values()), "Compose build sections are forbidden") - for service_name, service in services.items(): - image = str(service.get("image", "")) - require("@sha256:" in image, f"{service_name} is not repository-digest pinned: {image}") - for volume in service.get("volumes", []) or []: - volume_type = volume.get("type") if isinstance(volume, dict) else "volume" - require(volume_type != "bind", f"bind mount is forbidden for {service_name}: {volume}") - require(sum(1 for name in services if name == "scheduler") == 1, "exactly one scheduler service is required") - require(sum(1 for name in services if name == "queue-worker") == 1, "exactly one queue worker service is required") +def inspect_php_workflow_package() -> dict[str, Any]: package_script = r''' $lock = json_decode(file_get_contents('/app/composer.lock'), true); $package = null; @@ -1018,7 +1004,109 @@ def provenance_phase() -> dict[str, Any]: bool(workflow_package.get("source_reference") or workflow_package.get("dist_reference") or workflow_package.get("image_package_provenance")), "embedded workflow package has no exact source reference", ) - RESULT["artifacts"]["embedded_workflow_package"] = workflow_package + return {"embedded_workflow_package": workflow_package} + + +def inspect_native_server(image: str, metadata: dict[str, Any]) -> dict[str, Any]: + config = metadata.get("Config") or {} + labels = config.get("Labels") or {} + require(config.get("Entrypoint") == ["/usr/local/bin/durable-workflow-server"], "native image entrypoint mismatch") + require(metadata.get("Os") == "linux" and metadata.get("Architecture") in {"amd64", "arm64"}, "unsupported native image platform") + require(re.fullmatch(r"(?:docker\.io/|index\.docker\.io/)?durableworkflow/server@sha256:[0-9a-f]{64}", image) is not None, "native image must be an exact public repository digest") + canonical = lambda value: re.sub(r"^(?:docker\.io/|index\.docker\.io/)", "", value) + require(canonical(image) in {canonical(value) for value in metadata.get("RepoDigests", [])}, "native image digest is not bound to inspected config") + require(labels.get("org.opencontainers.image.source") == "https://github.com/durable-workflow/server", "native image source repository mismatch") + require(labels.get("org.opencontainers.image.title") == "Durable Workflow Server (native)", "native image title mismatch") + revision = labels.get("org.opencontainers.image.revision", "") + tag = labels.get("dev.durable-workflow.release.tag", "") + run_id = labels.get("dev.durable-workflow.release.run-id", "") + require(re.fullmatch(r"[0-9a-f]{40}", revision) is not None and revision != "0" * 40, "native image source revision missing") + require(re.fullmatch(r"(?:native-)?(?:[3-9]|[1-9][0-9]+)\.[0-9]+\.[0-9]+(?:-rc\.[1-9][0-9]*)?", tag) is not None, "native image release tag invalid") + require(re.fullmatch(r"[1-9][0-9]*", run_id) is not None, "native image release run missing") + version = tag.removeprefix("native-") + + def manifest(reference: str) -> dict[str, Any]: + raw = command(["docker", "buildx", "imagetools", "inspect", "--raw", reference], timeout=60).stdout + require("sha256:" + hashlib.sha256(raw.encode()).hexdigest() == reference.split("@", 1)[1], "native OCI manifest digest mismatch") + return json.loads(raw) + + index = manifest(image) + require(index.get("mediaType") == "application/vnd.oci.image.index.v1+json", "native image must resolve to an OCI index") + platforms = [entry for entry in index.get("manifests", []) + if entry.get("platform", {}).get("os") == metadata["Os"] + and entry.get("platform", {}).get("architecture") == metadata["Architecture"]] + require(len(platforms) == 1, "native OCI platform selection is ambiguous") + platform_digest = platforms[0].get("digest", "") + require(re.fullmatch(r"sha256:[0-9a-f]{64}", platform_digest) is not None, "native OCI platform digest missing") + platform = manifest(image.split("@", 1)[0] + "@" + platform_digest) + require(platform.get("mediaType") == "application/vnd.oci.image.manifest.v1+json", "native OCI platform manifest invalid") + config_digest = platform.get("config", {}).get("digest", "") + require(re.fullmatch(r"sha256:[0-9a-f]{64}", config_digest) is not None, "native OCI config digest missing") + # Docker's classic and containerd stores expose different image IDs. + require(metadata.get("Id") in {image.split("@", 1)[1], platform_digest, config_digest}, "native local image ID is not bound to its OCI descriptors") + verified = command([ + os.environ.get("GH", "gh"), "attestation", "verify", "oci://" + image, + "--repo", "durable-workflow/server", + "--signer-workflow", "github.com/durable-workflow/server/.github/workflows/native-release.yml", + "--source-ref", "refs/tags/" + tag, "--source-digest", revision, + "--deny-self-hosted-runners", "--format", "json", + ], timeout=60) + attestations = json.loads(verified.stdout) + require(isinstance(attestations, list) and bool(attestations), "native image has no verified attestation") + invocations = [] + for attestation in attestations: + verification = attestation.get("verificationResult", {}) + subjects = verification.get("statement", {}).get("subject", []) + require(any(subject.get("digest", {}).get("sha256") == image.split("@sha256:", 1)[1] for subject in subjects), "native attestation subject mismatch") + certificate = verification.get("signature", {}).get("certificate", {}) + require(certificate.get("sourceRepositoryURI") == "https://github.com/durable-workflow/server", "native attestation repository mismatch") + require(certificate.get("sourceRepositoryDigest") == revision, "native attestation source mismatch") + require(certificate.get("sourceRepositoryRef") == "refs/tags/" + tag, "native attestation tag mismatch") + require(certificate.get("runnerEnvironment") == "github-hosted", "native attestation runner mismatch") + require(certificate.get("buildSignerURI") == "https://github.com/durable-workflow/server/.github/workflows/native-release.yml@refs/tags/" + tag, "native attestation signer mismatch") + invocation = certificate.get("runInvocationURI", "") + require(re.fullmatch(r"https://github\.com/durable-workflow/server/actions/runs/" + run_id + r"/attempts/[1-9][0-9]*", invocation) is not None, "native attestation release run mismatch") + invocations.append(invocation) + return {"native_server_identity": { + "runtime": "native", "image_repo_digest": image, + "image_manifest_digest": platform_digest, "image_config_digest": config_digest, + "local_image_id": metadata["Id"], + "platform": metadata["Os"] + "/" + metadata["Architecture"], + "source_repository": labels["org.opencontainers.image.source"], "source_revision": revision, + "release_tag": tag, "release_version": version, "release_run_id": run_id, + "attestation_invocations": sorted(set(invocations)), + }} + + +def inspect_published_server() -> dict[str, Any]: + image = os.environ["DW_FAILOVER_SERVER_IMAGE"] + inspected = json.loads(command(["docker", "image", "inspect", image], timeout=30).stdout) + require(isinstance(inspected, list) and len(inspected) == 1, "server image inspection is ambiguous") + metadata = inspected[0] + config = metadata.get("Config") or {} + labels = config.get("Labels") or {} + if config.get("Entrypoint") == ["/usr/local/bin/durable-workflow-server"] or labels.get("org.opencontainers.image.title") == "Durable Workflow Server (native)": + return inspect_native_server(image, metadata) + return inspect_php_workflow_package() + + +def provenance_phase() -> dict[str, Any]: + require(CONFIGURATION_ERROR is None, CONFIGURATION_ERROR or "invalid runner configuration") + raw = compose("config", "--format", "json").stdout + config = json.loads(raw) + services = config.get("services", {}) + require(set(services) == {"bootstrap", "server-a", "server-b", "queue-worker", "scheduler", "load-balancer", "mysql", "redis"}, "unexpected Compose topology") + require(all("build" not in service for service in services.values()), "Compose build sections are forbidden") + for service_name, service in services.items(): + image = str(service.get("image", "")) + require("@sha256:" in image, f"{service_name} is not repository-digest pinned: {image}") + for volume in service.get("volumes", []) or []: + volume_type = volume.get("type") if isinstance(volume, dict) else "volume" + require(volume_type != "bind", f"bind mount is forbidden for {service_name}: {volume}") + require(sum(1 for name in services if name == "scheduler") == 1, "exactly one scheduler service is required") + require(sum(1 for name in services if name == "queue-worker") == 1, "exactly one queue worker service is required") + identity = inspect_published_server() + RESULT["artifacts"].update(identity) digest = hashlib.sha256(raw.encode()).hexdigest() RESULT["artifacts"]["compose_config_sha256"] = digest return { @@ -1027,7 +1115,7 @@ def provenance_phase() -> dict[str, Any]: "all_runtime_images_repo_digest_pinned": True, "compose_build_sections_present": False, "product_source_bind_mounts_present": False, - "embedded_workflow_package": workflow_package, + **identity, } diff --git a/tests/Unit/Support/single_region_failover_identity_test.py b/tests/Unit/Support/single_region_failover_identity_test.py new file mode 100644 index 000000000..af16e0acc --- /dev/null +++ b/tests/Unit/Support/single_region_failover_identity_test.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""Artifact identity controls; real image/signature checks are separate receipts.""" + +import hashlib +import json +import os +import subprocess +import unittest +from unittest import mock + +from single_region_failover_result_gate_test import runner + + +CONFIG_DIGEST = "sha256:" + "c" * 64 +MANIFEST_RAW = json.dumps({"mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": {"digest": CONFIG_DIGEST}}) +MANIFEST_DIGEST = "sha256:" + hashlib.sha256(MANIFEST_RAW.encode()).hexdigest() +INDEX_RAW = json.dumps({"mediaType": "application/vnd.oci.image.index.v1+json", "manifests": [ + {"digest": MANIFEST_DIGEST, "platform": {"os": "linux", "architecture": "amd64"}}]}) +IMAGE = "durableworkflow/server@sha256:" + hashlib.sha256(INDEX_RAW.encode()).hexdigest() +SOURCE = "b" * 40 +REPOSITORY = "https://github.com/durable-workflow/server" +WORKFLOW = REPOSITORY + "/.github/workflows/native-release.yml" + + +class PublishedIdentityTest(unittest.TestCase): + def setUp(self) -> None: + self.environment = mock.patch.dict(os.environ, { + "DW_FAILOVER_SERVER_IMAGE": IMAGE, + "GH": "gh", + }) + self.environment.start() + self.addCleanup(self.environment.stop) + self.commands = [] + self.index_raw = INDEX_RAW + self.manifest_raw = MANIFEST_RAW + self.metadata = { + "Id": CONFIG_DIGEST, "RepoDigests": [IMAGE], + "Os": "linux", "Architecture": "amd64", + "Config": {"Entrypoint": ["/usr/local/bin/durable-workflow-server"], "Labels": { + "org.opencontainers.image.source": REPOSITORY, + "org.opencontainers.image.title": "Durable Workflow Server (native)", + "org.opencontainers.image.revision": SOURCE, + "dev.durable-workflow.release.tag": "3.1.0", + "dev.durable-workflow.release.run-id": "12345", + }}, + } + self.certificate = { + "sourceRepositoryURI": REPOSITORY, "sourceRepositoryDigest": SOURCE, + "sourceRepositoryRef": "refs/tags/3.1.0", "runnerEnvironment": "github-hosted", + "buildSignerURI": WORKFLOW + "@refs/tags/3.1.0", + "runInvocationURI": REPOSITORY + "/actions/runs/12345/attempts/1", + } + self.attestations = [{"verificationResult": { + "signature": {"certificate": self.certificate}, + "statement": {"subject": [{"digest": {"sha256": IMAGE.split("@sha256:")[1]}}]}, + }}] + self.php_package = {"name": "durable-workflow/workflow", "version": "2.15.12", + "source_reference": "d" * 40, "dist_reference": "d" * 40, + "image_package_provenance": [], "php_version": "8.4.27"} + self.verifier_error = None + + def command(self, args, **kwargs): + self.commands.append((args, kwargs)) + if args[:3] == ["docker", "image", "inspect"]: + value = [self.metadata] + elif args[:4] == ["docker", "buildx", "imagetools", "inspect"]: + value = self.index_raw if args[-1] == IMAGE else self.manifest_raw + return subprocess.CompletedProcess(args, 0, value, "") + elif args[:3] == ["gh", "attestation", "verify"]: + if self.verifier_error: + raise self.verifier_error + value = self.attestations + elif args[:3] == ["docker", "run", "--rm"] and "php" in args: + value = self.php_package + else: + self.fail("Unexpected external command") + return subprocess.CompletedProcess(args, 0, json.dumps(value), "") + + def inspect(self): + with mock.patch.object(runner, "command", side_effect=self.command): + return runner.inspect_published_server() + + def test_native_identity_binds_config_source_build_and_verified_subject(self): + value = self.inspect() + self.assertNotIn("embedded_workflow_package", value) + identity = value["native_server_identity"] + self.assertEqual(IMAGE, identity["image_repo_digest"]) + self.assertEqual(CONFIG_DIGEST, identity["image_config_digest"]) + self.assertEqual(MANIFEST_DIGEST, identity["image_manifest_digest"]) + self.assertEqual(SOURCE, identity["source_revision"]) + self.assertEqual("3.1.0", identity["release_version"]) + self.assertEqual("12345", identity["release_run_id"]) + self.assertEqual([self.certificate["runInvocationURI"]], identity["attestation_invocations"]) + self.assertFalse(any(args[:2] == ["docker", "run"] for args, _ in self.commands)) + verifier, options = next((args, options) for args, options in self.commands if args[0] == "gh") + for flag, expected in [("--repo", "durable-workflow/server"), + ("--source-ref", "refs/tags/3.1.0"), ("--source-digest", SOURCE), + ("--signer-workflow", WORKFLOW.removeprefix("https://"))]: + self.assertEqual(expected, verifier[verifier.index(flag) + 1]) + self.assertIn("--deny-self-hosted-runners", verifier) + self.assertEqual(60, options["timeout"]) + + def test_containerd_image_id_is_recorded_separately_from_config_digest(self): + self.metadata["Id"] = IMAGE.split("@", 1)[1] + identity = self.inspect()["native_server_identity"] + self.assertEqual(self.metadata["Id"], identity["local_image_id"]) + self.assertEqual(CONFIG_DIGEST, identity["image_config_digest"]) + self.assertNotEqual(identity["local_image_id"], identity["image_config_digest"]) + + def test_php_inspection_preserves_the_original_package_record(self): + self.metadata["Config"] = {"Entrypoint": ["docker-php-entrypoint"], "Labels": {}} + self.assertEqual({"embedded_workflow_package": self.php_package}, self.inspect()) + self.assertEqual(2, len(self.commands)) + self.assertIn("/app/composer.lock", self.commands[1][0][-1]) + self.assertFalse(any(args[0] == "gh" for args, _ in self.commands)) + + def test_php_inspection_still_rejects_missing_package_provenance(self): + self.metadata["Config"] = {"Entrypoint": ["docker-php-entrypoint"], "Labels": {}} + self.php_package["source_reference"] = None + self.php_package["dist_reference"] = None + with self.assertRaisesRegex(AssertionError, "no exact source reference"): + self.inspect() + + def test_failed_signature_verification_never_falls_back_to_php(self): + self.verifier_error = subprocess.CalledProcessError(1, ["gh", "attestation", "verify"]) + with self.assertRaises(subprocess.CalledProcessError): + self.inspect() + self.assertEqual(4, len(self.commands)) + + def test_native_identity_phase_does_not_pass_recovery_cells(self): + services = {name: {"image": IMAGE} for name in ["bootstrap", "server-a", "server-b", + "queue-worker", "scheduler", "load-balancer", "mysql", "redis"]} + raw = json.dumps({"services": services}) + with mock.patch.object(runner, "command", side_effect=self.command), \ + mock.patch.object(runner, "compose", return_value=subprocess.CompletedProcess([], 0, raw, "")), \ + mock.patch.dict(runner.RESULT, {"artifacts": {}, "phase_outcomes": {}}, clear=False): + evidence = runner.provenance_phase() + self.assertEqual(evidence["native_server_identity"], runner.RESULT["artifacts"]["native_server_identity"]) + self.assertNotIn("embedded_workflow_package", evidence) + runner.RESULT["phase_outcomes"]["published_artifact_provenance"] = {"status": "pass"} + with self.assertRaisesRegex(AssertionError, "required scenario result missing"): + runner.require_passing_result() + + +def rejection(name, mutate): + def test(self): + mutate(self) + with self.assertRaises(AssertionError): + self.inspect() + test.__name__ = "test_rejects_" + name + setattr(PublishedIdentityTest, test.__name__, test) + + +for name, mutate in [ + ("mutable_image", lambda t: os.environ.update(DW_FAILOVER_SERVER_IMAGE="durableworkflow/server:3.1.0")), + ("other_repository", lambda t: os.environ.update(DW_FAILOVER_SERVER_IMAGE="other/server@sha256:" + "a" * 64)), + ("unbound_config", lambda t: t.metadata.update(RepoDigests=[])), + ("invalid_local_image_id", lambda t: t.metadata.update(Id="tag")), + ("unrelated_local_image_id", lambda t: t.metadata.update(Id="sha256:" + "e" * 64)), + ("modified_index_bytes", lambda t: setattr(t, "index_raw", t.index_raw + " ")), + ("modified_platform_bytes", lambda t: setattr(t, "manifest_raw", t.manifest_raw + " ")), + ("unsupported_platform", lambda t: t.metadata.update(Architecture="ppc64le")), + ("wrong_entrypoint", lambda t: t.metadata["Config"].update(Entrypoint=["php"])), + ("wrong_title", lambda t: t.metadata["Config"]["Labels"].update({"org.opencontainers.image.title": "other"})), + ("wrong_source_repository", lambda t: t.metadata["Config"]["Labels"].update({"org.opencontainers.image.source": "https://example.invalid"})), + ("missing_source", lambda t: t.metadata["Config"]["Labels"].pop("org.opencontainers.image.revision")), + ("zero_source", lambda t: t.metadata["Config"]["Labels"].update({"org.opencontainers.image.revision": "0" * 40})), + ("rolling_release_tag", lambda t: t.metadata["Config"]["Labels"].update({"dev.durable-workflow.release.tag": "latest"})), + ("php_version_as_native", lambda t: t.metadata["Config"]["Labels"].update({"dev.durable-workflow.release.tag": "2.11.8"})), + ("missing_build_run", lambda t: t.metadata["Config"]["Labels"].pop("dev.durable-workflow.release.run-id")), + ("empty_attestations", lambda t: t.attestations.clear()), + ("different_signed_subject", lambda t: t.attestations[0]["verificationResult"]["statement"]["subject"][0]["digest"].update(sha256="e" * 64)), + ("different_signed_repository", lambda t: t.certificate.update(sourceRepositoryURI="https://example.invalid")), + ("different_signed_source", lambda t: t.certificate.update(sourceRepositoryDigest="e" * 40)), + ("different_signed_tag", lambda t: t.certificate.update(sourceRepositoryRef="refs/tags/3.0.5")), + ("self_hosted_signer", lambda t: t.certificate.update(runnerEnvironment="self-hosted")), + ("different_workflow_signer", lambda t: t.certificate.update(buildSignerURI=REPOSITORY + "/.github/workflows/other.yml@refs/tags/3.1.0")), + ("different_signed_build_run", lambda t: t.certificate.update(runInvocationURI=REPOSITORY + "/actions/runs/67890/attempts/1")), + ("missing_build_attempt", lambda t: t.certificate.update(runInvocationURI=REPOSITORY + "/actions/runs/12345")), +]: + rejection(name, mutate) + + +if __name__ == "__main__": + unittest.main() From 3209f72f5f3231fbd180c0cf7ff19094e47ec07d Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:59:09 -0600 Subject: [PATCH 2/7] Bind implementation, registry tag and current Git tag identities --- docs/ha-failover-validation.md | 6 ++ ...gle-region-failover-published-artifacts.py | 43 ++++++++- .../single_region_failover_identity_test.py | 91 ++++++++++++++++++- 3 files changed, 132 insertions(+), 8 deletions(-) diff --git a/docs/ha-failover-validation.md b/docs/ha-failover-validation.md index 17197a89b..e8c9a9e03 100644 --- a/docs/ha-failover-validation.md +++ b/docs/ha-failover-validation.md @@ -295,6 +295,12 @@ verified release tag; serving-version and recovery observations remain part of the later runtime phases. It neither invokes PHP nor invents Composer package records for a native image. +The requested registry tag must match the verified release version (`native-` +is removed from historical Git tag names). The current registry tag must still +resolve to the inspected index. The checked-out and current remote Git tag must +both peel to the recorded source commit. Unknown or mixed implementation +identities are refused before any in-image inspection. + PHP images keep the existing `embedded_workflow_package` version/reference inspection. Passing either artifact inspector supplies only the provenance phase: every topology and recovery cell is still required before the complete diff --git a/scripts/conformance/single-region-failover-published-artifacts.py b/scripts/conformance/single-region-failover-published-artifacts.py index b65b961ad..833a8a928 100755 --- a/scripts/conformance/single-region-failover-published-artifacts.py +++ b/scripts/conformance/single-region-failover-published-artifacts.py @@ -1025,9 +1025,9 @@ def inspect_native_server(image: str, metadata: dict[str, Any]) -> dict[str, Any require(re.fullmatch(r"[1-9][0-9]*", run_id) is not None, "native image release run missing") version = tag.removeprefix("native-") - def manifest(reference: str) -> dict[str, Any]: + def manifest(reference: str, expected_digest: str | None = None) -> dict[str, Any]: raw = command(["docker", "buildx", "imagetools", "inspect", "--raw", reference], timeout=60).stdout - require("sha256:" + hashlib.sha256(raw.encode()).hexdigest() == reference.split("@", 1)[1], "native OCI manifest digest mismatch") + require("sha256:" + hashlib.sha256(raw.encode()).hexdigest() == (expected_digest or reference.split("@", 1)[1]), "native OCI manifest digest mismatch") return json.loads(raw) index = manifest(image) @@ -1044,6 +1044,20 @@ def manifest(reference: str) -> dict[str, Any]: require(re.fullmatch(r"sha256:[0-9a-f]{64}", config_digest) is not None, "native OCI config digest missing") # Docker's classic and containerd stores expose different image IDs. require(metadata.get("Id") in {image.split("@", 1)[1], platform_digest, config_digest}, "native local image ID is not bound to its OCI descriptors") + requested = canonical(os.environ["DW_FAILOVER_SERVER_IMAGE_REQUESTED"]) + if "@" in requested: + require(requested == canonical(image), "requested native image digest mismatch") + else: + require(requested == "durableworkflow/server:" + version, "requested native registry tag mismatch") + manifest("durableworkflow/server:" + version, image.split("@", 1)[1]) + tag_ref = "refs/tags/" + tag + tag_commit = command(["git", "rev-parse", "--verify", tag_ref + "^{commit}"], timeout=30).stdout.strip() + require(tag_commit == revision, "checked-out native Git tag commit mismatch") + remote = command(["git", "ls-remote", "--exit-code", "https://github.com/durable-workflow/server.git", tag_ref, tag_ref + "^{}"], timeout=60).stdout.splitlines() + refs = {reference: commit for commit, reference in (line.split() for line in remote)} + require(len(refs) == len(remote) and tag_ref in refs and set(refs) <= {tag_ref, tag_ref + "^{}"}, "current native Git tag missing or ambiguous") + require(all(re.fullmatch(r"[0-9a-f]{40}", commit) is not None for commit in refs.values()), "current native Git tag object invalid") + require(refs.get(tag_ref + "^{}", refs[tag_ref]) == revision, "current native Git tag commit mismatch") verified = command([ os.environ.get("GH", "gh"), "attestation", "verify", "oci://" + image, "--repo", "durable-workflow/server", @@ -1073,7 +1087,8 @@ def manifest(reference: str) -> dict[str, Any]: "local_image_id": metadata["Id"], "platform": metadata["Os"] + "/" + metadata["Architecture"], "source_repository": labels["org.opencontainers.image.source"], "source_revision": revision, - "release_tag": tag, "release_version": version, "release_run_id": run_id, + "release_tag": tag, "registry_tag": version, "release_version": version, + "git_tag_commit": tag_commit, "git_tag_object": refs[tag_ref], "release_run_id": run_id, "attestation_invocations": sorted(set(invocations)), }} @@ -1085,7 +1100,27 @@ def inspect_published_server() -> dict[str, Any]: metadata = inspected[0] config = metadata.get("Config") or {} labels = config.get("Labels") or {} - if config.get("Entrypoint") == ["/usr/local/bin/durable-workflow-server"] or labels.get("org.opencontainers.image.title") == "Durable Workflow Server (native)": + native = ( + config.get("Entrypoint") == ["/usr/local/bin/durable-workflow-server"] + and labels.get("org.opencontainers.image.title") == "Durable Workflow Server (native)" + and "dev.durable-workflow.package.authority" not in labels + and not any(key.startswith("dev.durable-workflow.workflow.") for key in labels) + ) + php = ( + config.get("Entrypoint") == ["server-entrypoint"] + and labels.get("org.opencontainers.image.title") in {"server", "Durable Workflow Server"} + and labels.get("org.opencontainers.image.source") == "https://github.com/durable-workflow/server" + and labels.get("dev.durable-workflow.package.authority") == "composer.lock" + and labels.get("dev.durable-workflow.workflow.package") == "durable-workflow/workflow" + and labels.get("dev.durable-workflow.workflow.source") == "https://github.com/durable-workflow/workflow.git" + and re.fullmatch(r"[0-9a-f]{40}", str(labels.get("dev.durable-workflow.workflow.commit", ""))) is not None + and labels.get("dev.durable-workflow.workflow.commit") != "0" * 40 + and bool(labels.get("dev.durable-workflow.workflow.version")) + and re.fullmatch(r"2\.[0-9]+\.[0-9]+(?:-rc\.[1-9][0-9]*)?", str(labels.get("dev.durable-workflow.release.tag", ""))) is not None + and labels.get("org.opencontainers.image.version") == labels.get("dev.durable-workflow.release.tag") + ) + require(native != php, "unknown or contradictory server implementation identity") + if native: return inspect_native_server(image, metadata) return inspect_php_workflow_package() diff --git a/tests/Unit/Support/single_region_failover_identity_test.py b/tests/Unit/Support/single_region_failover_identity_test.py index af16e0acc..84efe957a 100644 --- a/tests/Unit/Support/single_region_failover_identity_test.py +++ b/tests/Unit/Support/single_region_failover_identity_test.py @@ -27,6 +27,7 @@ class PublishedIdentityTest(unittest.TestCase): def setUp(self) -> None: self.environment = mock.patch.dict(os.environ, { "DW_FAILOVER_SERVER_IMAGE": IMAGE, + "DW_FAILOVER_SERVER_IMAGE_REQUESTED": "durableworkflow/server:3.1.0", "GH": "gh", }) self.environment.start() @@ -34,6 +35,9 @@ def setUp(self) -> None: self.commands = [] self.index_raw = INDEX_RAW self.manifest_raw = MANIFEST_RAW + self.live_raw = INDEX_RAW + self.local_peel = SOURCE + self.remote_refs = "d" * 40 + "\trefs/tags/3.1.0\n" + SOURCE + "\trefs/tags/3.1.0^{}\n" self.metadata = { "Id": CONFIG_DIGEST, "RepoDigests": [IMAGE], "Os": "linux", "Architecture": "amd64", @@ -65,8 +69,12 @@ def command(self, args, **kwargs): if args[:3] == ["docker", "image", "inspect"]: value = [self.metadata] elif args[:4] == ["docker", "buildx", "imagetools", "inspect"]: - value = self.index_raw if args[-1] == IMAGE else self.manifest_raw + value = self.live_raw if ":3.1.0" in args[-1] else self.index_raw if args[-1] == IMAGE else self.manifest_raw return subprocess.CompletedProcess(args, 0, value, "") + elif args[:2] == ["git", "rev-parse"]: + return subprocess.CompletedProcess(args, 0, self.local_peel + "\n", "") + elif args[:2] == ["git", "ls-remote"]: + return subprocess.CompletedProcess(args, 0, self.remote_refs, "") elif args[:3] == ["gh", "attestation", "verify"]: if self.verifier_error: raise self.verifier_error @@ -91,6 +99,8 @@ def test_native_identity_binds_config_source_build_and_verified_subject(self): self.assertEqual(SOURCE, identity["source_revision"]) self.assertEqual("3.1.0", identity["release_version"]) self.assertEqual("12345", identity["release_run_id"]) + self.assertEqual("3.1.0", identity["registry_tag"]) + self.assertEqual(SOURCE, identity["git_tag_commit"]) self.assertEqual([self.certificate["runInvocationURI"]], identity["attestation_invocations"]) self.assertFalse(any(args[:2] == ["docker", "run"] for args, _ in self.commands)) verifier, options = next((args, options) for args, options in self.commands if args[0] == "gh") @@ -109,14 +119,14 @@ def test_containerd_image_id_is_recorded_separately_from_config_digest(self): self.assertNotEqual(identity["local_image_id"], identity["image_config_digest"]) def test_php_inspection_preserves_the_original_package_record(self): - self.metadata["Config"] = {"Entrypoint": ["docker-php-entrypoint"], "Labels": {}} + self.legacy_php() self.assertEqual({"embedded_workflow_package": self.php_package}, self.inspect()) self.assertEqual(2, len(self.commands)) self.assertIn("/app/composer.lock", self.commands[1][0][-1]) self.assertFalse(any(args[0] == "gh" for args, _ in self.commands)) def test_php_inspection_still_rejects_missing_package_provenance(self): - self.metadata["Config"] = {"Entrypoint": ["docker-php-entrypoint"], "Labels": {}} + self.legacy_php() self.php_package["source_reference"] = None self.php_package["dist_reference"] = None with self.assertRaisesRegex(AssertionError, "no exact source reference"): @@ -126,7 +136,70 @@ def test_failed_signature_verification_never_falls_back_to_php(self): self.verifier_error = subprocess.CalledProcessError(1, ["gh", "attestation", "verify"]) with self.assertRaises(subprocess.CalledProcessError): self.inspect() - self.assertEqual(4, len(self.commands)) + self.assertFalse(any(args[:2] == ["docker", "run"] for args, _ in self.commands)) + + def legacy_php(self): + self.metadata["Config"] = {"Entrypoint": ["server-entrypoint"], "Labels": { + "org.opencontainers.image.title": "server", "org.opencontainers.image.source": REPOSITORY, + "org.opencontainers.image.version": "2.11.8", "dev.durable-workflow.release.tag": "2.11.8", + "dev.durable-workflow.package.authority": "composer.lock", + "dev.durable-workflow.workflow.package": "durable-workflow/workflow", + "dev.durable-workflow.workflow.source": "https://github.com/durable-workflow/workflow.git", + "dev.durable-workflow.workflow.version": "2.15.12", "dev.durable-workflow.workflow.commit": "d" * 40, + }} + + def test_unknown_identity_cannot_fall_back_to_valid_looking_php_package(self): + self.metadata["Config"]["Entrypoint"] = ["php"] + self.metadata["Config"]["Labels"].pop("org.opencontainers.image.title") + with self.assertRaisesRegex(AssertionError, "implementation identity"): + self.inspect() + self.assertEqual(1, len(self.commands)) + + def test_php_discriminators_with_native_release_labels_are_refused_before_execution(self): + self.legacy_php() + self.metadata["Config"]["Labels"].update({"dev.durable-workflow.release.tag": "3.1.0", "org.opencontainers.image.version": "3.1.0"}) + with self.assertRaisesRegex(AssertionError, "implementation identity"): + self.inspect() + self.assertEqual(1, len(self.commands)) + + def test_incomplete_or_mixed_php_identity_never_launches_package_inspector(self): + fields = ["org.opencontainers.image.title", "org.opencontainers.image.source", + "org.opencontainers.image.version", "dev.durable-workflow.release.tag", + "dev.durable-workflow.package.authority", "dev.durable-workflow.workflow.package", + "dev.durable-workflow.workflow.source", "dev.durable-workflow.workflow.commit", + "dev.durable-workflow.workflow.version"] + for field in fields: + with self.subTest(missing=field): + self.legacy_php() + self.commands.clear() + self.metadata["Config"]["Labels"].pop(field) + with self.assertRaisesRegex(AssertionError, "implementation identity"): + self.inspect() + self.assertEqual(1, len(self.commands)) + for entrypoint, title in [(["/usr/local/bin/durable-workflow-server"], "server"), + (["server-entrypoint"], "Durable Workflow Server (native)")]: + with self.subTest(entrypoint=entrypoint, title=title): + self.legacy_php() + self.commands.clear() + self.metadata["Config"]["Entrypoint"] = entrypoint + self.metadata["Config"]["Labels"]["org.opencontainers.image.title"] = title + with self.assertRaisesRegex(AssertionError, "implementation identity"): + self.inspect() + self.assertEqual(1, len(self.commands)) + + def test_historical_native_git_tag_normalizes_to_registry_version(self): + self.metadata["Config"]["Labels"]["dev.durable-workflow.release.tag"] = "native-3.1.0" + self.certificate["sourceRepositoryRef"] = "refs/tags/native-3.1.0" + self.certificate["buildSignerURI"] = WORKFLOW + "@refs/tags/native-3.1.0" + self.remote_refs = self.remote_refs.replace("refs/tags/3.1.0", "refs/tags/native-3.1.0") + identity = self.inspect()["native_server_identity"] + self.assertEqual("native-3.1.0", identity["release_tag"]) + self.assertEqual("3.1.0", identity["registry_tag"]) + + def test_exact_digest_request_still_requires_the_live_version_tag(self): + os.environ["DW_FAILOVER_SERVER_IMAGE_REQUESTED"] = IMAGE + self.assertEqual("3.1.0", self.inspect()["native_server_identity"]["registry_tag"]) + self.assertTrue(any(args[-1] == "durableworkflow/server:3.1.0" for args, _ in self.commands)) def test_native_identity_phase_does_not_pass_recovery_cells(self): services = {name: {"image": IMAGE} for name in ["bootstrap", "server-a", "server-b", @@ -169,6 +242,16 @@ def test(self): ("rolling_release_tag", lambda t: t.metadata["Config"]["Labels"].update({"dev.durable-workflow.release.tag": "latest"})), ("php_version_as_native", lambda t: t.metadata["Config"]["Labels"].update({"dev.durable-workflow.release.tag": "2.11.8"})), ("missing_build_run", lambda t: t.metadata["Config"]["Labels"].pop("dev.durable-workflow.release.run-id")), + ("native_with_php_authority", lambda t: t.metadata["Config"]["Labels"].update({"dev.durable-workflow.package.authority": "composer.lock"})), + ("native_with_php_package", lambda t: t.metadata["Config"]["Labels"].update({"dev.durable-workflow.workflow.package": "durable-workflow/workflow"})), + ("wrong_requested_tag", lambda t: os.environ.update(DW_FAILOVER_SERVER_IMAGE_REQUESTED="durableworkflow/server:3.2.0")), + ("wrong_requested_digest", lambda t: os.environ.update(DW_FAILOVER_SERVER_IMAGE_REQUESTED="durableworkflow/server@sha256:" + "e" * 64)), + ("moved_registry_tag", lambda t: setattr(t, "live_raw", t.live_raw + " ")), + ("missing_local_git_tag", lambda t: setattr(t, "local_peel", "")), + ("moved_local_git_tag", lambda t: setattr(t, "local_peel", "e" * 40)), + ("missing_remote_git_tag", lambda t: setattr(t, "remote_refs", "")), + ("moved_remote_git_tag", lambda t: setattr(t, "remote_refs", "e" * 40 + "\trefs/tags/3.1.0\n")), + ("duplicate_remote_git_tag", lambda t: setattr(t, "remote_refs", t.remote_refs + t.remote_refs)), ("empty_attestations", lambda t: t.attestations.clear()), ("different_signed_subject", lambda t: t.attestations[0]["verificationResult"]["statement"]["subject"][0]["digest"].update(sha256="e" * 64)), ("different_signed_repository", lambda t: t.certificate.update(sourceRepositoryURI="https://example.invalid")), From 3848ebc95de3bbb3bf4bcf180423ac83b040e80f Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:59:19 -0600 Subject: [PATCH 3/7] Adopt the required locked central-policy bootstrap --- .github/workflows/phpunit-feature.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index e92cfb3e6..ea2ff8def 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -49,9 +49,9 @@ jobs: with: python-version: "3.13" - name: Bootstrap patched pip trust - uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@75587fc33ff40c34f32ef777293dad56bb6f48c7 # clean-trust + uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@fea245682582d0620054e8547a5f3c968f9ef0f0 # clean-trust - name: Install policy dependencies - run: python -m pip install PyYAML==6.0.2 + run: python -I -m pip --isolated install --require-hashes --only-binary=:all: --no-deps -r .central-action-policy/qualification/requirements.lock - name: Require centrally approved immutable action commits run: >- python .central-action-policy/scripts/qualification_policy.py validate From 8f64bd8cad44fc6f06c2841acd534435e32e85e6 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:01:46 -0600 Subject: [PATCH 4/7] Quote the locked pip command through folded YAML --- .github/workflows/phpunit-feature.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index ea2ff8def..62d6ba8d9 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -51,7 +51,9 @@ jobs: - name: Bootstrap patched pip trust uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@fea245682582d0620054e8547a5f3c968f9ef0f0 # clean-trust - name: Install policy dependencies - run: python -I -m pip --isolated install --require-hashes --only-binary=:all: --no-deps -r .central-action-policy/qualification/requirements.lock + run: >- + python -I -m pip --isolated install --require-hashes --only-binary=:all: + --no-deps -r .central-action-policy/qualification/requirements.lock - name: Require centrally approved immutable action commits run: >- python .central-action-policy/scripts/qualification_policy.py validate From a88152ba405b6ea17d377987b116d7d7bd8e75db Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:18:45 -0600 Subject: [PATCH 5/7] Bind requested PHP tags and digests before package inspection --- docs/ha-failover-validation.md | 8 +++-- ...gle-region-failover-published-artifacts.py | 3 ++ .../single_region_failover_identity_test.py | 30 +++++++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/docs/ha-failover-validation.md b/docs/ha-failover-validation.md index e8c9a9e03..f42cd071d 100644 --- a/docs/ha-failover-validation.md +++ b/docs/ha-failover-validation.md @@ -295,14 +295,16 @@ verified release tag; serving-version and recovery observations remain part of the later runtime phases. It neither invokes PHP nor invents Composer package records for a native image. -The requested registry tag must match the verified release version (`native-` -is removed from historical Git tag names). The current registry tag must still +For native images, the requested registry tag must match the verified release +version (`native-` is removed from historical Git tag names). The current registry tag must still resolve to the inspected index. The checked-out and current remote Git tag must both peel to the recorded source commit. Unknown or mixed implementation identities are refused before any in-image inspection. PHP images keep the existing `embedded_workflow_package` version/reference -inspection. Passing either artifact inspector supplies only the provenance +inspection. Their requested digest or exact tag must match the recognized PHP +image identity before the package inspector runs. Passing either artifact +inspector supplies only the provenance phase: every topology and recovery cell is still required before the complete rehearsal can pass. An inspector-only check does not qualify native failover. diff --git a/scripts/conformance/single-region-failover-published-artifacts.py b/scripts/conformance/single-region-failover-published-artifacts.py index 833a8a928..00add54dd 100755 --- a/scripts/conformance/single-region-failover-published-artifacts.py +++ b/scripts/conformance/single-region-failover-published-artifacts.py @@ -1100,6 +1100,8 @@ def inspect_published_server() -> dict[str, Any]: metadata = inspected[0] config = metadata.get("Config") or {} labels = config.get("Labels") or {} + requested = re.sub(r"^(?:docker\.io/|index\.docker\.io/)", "", os.environ["DW_FAILOVER_SERVER_IMAGE_REQUESTED"]) + resolved = re.sub(r"^(?:docker\.io/|index\.docker\.io/)", "", image) native = ( config.get("Entrypoint") == ["/usr/local/bin/durable-workflow-server"] and labels.get("org.opencontainers.image.title") == "Durable Workflow Server (native)" @@ -1118,6 +1120,7 @@ def inspect_published_server() -> dict[str, Any]: and bool(labels.get("dev.durable-workflow.workflow.version")) and re.fullmatch(r"2\.[0-9]+\.[0-9]+(?:-rc\.[1-9][0-9]*)?", str(labels.get("dev.durable-workflow.release.tag", ""))) is not None and labels.get("org.opencontainers.image.version") == labels.get("dev.durable-workflow.release.tag") + and requested == (resolved if "@" in requested else "durableworkflow/server:" + str(labels.get("dev.durable-workflow.release.tag"))) ) require(native != php, "unknown or contradictory server implementation identity") if native: diff --git a/tests/Unit/Support/single_region_failover_identity_test.py b/tests/Unit/Support/single_region_failover_identity_test.py index 84efe957a..e78b83b66 100644 --- a/tests/Unit/Support/single_region_failover_identity_test.py +++ b/tests/Unit/Support/single_region_failover_identity_test.py @@ -139,6 +139,7 @@ def test_failed_signature_verification_never_falls_back_to_php(self): self.assertFalse(any(args[:2] == ["docker", "run"] for args, _ in self.commands)) def legacy_php(self): + os.environ["DW_FAILOVER_SERVER_IMAGE_REQUESTED"] = IMAGE self.metadata["Config"] = {"Entrypoint": ["server-entrypoint"], "Labels": { "org.opencontainers.image.title": "server", "org.opencontainers.image.source": REPOSITORY, "org.opencontainers.image.version": "2.11.8", "dev.durable-workflow.release.tag": "2.11.8", @@ -162,6 +163,35 @@ def test_php_discriminators_with_native_release_labels_are_refused_before_execut self.inspect() self.assertEqual(1, len(self.commands)) + def test_native_tag_request_cannot_select_an_otherwise_recognized_php_image(self): + self.legacy_php() + os.environ["DW_FAILOVER_SERVER_IMAGE_REQUESTED"] = "durableworkflow/server:3.1.0" + with self.assertRaisesRegex(AssertionError, "implementation identity"): + self.inspect() + self.assertEqual(1, len(self.commands)) + + def test_php_requested_tag_and_digest_accept_public_repository_aliases(self): + for prefix in ("", "docker.io/", "index.docker.io/"): + for reference in ("durableworkflow/server:2.11.8", IMAGE): + with self.subTest(prefix=prefix, reference=reference): + self.legacy_php() + os.environ["DW_FAILOVER_SERVER_IMAGE_REQUESTED"] = prefix + reference + self.assertEqual({"embedded_workflow_package": self.php_package}, self.inspect()) + + def test_php_wrong_requested_tag_is_refused_before_execution(self): + self.legacy_php() + os.environ["DW_FAILOVER_SERVER_IMAGE_REQUESTED"] = "durableworkflow/server:2.11.7" + with self.assertRaisesRegex(AssertionError, "implementation identity"): + self.inspect() + self.assertEqual(1, len(self.commands)) + + def test_php_wrong_requested_digest_is_refused_before_execution(self): + self.legacy_php() + os.environ["DW_FAILOVER_SERVER_IMAGE_REQUESTED"] = "docker.io/durableworkflow/server@sha256:" + "e" * 64 + with self.assertRaisesRegex(AssertionError, "implementation identity"): + self.inspect() + self.assertEqual(1, len(self.commands)) + def test_incomplete_or_mixed_php_identity_never_launches_package_inspector(self): fields = ["org.opencontainers.image.title", "org.opencontainers.image.source", "org.opencontainers.image.version", "dev.durable-workflow.release.tag", From 3d0411bba7725ddd2d01ef4cfef9a8e66d90d43a Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:18:46 -0600 Subject: [PATCH 6/7] Own the hash-locked pip sink and prove its Linux root confinement --- tests/Unit/Support/patched_pip_test.py | 68 ++++++++++++++++++++++++-- tests/Unit/Support/pip-sinks.json | 21 +++----- 2 files changed, 70 insertions(+), 19 deletions(-) diff --git a/tests/Unit/Support/patched_pip_test.py b/tests/Unit/Support/patched_pip_test.py index e2f70b01f..ac16ae365 100644 --- a/tests/Unit/Support/patched_pip_test.py +++ b/tests/Unit/Support/patched_pip_test.py @@ -73,10 +73,18 @@ # The bootstrap's own install of the hash-pinned wheel, in the same command. "pinned-wheel": sink_guards.GuardType(re.compile(r"#sha256=[0-9a-f]{64}"), 1), } -CENTRAL_ACTION = "durable-workflow/.github/.github/actions/bootstrap-patched-pip@75587fc33ff40c34f32ef777293dad56bb6f48c7" +CENTRAL_ACTIONS = { + "durable-workflow/.github/.github/actions/bootstrap-patched-pip@75587fc33ff40c34f32ef777293dad56bb6f48c7", + "durable-workflow/.github/.github/actions/bootstrap-patched-pip@fea245682582d0620054e8547a5f3c968f9ef0f0", +} +LOCKED_POLICY_ACTION = "uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@fea245682582d0620054e8547a5f3c968f9ef0f0 # clean-trust" +LOCKED_POLICY_COMMAND = ( + "python -I -m pip --isolated install --require-hashes --only-binary=:all:\n" + " --no-deps -r .central-action-policy/qualification/requirements.lock" +) SELF_CHECKS = { # The Security-accepted central action is itself the bootstrap. - "accepted-action": lambda occurrence: occurrence.line == f"uses: {CENTRAL_ACTION} # clean-trust", + "accepted-action": lambda occurrence: any(occurrence.line == f"uses: {action} # clean-trust" for action in CENTRAL_ACTIONS), # "pip --version" makes no request. "version-query": lambda occurrence: re.search(r"-m pip --version\b", occurrence.line) is not None, "evidence-record": sink_guards.evidence_record(ROOT), @@ -121,6 +129,24 @@ def unreviewed_sinks(root: Path, entries: list[dict[str, object]]) -> tuple[list return describe(found - reviewed), describe(reviewed - found) +def locked_policy_trust(occurrence: sink_guards.Occurrence, source: str) -> bool: + """The Linux policy install uses the action's SSL_CERT_FILE/empty SSL_CERT_DIR. + + Unlike pip's PIP_* variables, these OpenSSL settings survive --isolated. + Keep this alternative tied to the exact reviewed bootstrap and command. + """ + if occurrence.path != ".github/workflows/phpunit-feature.yml": + return False + lines = source.split("\n") + if "\n".join(lines[occurrence.index:occurrence.index + 2]).strip() != LOCKED_POLICY_COMMAND: + return False + job = next((index for index in range(occurrence.index, -1, -1) + if sink_guards.WORKFLOW_JOB.match(lines[index])), None) + if job is None or lines[job] != " action-policy:" or " runs-on: ubuntu-latest" not in lines[job:occurrence.index]: + return False + return sink_guards.find_guard(lines, occurrence, GUARDS["bootstrap"]) == (LOCKED_POLICY_ACTION, 3) + + def make_root(directory: Path, common_name: str) -> str: stem = directory / re.sub(r"\W+", "-", common_name) openssl( @@ -383,6 +409,20 @@ def test_job_environment_confinement_refuses_the_affected_store(self) -> None: PIP_USE_DEPRECATED="legacy-certs", )) + @unittest.skipUnless(sys.platform.startswith("linux"), "the reviewed policy preflight runs on Linux") + def test_isolated_policy_install_keeps_the_bootstrap_openssl_roots(self) -> None: + config = self.venv / "pip.conf" + saved = config.read_bytes() + config.unlink() + try: + result, paths = self.fetch("--isolated") + self.assertEqual(["/simple/dw-trust-control/"], paths, result.stdout + result.stderr) + self.assertRefused(*self.fetch( + "--isolated", SSL_CERT_FILE=self.pip_bundle, SSL_CERT_DIR=str(self.empty_store_dir), + )) + finally: + config.write_bytes(saved) + def test_patched_pip_wrapper_keeps_pip_status_and_removes_its_venv(self) -> None: temporary = self.work / "wrapper-tmp" for arguments, expected in ((["--version"], 0), (["install", "--no-index", "dw-trust-missing"], 1)): @@ -552,8 +592,9 @@ def test_unconfined_sinks_fail_the_review(self) -> None: self.assertTrue(any(item.startswith(f"{relative}: ") for item in unreviewed), unreviewed) def test_isolated_pip_commands_keep_legacy_certs(self) -> None: - # --isolated ignores the venv's pip.conf, so the command itself must - # keep pip off the operating system store. + # Isolated commands must keep pip off the operating system store, or + # use the reviewed Linux + # bootstrap's OpenSSL root configuration for the exact policy install. loose = [] for directory in SCANNED: for path in sorted((ROOT / directory).rglob("*")): @@ -565,11 +606,28 @@ def test_isolated_pip_commands_keep_legacy_certs(self) -> None: continue for command in re.finditer(rf"\bpip{SEPARATOR}--isolated\b", text): following = text[command.start():command.start() + 400] - if "legacy-certs" not in following: + index = text.count("\n", 0, command.start()) + line = text.split("\n")[index] + occurrence = sink_guards.Occurrence(path.relative_to(ROOT).as_posix(), line.strip(), index, + command.start() - (text.rfind("\n", 0, command.start()) + 1), command.group(0)) + if "legacy-certs" not in following and not locked_policy_trust(occurrence, text): loose.append(f"{path.relative_to(ROOT).as_posix()}:{text.count(chr(10), 0, command.start()) + 1}") self.assertEqual([], loose) + def test_locked_policy_trust_requires_exact_bootstrap_command_and_linux_job(self) -> None: + path = ".github/workflows/phpunit-feature.yml" + source = (ROOT / path).read_text() + occurrence = next(item for item in sink_guards.occurrences(ROOT, [ROOT / path], PIP_SINK) + if item.line.startswith("python -I -m pip --isolated")) + self.assertTrue(locked_policy_trust(occurrence, source)) + for before, after in [("fea245682582d0620054e8547a5f3c968f9ef0f0", "0" * 40), + ("--require-hashes", "--no-cache-dir"), + ("qualification/requirements.lock", "qualification/other.lock"), + ("runs-on: ubuntu-latest", "runs-on: windows-latest")]: + with self.subTest(mutation=before): + self.assertFalse(locked_policy_trust(occurrence, source.replace(before, after))) + if __name__ == "__main__": unittest.main() diff --git a/tests/Unit/Support/pip-sinks.json b/tests/Unit/Support/pip-sinks.json index 6ab5d54c3..ad6eca3fd 100644 --- a/tests/Unit/Support/pip-sinks.json +++ b/tests/Unit/Support/pip-sinks.json @@ -49,22 +49,22 @@ }, { "path": ".github/workflows/phpunit-feature.yml", - "line": "run: python -m pip install PyYAML==6.0.2", + "line": "python -I -m pip --isolated install --require-hashes --only-binary=:all:", "count": 1, - "confinement": "The clean-trust action at the preflight's reviewed step exports the checked bundle as PIP_CERT and SSL_CERT_FILE with an empty SSL_CERT_DIR, so pip and its truststore see only that bundle.", + "confinement": "The reviewed Linux clean-trust bootstrap exports checked SSL_CERT_FILE and an empty SSL_CERT_DIR before the isolated, hash-locked, wheel-only policy install. These OpenSSL settings survive pip --isolated. Python -I prevents checked-out modules from replacing pip; the protected central requirements lock supplies dependency hashes.", "guards": [ { "type": "bootstrap", - "line": "uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@75587fc33ff40c34f32ef777293dad56bb6f48c7 # clean-trust", - "distance": 2 + "line": "uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@fea245682582d0620054e8547a5f3c968f9ef0f0 # clean-trust", + "distance": 3 } ] }, { "path": ".github/workflows/phpunit-feature.yml", - "line": "uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@75587fc33ff40c34f32ef777293dad56bb6f48c7 # clean-trust", + "line": "uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@fea245682582d0620054e8547a5f3c968f9ef0f0 # clean-trust", "count": 1, - "confinement": "The central clean-trust action (Security-accepted .github #239): installs the hash-pinned pip 26.2.2, checks its bundle and exports it, with an empty SSL_CERT_DIR, as PIP_CERT and SSL_CERT_FILE to later steps. At the policy's reviewed preflight step.", + "confinement": "The currently required central clean-trust bootstrap installs hash-pinned pip 26.2.2, checks and exports its CA bundle with an empty SSL_CERT_DIR. Its exact immutable revision is also enforced by central qualification policy.", "guards": [ { "type": "accepted-action", @@ -518,12 +518,5 @@ ] } ], - "isolation_exceptions": [ - { - "path": ".github/workflows/phpunit-feature.yml", - "line": "run: python -m pip install PyYAML==6.0.2", - "owner": "durable-workflow/.github central qualification policy", - "reason": "The central policy admits only this exact action-policy preflight install. General is asked (cloud#438 19131108) to admit python -I -m pip; adopt it here when it does." - } - ] + "isolation_exceptions": [] } From 0bd123d22328d05cc3d4d9a127a637fa0baec719 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:48:19 -0600 Subject: [PATCH 7/7] Restore the supported central-policy bootstrap baseline --- .github/workflows/phpunit-feature.yml | 6 +-- tests/Unit/Support/patched_pip_test.py | 68 ++------------------------ tests/Unit/Support/pip-sinks.json | 21 +++++--- 3 files changed, 21 insertions(+), 74 deletions(-) diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index 62d6ba8d9..e92cfb3e6 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -49,11 +49,9 @@ jobs: with: python-version: "3.13" - name: Bootstrap patched pip trust - uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@fea245682582d0620054e8547a5f3c968f9ef0f0 # clean-trust + uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@75587fc33ff40c34f32ef777293dad56bb6f48c7 # clean-trust - name: Install policy dependencies - run: >- - python -I -m pip --isolated install --require-hashes --only-binary=:all: - --no-deps -r .central-action-policy/qualification/requirements.lock + run: python -m pip install PyYAML==6.0.2 - name: Require centrally approved immutable action commits run: >- python .central-action-policy/scripts/qualification_policy.py validate diff --git a/tests/Unit/Support/patched_pip_test.py b/tests/Unit/Support/patched_pip_test.py index ac16ae365..e2f70b01f 100644 --- a/tests/Unit/Support/patched_pip_test.py +++ b/tests/Unit/Support/patched_pip_test.py @@ -73,18 +73,10 @@ # The bootstrap's own install of the hash-pinned wheel, in the same command. "pinned-wheel": sink_guards.GuardType(re.compile(r"#sha256=[0-9a-f]{64}"), 1), } -CENTRAL_ACTIONS = { - "durable-workflow/.github/.github/actions/bootstrap-patched-pip@75587fc33ff40c34f32ef777293dad56bb6f48c7", - "durable-workflow/.github/.github/actions/bootstrap-patched-pip@fea245682582d0620054e8547a5f3c968f9ef0f0", -} -LOCKED_POLICY_ACTION = "uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@fea245682582d0620054e8547a5f3c968f9ef0f0 # clean-trust" -LOCKED_POLICY_COMMAND = ( - "python -I -m pip --isolated install --require-hashes --only-binary=:all:\n" - " --no-deps -r .central-action-policy/qualification/requirements.lock" -) +CENTRAL_ACTION = "durable-workflow/.github/.github/actions/bootstrap-patched-pip@75587fc33ff40c34f32ef777293dad56bb6f48c7" SELF_CHECKS = { # The Security-accepted central action is itself the bootstrap. - "accepted-action": lambda occurrence: any(occurrence.line == f"uses: {action} # clean-trust" for action in CENTRAL_ACTIONS), + "accepted-action": lambda occurrence: occurrence.line == f"uses: {CENTRAL_ACTION} # clean-trust", # "pip --version" makes no request. "version-query": lambda occurrence: re.search(r"-m pip --version\b", occurrence.line) is not None, "evidence-record": sink_guards.evidence_record(ROOT), @@ -129,24 +121,6 @@ def unreviewed_sinks(root: Path, entries: list[dict[str, object]]) -> tuple[list return describe(found - reviewed), describe(reviewed - found) -def locked_policy_trust(occurrence: sink_guards.Occurrence, source: str) -> bool: - """The Linux policy install uses the action's SSL_CERT_FILE/empty SSL_CERT_DIR. - - Unlike pip's PIP_* variables, these OpenSSL settings survive --isolated. - Keep this alternative tied to the exact reviewed bootstrap and command. - """ - if occurrence.path != ".github/workflows/phpunit-feature.yml": - return False - lines = source.split("\n") - if "\n".join(lines[occurrence.index:occurrence.index + 2]).strip() != LOCKED_POLICY_COMMAND: - return False - job = next((index for index in range(occurrence.index, -1, -1) - if sink_guards.WORKFLOW_JOB.match(lines[index])), None) - if job is None or lines[job] != " action-policy:" or " runs-on: ubuntu-latest" not in lines[job:occurrence.index]: - return False - return sink_guards.find_guard(lines, occurrence, GUARDS["bootstrap"]) == (LOCKED_POLICY_ACTION, 3) - - def make_root(directory: Path, common_name: str) -> str: stem = directory / re.sub(r"\W+", "-", common_name) openssl( @@ -409,20 +383,6 @@ def test_job_environment_confinement_refuses_the_affected_store(self) -> None: PIP_USE_DEPRECATED="legacy-certs", )) - @unittest.skipUnless(sys.platform.startswith("linux"), "the reviewed policy preflight runs on Linux") - def test_isolated_policy_install_keeps_the_bootstrap_openssl_roots(self) -> None: - config = self.venv / "pip.conf" - saved = config.read_bytes() - config.unlink() - try: - result, paths = self.fetch("--isolated") - self.assertEqual(["/simple/dw-trust-control/"], paths, result.stdout + result.stderr) - self.assertRefused(*self.fetch( - "--isolated", SSL_CERT_FILE=self.pip_bundle, SSL_CERT_DIR=str(self.empty_store_dir), - )) - finally: - config.write_bytes(saved) - def test_patched_pip_wrapper_keeps_pip_status_and_removes_its_venv(self) -> None: temporary = self.work / "wrapper-tmp" for arguments, expected in ((["--version"], 0), (["install", "--no-index", "dw-trust-missing"], 1)): @@ -592,9 +552,8 @@ def test_unconfined_sinks_fail_the_review(self) -> None: self.assertTrue(any(item.startswith(f"{relative}: ") for item in unreviewed), unreviewed) def test_isolated_pip_commands_keep_legacy_certs(self) -> None: - # Isolated commands must keep pip off the operating system store, or - # use the reviewed Linux - # bootstrap's OpenSSL root configuration for the exact policy install. + # --isolated ignores the venv's pip.conf, so the command itself must + # keep pip off the operating system store. loose = [] for directory in SCANNED: for path in sorted((ROOT / directory).rglob("*")): @@ -606,28 +565,11 @@ def test_isolated_pip_commands_keep_legacy_certs(self) -> None: continue for command in re.finditer(rf"\bpip{SEPARATOR}--isolated\b", text): following = text[command.start():command.start() + 400] - index = text.count("\n", 0, command.start()) - line = text.split("\n")[index] - occurrence = sink_guards.Occurrence(path.relative_to(ROOT).as_posix(), line.strip(), index, - command.start() - (text.rfind("\n", 0, command.start()) + 1), command.group(0)) - if "legacy-certs" not in following and not locked_policy_trust(occurrence, text): + if "legacy-certs" not in following: loose.append(f"{path.relative_to(ROOT).as_posix()}:{text.count(chr(10), 0, command.start()) + 1}") self.assertEqual([], loose) - def test_locked_policy_trust_requires_exact_bootstrap_command_and_linux_job(self) -> None: - path = ".github/workflows/phpunit-feature.yml" - source = (ROOT / path).read_text() - occurrence = next(item for item in sink_guards.occurrences(ROOT, [ROOT / path], PIP_SINK) - if item.line.startswith("python -I -m pip --isolated")) - self.assertTrue(locked_policy_trust(occurrence, source)) - for before, after in [("fea245682582d0620054e8547a5f3c968f9ef0f0", "0" * 40), - ("--require-hashes", "--no-cache-dir"), - ("qualification/requirements.lock", "qualification/other.lock"), - ("runs-on: ubuntu-latest", "runs-on: windows-latest")]: - with self.subTest(mutation=before): - self.assertFalse(locked_policy_trust(occurrence, source.replace(before, after))) - if __name__ == "__main__": unittest.main() diff --git a/tests/Unit/Support/pip-sinks.json b/tests/Unit/Support/pip-sinks.json index ad6eca3fd..6ab5d54c3 100644 --- a/tests/Unit/Support/pip-sinks.json +++ b/tests/Unit/Support/pip-sinks.json @@ -49,22 +49,22 @@ }, { "path": ".github/workflows/phpunit-feature.yml", - "line": "python -I -m pip --isolated install --require-hashes --only-binary=:all:", + "line": "run: python -m pip install PyYAML==6.0.2", "count": 1, - "confinement": "The reviewed Linux clean-trust bootstrap exports checked SSL_CERT_FILE and an empty SSL_CERT_DIR before the isolated, hash-locked, wheel-only policy install. These OpenSSL settings survive pip --isolated. Python -I prevents checked-out modules from replacing pip; the protected central requirements lock supplies dependency hashes.", + "confinement": "The clean-trust action at the preflight's reviewed step exports the checked bundle as PIP_CERT and SSL_CERT_FILE with an empty SSL_CERT_DIR, so pip and its truststore see only that bundle.", "guards": [ { "type": "bootstrap", - "line": "uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@fea245682582d0620054e8547a5f3c968f9ef0f0 # clean-trust", - "distance": 3 + "line": "uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@75587fc33ff40c34f32ef777293dad56bb6f48c7 # clean-trust", + "distance": 2 } ] }, { "path": ".github/workflows/phpunit-feature.yml", - "line": "uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@fea245682582d0620054e8547a5f3c968f9ef0f0 # clean-trust", + "line": "uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@75587fc33ff40c34f32ef777293dad56bb6f48c7 # clean-trust", "count": 1, - "confinement": "The currently required central clean-trust bootstrap installs hash-pinned pip 26.2.2, checks and exports its CA bundle with an empty SSL_CERT_DIR. Its exact immutable revision is also enforced by central qualification policy.", + "confinement": "The central clean-trust action (Security-accepted .github #239): installs the hash-pinned pip 26.2.2, checks its bundle and exports it, with an empty SSL_CERT_DIR, as PIP_CERT and SSL_CERT_FILE to later steps. At the policy's reviewed preflight step.", "guards": [ { "type": "accepted-action", @@ -518,5 +518,12 @@ ] } ], - "isolation_exceptions": [] + "isolation_exceptions": [ + { + "path": ".github/workflows/phpunit-feature.yml", + "line": "run: python -m pip install PyYAML==6.0.2", + "owner": "durable-workflow/.github central qualification policy", + "reason": "The central policy admits only this exact action-policy preflight install. General is asked (cloud#438 19131108) to admit python -I -m pip; adopt it here when it does." + } + ] }