Verify native OCI provenance in failover artifact inspector

.github/workflows/phpunit-feature.yml

@@ -152,6 +152,9 @@ jobs:
       - name: Prove the image installs and checks the fixed Brotli decoder
         run: python3 -m unittest tests/Unit/Support/brotli_trust_test.py
 
+      - name: Check published failover image identity and recovery result gates
+        run: python3 -m unittest discover -s tests/Unit/Support -p 'single_region_failover*test.py'
+
   feature:
     name: PHPUnit feature suite
     needs: action-policy

.github/workflows/single-region-failover.yml

@@ -11,6 +11,7 @@ on:
 
 permissions:
   contents: read
+  attestations: read
 
 concurrency:
   group: single-region-failover-${{ github.event_name }}-${{ github.ref }}
@@ -37,6 +38,7 @@ jobs:
 
       - name: Run bounded published-artifact rehearsal
         env:
+          GH_TOKEN: ${{ github.token }}
           DW_FAILOVER_MODE: bounded
           DW_FAILOVER_PROJECT: single-region-failover-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}
           DW_FAILOVER_SERVER_A_PORT: "0"

docs/ha-failover-validation.md

@@ -285,6 +285,29 @@ DW_SERVER_IMAGE=durableworkflow/server:<released-version> \
   --result-dir ./failover-result
 ```
 
+Native images also require Docker Buildx and a GitHub CLI that supports `gh attestation verify`
+and `--deny-self-hosted-runners`; `GH` can select its executable. The host
+inspector checks the exact OCI index, platform manifest and configuration digest, native entrypoint,
+source/release/build labels, and signed provenance for that exact image,
+repository, workflow, tag, source commit and build run. It records
+`artifacts.native_server_identity`. Its `release_version` comes from the
+verified release tag; serving-version and recovery observations remain part of
+the later runtime phases. It neither invokes PHP nor invents Composer package
+records for a native image.
+
+For native images, the requested registry tag must match the verified release
+version (`native-` is removed from historical Git tag names). The current registry tag must still
+resolve to the inspected index. The checked-out and current remote Git tag must
+both peel to the recorded source commit. Unknown or mixed implementation
+identities are refused before any in-image inspection.
+
+PHP images keep the existing `embedded_workflow_package` version/reference
+inspection. Their requested digest or exact tag must match the recognized PHP
+image identity before the package inspector runs. Passing either artifact
+inspector supplies only the provenance
+phase: every topology and recovery cell is still required before the complete
+rehearsal can pass. An inspector-only check does not qualify native failover.
+
 The runner connects to all three published endpoints through `127.0.0.1` by
 default. When the runner itself is in a container that controls a Docker daemon
 through a mounted socket, set `DW_FAILOVER_CONNECT_HOST` to the Docker host

scripts/conformance/single-region-failover-published-artifacts.py

@@ -971,21 +971,7 @@ def require_passing_result() -> None:
     require_recovery_bounds(BOUNDS)
 
 
-def provenance_phase() -> dict[str, Any]:
-    require(CONFIGURATION_ERROR is None, CONFIGURATION_ERROR or "invalid runner configuration")
-    raw = compose("config", "--format", "json").stdout
-    config = json.loads(raw)
-    services = config.get("services", {})
-    require(set(services) == {"bootstrap", "server-a", "server-b", "queue-worker", "scheduler", "load-balancer", "mysql", "redis"}, "unexpected Compose topology")
-    require(all("build" not in service for service in services.values()), "Compose build sections are forbidden")
-    for service_name, service in services.items():
-        image = str(service.get("image", ""))
-        require("@sha256:" in image, f"{service_name} is not repository-digest pinned: {image}")
-        for volume in service.get("volumes", []) or []:
-            volume_type = volume.get("type") if isinstance(volume, dict) else "volume"
-            require(volume_type != "bind", f"bind mount is forbidden for {service_name}: {volume}")
-    require(sum(1 for name in services if name == "scheduler") == 1, "exactly one scheduler service is required")
-    require(sum(1 for name in services if name == "queue-worker") == 1, "exactly one queue worker service is required")
+def inspect_php_workflow_package() -> dict[str, Any]:
     package_script = r'''
 $lock = json_decode(file_get_contents('/app/composer.lock'), true);
 $package = null;
@@ -1018,7 +1004,147 @@ def provenance_phase() -> dict[str, Any]:
         bool(workflow_package.get("source_reference") or workflow_package.get("dist_reference") or workflow_package.get("image_package_provenance")),
         "embedded workflow package has no exact source reference",
     )
-    RESULT["artifacts"]["embedded_workflow_package"] = workflow_package
+    return {"embedded_workflow_package": workflow_package}
+
+
+def inspect_native_server(image: str, metadata: dict[str, Any]) -> dict[str, Any]:
+    config = metadata.get("Config") or {}
+    labels = config.get("Labels") or {}
+    require(config.get("Entrypoint") == ["/usr/local/bin/durable-workflow-server"], "native image entrypoint mismatch")
+    require(metadata.get("Os") == "linux" and metadata.get("Architecture") in {"amd64", "arm64"}, "unsupported native image platform")
+    require(re.fullmatch(r"(?:docker\.io/|index\.docker\.io/)?durableworkflow/server@sha256:[0-9a-f]{64}", image) is not None, "native image must be an exact public repository digest")
+    canonical = lambda value: re.sub(r"^(?:docker\.io/|index\.docker\.io/)", "", value)
+    require(canonical(image) in {canonical(value) for value in metadata.get("RepoDigests", [])}, "native image digest is not bound to inspected config")
+    require(labels.get("org.opencontainers.image.source") == "https://github.com/durable-workflow/server", "native image source repository mismatch")
+    require(labels.get("org.opencontainers.image.title") == "Durable Workflow Server (native)", "native image title mismatch")
+    revision = labels.get("org.opencontainers.image.revision", "")
+    tag = labels.get("dev.durable-workflow.release.tag", "")
+    run_id = labels.get("dev.durable-workflow.release.run-id", "")
+    require(re.fullmatch(r"[0-9a-f]{40}", revision) is not None and revision != "0" * 40, "native image source revision missing")
+    require(re.fullmatch(r"(?:native-)?(?:[3-9]|[1-9][0-9]+)\.[0-9]+\.[0-9]+(?:-rc\.[1-9][0-9]*)?", tag) is not None, "native image release tag invalid")
+    require(re.fullmatch(r"[1-9][0-9]*", run_id) is not None, "native image release run missing")
+    version = tag.removeprefix("native-")
+
+    def manifest(reference: str, expected_digest: str | None = None) -> dict[str, Any]:
+        raw = command(["docker", "buildx", "imagetools", "inspect", "--raw", reference], timeout=60).stdout
+        require("sha256:" + hashlib.sha256(raw.encode()).hexdigest() == (expected_digest or reference.split("@", 1)[1]), "native OCI manifest digest mismatch")
+        return json.loads(raw)
+
+    index = manifest(image)
+    require(index.get("mediaType") == "application/vnd.oci.image.index.v1+json", "native image must resolve to an OCI index")
+    platforms = [entry for entry in index.get("manifests", [])
+                 if entry.get("platform", {}).get("os") == metadata["Os"]
+                 and entry.get("platform", {}).get("architecture") == metadata["Architecture"]]
+    require(len(platforms) == 1, "native OCI platform selection is ambiguous")
+    platform_digest = platforms[0].get("digest", "")
+    require(re.fullmatch(r"sha256:[0-9a-f]{64}", platform_digest) is not None, "native OCI platform digest missing")
+    platform = manifest(image.split("@", 1)[0] + "@" + platform_digest)
+    require(platform.get("mediaType") == "application/vnd.oci.image.manifest.v1+json", "native OCI platform manifest invalid")
+    config_digest = platform.get("config", {}).get("digest", "")
+    require(re.fullmatch(r"sha256:[0-9a-f]{64}", config_digest) is not None, "native OCI config digest missing")
+    # Docker's classic and containerd stores expose different image IDs.
+    require(metadata.get("Id") in {image.split("@", 1)[1], platform_digest, config_digest}, "native local image ID is not bound to its OCI descriptors")
+    requested = canonical(os.environ["DW_FAILOVER_SERVER_IMAGE_REQUESTED"])
+    if "@" in requested:
+        require(requested == canonical(image), "requested native image digest mismatch")
+    else:
+        require(requested == "durableworkflow/server:" + version, "requested native registry tag mismatch")
+    manifest("durableworkflow/server:" + version, image.split("@", 1)[1])
+    tag_ref = "refs/tags/" + tag
+    tag_commit = command(["git", "rev-parse", "--verify", tag_ref + "^{commit}"], timeout=30).stdout.strip()
+    require(tag_commit == revision, "checked-out native Git tag commit mismatch")
+    remote = command(["git", "ls-remote", "--exit-code", "https://github.com/durable-workflow/server.git", tag_ref, tag_ref + "^{}"], timeout=60).stdout.splitlines()
+    refs = {reference: commit for commit, reference in (line.split() for line in remote)}
+    require(len(refs) == len(remote) and tag_ref in refs and set(refs) <= {tag_ref, tag_ref + "^{}"}, "current native Git tag missing or ambiguous")
+    require(all(re.fullmatch(r"[0-9a-f]{40}", commit) is not None for commit in refs.values()), "current native Git tag object invalid")
+    require(refs.get(tag_ref + "^{}", refs[tag_ref]) == revision, "current native Git tag commit mismatch")
+    verified = command([
+        os.environ.get("GH", "gh"), "attestation", "verify", "oci://" + image,
+        "--repo", "durable-workflow/server",
+        "--signer-workflow", "github.com/durable-workflow/server/.github/workflows/native-release.yml",
+        "--source-ref", "refs/tags/" + tag, "--source-digest", revision,
+        "--deny-self-hosted-runners", "--format", "json",
+    ], timeout=60)
+    attestations = json.loads(verified.stdout)
+    require(isinstance(attestations, list) and bool(attestations), "native image has no verified attestation")
+    invocations = []
+    for attestation in attestations:
+        verification = attestation.get("verificationResult", {})
+        subjects = verification.get("statement", {}).get("subject", [])
+        require(any(subject.get("digest", {}).get("sha256") == image.split("@sha256:", 1)[1] for subject in subjects), "native attestation subject mismatch")
+        certificate = verification.get("signature", {}).get("certificate", {})
+        require(certificate.get("sourceRepositoryURI") == "https://github.com/durable-workflow/server", "native attestation repository mismatch")
+        require(certificate.get("sourceRepositoryDigest") == revision, "native attestation source mismatch")
+        require(certificate.get("sourceRepositoryRef") == "refs/tags/" + tag, "native attestation tag mismatch")
+        require(certificate.get("runnerEnvironment") == "github-hosted", "native attestation runner mismatch")
+        require(certificate.get("buildSignerURI") == "https://github.com/durable-workflow/server/.github/workflows/native-release.yml@refs/tags/" + tag, "native attestation signer mismatch")
+        invocation = certificate.get("runInvocationURI", "")
+        require(re.fullmatch(r"https://github\.com/durable-workflow/server/actions/runs/" + run_id + r"/attempts/[1-9][0-9]*", invocation) is not None, "native attestation release run mismatch")
+        invocations.append(invocation)
+    return {"native_server_identity": {
+        "runtime": "native", "image_repo_digest": image,
+        "image_manifest_digest": platform_digest, "image_config_digest": config_digest,
+        "local_image_id": metadata["Id"],
+        "platform": metadata["Os"] + "/" + metadata["Architecture"],
+        "source_repository": labels["org.opencontainers.image.source"], "source_revision": revision,
+        "release_tag": tag, "registry_tag": version, "release_version": version,
+        "git_tag_commit": tag_commit, "git_tag_object": refs[tag_ref], "release_run_id": run_id,
+        "attestation_invocations": sorted(set(invocations)),
+    }}
+
+
+def inspect_published_server() -> dict[str, Any]:
+    image = os.environ["DW_FAILOVER_SERVER_IMAGE"]
+    inspected = json.loads(command(["docker", "image", "inspect", image], timeout=30).stdout)
+    require(isinstance(inspected, list) and len(inspected) == 1, "server image inspection is ambiguous")
+    metadata = inspected[0]
+    config = metadata.get("Config") or {}
+    labels = config.get("Labels") or {}
+    requested = re.sub(r"^(?:docker\.io/|index\.docker\.io/)", "", os.environ["DW_FAILOVER_SERVER_IMAGE_REQUESTED"])
+    resolved = re.sub(r"^(?:docker\.io/|index\.docker\.io/)", "", image)
+    native = (
+        config.get("Entrypoint") == ["/usr/local/bin/durable-workflow-server"]
+        and labels.get("org.opencontainers.image.title") == "Durable Workflow Server (native)"
+        and "dev.durable-workflow.package.authority" not in labels
+        and not any(key.startswith("dev.durable-workflow.workflow.") for key in labels)
+    )
+    php = (
+        config.get("Entrypoint") == ["server-entrypoint"]
+        and labels.get("org.opencontainers.image.title") in {"server", "Durable Workflow Server"}
+        and labels.get("org.opencontainers.image.source") == "https://github.com/durable-workflow/server"
+        and labels.get("dev.durable-workflow.package.authority") == "composer.lock"
+        and labels.get("dev.durable-workflow.workflow.package") == "durable-workflow/workflow"
+        and labels.get("dev.durable-workflow.workflow.source") == "https://github.com/durable-workflow/workflow.git"
+        and re.fullmatch(r"[0-9a-f]{40}", str(labels.get("dev.durable-workflow.workflow.commit", ""))) is not None
+        and labels.get("dev.durable-workflow.workflow.commit") != "0" * 40
+        and bool(labels.get("dev.durable-workflow.workflow.version"))
+        and re.fullmatch(r"2\.[0-9]+\.[0-9]+(?:-rc\.[1-9][0-9]*)?", str(labels.get("dev.durable-workflow.release.tag", ""))) is not None
+        and labels.get("org.opencontainers.image.version") == labels.get("dev.durable-workflow.release.tag")
+        and requested == (resolved if "@" in requested else "durableworkflow/server:" + str(labels.get("dev.durable-workflow.release.tag")))
+    )
+    require(native != php, "unknown or contradictory server implementation identity")
+    if native:
+        return inspect_native_server(image, metadata)
+    return inspect_php_workflow_package()
+
+
+def provenance_phase() -> dict[str, Any]:
+    require(CONFIGURATION_ERROR is None, CONFIGURATION_ERROR or "invalid runner configuration")
+    raw = compose("config", "--format", "json").stdout
+    config = json.loads(raw)
+    services = config.get("services", {})
+    require(set(services) == {"bootstrap", "server-a", "server-b", "queue-worker", "scheduler", "load-balancer", "mysql", "redis"}, "unexpected Compose topology")
+    require(all("build" not in service for service in services.values()), "Compose build sections are forbidden")
+    for service_name, service in services.items():
+        image = str(service.get("image", ""))
+        require("@sha256:" in image, f"{service_name} is not repository-digest pinned: {image}")
+        for volume in service.get("volumes", []) or []:
+            volume_type = volume.get("type") if isinstance(volume, dict) else "volume"
+            require(volume_type != "bind", f"bind mount is forbidden for {service_name}: {volume}")
+    require(sum(1 for name in services if name == "scheduler") == 1, "exactly one scheduler service is required")
+    require(sum(1 for name in services if name == "queue-worker") == 1, "exactly one queue worker service is required")
+    identity = inspect_published_server()
+    RESULT["artifacts"].update(identity)
     digest = hashlib.sha256(raw.encode()).hexdigest()
     RESULT["artifacts"]["compose_config_sha256"] = digest
     return {
@@ -1027,7 +1153,7 @@ def provenance_phase() -> dict[str, Any]:
         "all_runtime_images_repo_digest_pinned": True,
         "compose_build_sections_present": False,
         "product_source_bind_mounts_present": False,
-        "embedded_workflow_package": workflow_package,
+        **identity,
     }