diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 799a1e6..d1167c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -512,6 +512,36 @@ jobs: with: targets: wasm32-wasip1 - uses: Swatinem/rust-cache@v2 + # scry#215: before spending several minutes building witness, assert the + # evidence this job produces is made on the same substrate as the artifact + # we ship. Two skews were MEASURED on shipped artifacts, not inferred: + # v3.3.0's signed wasm records `processed-by rustc [1.91.1 ...]` while the + # MC/DC subject floated on @stable, and v3.3.0's evidence manifest records + # `witness_version: 0.30.0` while this gate pins v0.29.0. + # + # The rustc half is now closed (crates/scry-mcdc/rust-toolchain.toml). + # The witness half is a known-open entry in the script's EXPECTED_SKEWS, + # which REPORTS rather than fails — so this can be required today. That + # allowlist is self-cleaning: once the witness versions converge, the + # stale entry FAILS and names itself, so it cannot outlive the defect. + - name: Guard — evidence and artifact share a substrate (scry#215) + run: | + python3 tools/check-evidence-substrate.py --self-test + python3 tools/check-evidence-substrate.py + # The script compares rust-toolchain.toml against MODULE.bazel — + # which is a claim about what WILL compile, not a measurement of what + # does. rustup precedence lets RUSTUP_TOOLCHAIN or a directory + # override beat a toolchain file, and a green gate under the wrong + # rustc looks identical to a green gate under the right one. So + # assert the ACTIVE compiler where the build actually happens, and + # put it in the log so the evidence records its own substrate. + PINNED=$(sed -n 's/^channel = "\(.*\)"/\1/p' crates/scry-mcdc/rust-toolchain.toml) + ACTIVE=$(cd crates/scry-mcdc && rustc --version) + echo "MC/DC subject compiler: $ACTIVE (pinned: $PINNED)" + case "$ACTIVE" in + *"$PINNED"*) ;; + *) echo "::error title=scry#215 substrate::the MC/DC subject is about to be built by '$ACTIVE' but crates/scry-mcdc/rust-toolchain.toml pins $PINNED. A rustup override (RUSTUP_TOOLCHAIN or a directory override) is beating the toolchain file, so the coverage evidence would be produced by a compiler the repo does not declare." >&2; exit 1 ;; + esac # witness lives in a sibling repo (pulseengine/witness); pin the rev so # the coverage numbers are reproducible and the gate is deterministic. # Cache the two built binaries keyed by that rev so we don't rebuild diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bd5598b..612f4d3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -645,6 +645,22 @@ jobs: if ! git worktree add --detach -q "$WT" "$PREV"; then status "SKIPPED — cannot check out $PREV"; exit 0 fi + # scry#215: build BOTH sides of the delta with the same rustc. The + # worktree is the PREVIOUS tag, which predates + # crates/scry-mcdc/rust-toolchain.toml, so without this copy the old + # side compiles under whatever @stable resolves to while the new side + # is pinned — and the delta would attribute compiler-induced + # differences to source change, which is the one thing FEAT-072 + # exists to isolate. Measured locally, witness held constant: rustc + # 1.96.1 -> 1.91.1 alone moved decisions_total 1626 -> 1677, + # conditions_proved 395 -> 382, decisions_full_mcdc 18 -> 16. That is + # far larger than most real source deltas. + if [[ -f crates/scry-mcdc/rust-toolchain.toml ]]; then + cp crates/scry-mcdc/rust-toolchain.toml "$WT/crates/scry-mcdc/" + status "pinned the $PREV side to $(sed -n 's/^channel = "\(.*\)"/\1/p' crates/scry-mcdc/rust-toolchain.toml)" + else + status "WARNING — no rust-toolchain.toml to copy; the two sides of this delta may use different compilers (scry#215)" + fi if ! ( cd "$WT/crates/scry-mcdc" && cargo build --release --target wasm32-wasip1 -q ); then status "SKIPPED — $PREV does not build"; exit 0 fi diff --git a/artifacts/roadmap-v3.4-evidence-substrate.yaml b/artifacts/roadmap-v3.4-evidence-substrate.yaml new file mode 100644 index 0000000..1dd4dfc --- /dev/null +++ b/artifacts/roadmap-v3.4-evidence-substrate.yaml @@ -0,0 +1,143 @@ +# v3.4.0 addendum — own file per scry#143 fix (1). +artifacts: + - id: FEAT-102 + type: feature + title: "v3.4 — The MC/DC evidence and the shipped artifact share a compiler, and the skew that remains is tracked (scry#215)" + status: proposed + release: v3.4.0 + description: > + scry#215: the MC/DC gate, the released `mcdc-evidence` tarball, the Pages + self-analysis dashboard and the FEAT-072 release delta all measure + `scry_mcdc.wasm` — a wasip1 core module built by Cargo — while the thing + scry ships and signs is `scry.wasm`, a wasip2 component built by Bazel. + The wasip1/DWARF choice is deliberate and documented (witness needs a + DWARF-bearing core module). TWO OTHER SKEWS WERE NOT. + + BOTH MEASURED ON SHIPPED ARTIFACTS, not read off a config file, because a + build config is a hypothesis about an artifact and not evidence about it. + + (1) COMPILER. v3.3.0's signed `scry-3.3.0-wasm32-wasip2.wasm` records + `processed-by rustc [1.91.1 (ed61e7d7e 2025-11-07)]`, which confirms + MODULE.bazel's `rust.toolchain(versions = ["1.91.1"])`. The MC/DC subject + had no `rust-toolchain.toml` anywhere in the repo and was built by + whatever `dtolnay/rust-toolchain@stable` resolved to that day — so the + skew widened on its own and nothing recorded which compiler produced any + given number. + + (2) TOOL. v3.3.0's evidence manifest records + `witness_version: "0.30.0"`, while the CI gate installs witness at rev + 6ff96e75 — which `git ls-remote` resolves to `v0.29.0^{}`, not v0.30.0. + So the floors were proved by one tool version and the shipped evidence + produced by another. Upstream is already at v0.32.0. + + #215'S PROPOSED FIX IS NOT POSSIBLE AS WRITTEN, and the measurement is + why. "Pin the MC/DC path to the same rustc as MODULE.bazel" cannot apply + to the workspace: wasmtime 48.0.8 declares `rust-version = 1.95.0` and + will not build under 1.91.1. It IS possible for the MC/DC subject alone, + because `crates/scry-mcdc` is its own workspace (a `[workspace]` table in + its Cargo.toml), depends only on `scry-sai-core`, has zero wasmtime in + its graph, and `mcdc-gate.sh` does `cd "$(dirname "$0")"` before building + — so a toolchain file there governs exactly that build and nothing else. + + THE SKEW WAS WORTH 13 CONDITIONS. Measured locally with witness held + constant at the gate's own pinned rev, changing ONLY the compiler: + + rustc 1.96.1 -> 1.91.1 + decisions_total 1626 -> 1677 (+51) + conditions_proved 395 -> 382 (-13) + decisions_full_mcdc 18 -> 16 (-2) + + Both sides clear the floors (155 / 3) with wide margin, so the gate stays + green. But the coverage evidence was measurably compiler-dependent while + one side of the comparison floated, which is the whole of #215's claim + turned into a number. + + A CONSEQUENCE THE MEASUREMENT EXPOSED: the FEAT-072 release delta builds + the previous tag in a `git worktree`, which predates the toolchain file — + so without intervention the old side would compile under @stable and the + new side pinned, and a ~51-decision compiler artifact would be reported + as source change, which is the one thing that delta exists to isolate. + The delta step now copies the toolchain file into the worktree first. + + A FIFTH REFERENCE WAS FOUND BY WRITING THE CHECK. The witness rev also + appears bare in two `actions/cache` keys + (`key: witness-bins-${{ runner.os }}-6ff96e75...`). Bumping an install + rev while leaving a cache key stale makes the cache serve binaries the + install step no longer names — the gate would then report coverage from a + witness it does not pin. The check scans for bare 40-hex SHAs on any line + that mentions witness, which is the only way that site is reachable. + tags: [evidence-transfer, gate, substrate, mcdc, issue-driven, scry215, v3.4] + fields: + phase: phase-3 + acceptance-criteria: + - "Given the MC/DC subject, When it is built, Then it uses the same rustc as the shipped component. VERIFIED: `rustc --version` inside crates/scry-mcdc reports `1.91.1 (ed61e7d7e 2025-11-07)`, byte-identical to the `processed-by` string in v3.3.0's signed wasm." + - "Given the pin, When the MC/DC gate runs, Then it still passes. VERIFIED end-to-end with witness at the gate's own rev 6ff96e75: conditions_proved=382 >= 155, decisions_full_mcdc=16 >= 3, exit 0." + - "THE FILE IS NOT THE COMPILER. The script compares rust-toolchain.toml against MODULE.bazel, which is a claim about what WILL compile. The guard step therefore also asserts the ACTIVE rustc where the build happens (`cd crates/scry-mcdc && rustc --version`) and prints it, so the CI log records the evidence's own substrate. NEGATIVE-CONTROLLED locally, and the hazard is real rather than theoretical: `RUSTUP_TOOLCHAIN=1.98.1` DOES beat the toolchain file (measured: rustc 1.98.1), and the assertion catches it." + - "Given a substrate skew, When `tools/check-evidence-substrate.py` runs, Then it names the skew, the file and the line. VERIFIED red before the fix: 3 skews reported, exit 1." + - "Given the witness skew is known-open, When the check runs, Then it REPORTS rather than fails, so the check can be wired into a REQUIRED job today — the step sits in `MC/DC (witness)`, which carries no `continue-on-error` and no job-level `if:`." + - "SELF-CLEANING ALLOWLIST: Given an EXPECTED_SKEWS entry whose skew is no longer present, When the check runs, Then it FAILS and names the stale entry. An allowlist that can outlive its defect is just a silenced gate; a stale entry here is a live regression test of the checker." + - "NEVER ALLOWLISTABLE: Given zero witness references found, When the check runs, Then it fails even if someone adds `no-refs` to the allowlist — losing the subject is how a gate goes vacuous, so it cannot be silenced." + - "MUTATION-CHECKED on the final file: 9 mutations (M1-M9), 9 killed, unmutated control green. THREE survived a first pass and every one was a gap in the TABLE, not the code: the SHA-format regex was never load-bearing because every tag case also carried two DIFFERENT refs (fixed with identical tags); the lost-subject mutation was never tested by its own allowlist key; and dropping the bare-SHA scan's `witness`-line guard survived because no unrelated 40-hex string exists in the workflows TODAY — scry#232 is pinning every action to a SHA, so an unguarded scan would have started failing CI the day it merged. Fixed by running the PARSER against synthetic workflow text, which nothing else did." + residual: > + THE WITNESS SKEW IS NOT CLOSED, by decision. The gate proves the floors + with v0.29.0 and the released bundle is produced by v0.30.0 — and the + v0.30.0 side is internally coherent, since the `cargo install --tag` and + the `compliance@v0.30.0` action match each other. Converging them either + changes the tool that proved the floors (so the floors must be + re-measured) or alters an unsimulated release path. It is tracked by the + EXPECTED_SKEWS entry, which cannot rot: the day the versions converge, + the check fails until the entry is deleted. + + WHAT THE EVIDENCE DOES SUPPORT, stated so a dossier cannot widen it + later: a SOURCE-LEVEL decision-coverage claim over + `crates/scry-analyze-core`, measured on a proxy binary that is now built + with the same compiler as the shipped component and by the same witness + as the gate. It is NOT coverage of the shipped `scry.wasm`. The two + binaries differ by target (wasip1 core module vs wasip2 component) and + profile (opt-level 1 / debug 2 vs release), deliberately and + irreducibly — witness needs DWARF. "Coverage of the shipped component" + stays unearned, and the measured 13-condition compiler sensitivity is + the reason that gap is not a formality. + + The release's `mcdc-evidence` job runs `mcdc-gate.sh` with floors + DISABLED (its own step name says so). The floors are enforced only in + CI. Unchanged here, and worth its own item. + + scry#212's third wasm path (`wasm-lattice`, which no CI job compiles) is + not covered by this check. + + CORRECTION TO THIS ARTIFACT'S OWN FIRST DRAFT: it called the + mcdc-evidence tarball "signed". It is not. MEASURED on v3.3.0 — of 20 + primary assets, 18 are listed in a cosign-signed `SHA256SUMS.txt` and + 18 also carry individual .sig/.pem pairs; `scry-v3.3.0-mcdc-evidence. + tar.gz` is in NEITHER list and is the ONLY primary asset with no + integrity protection at all. The error came from filtering the asset + list on `mcdc|evidence`, seeing `compliance-evidence.tar.gz` return with + its .pem and .sig, and carrying the signing across to the neighbouring + bundle. scry#215's own body says "the evidence bundle we sign and ship", + so the mistake was already in circulation; it also propagated into + witness#360, which has been corrected there. + + That makes it a REQUIREMENT VIOLATION rather than a gap: REQ-010 — the + requirement this feature traces to — asks for the coverage evidence + "surfaced as a SIGNED truth-table report". The report is surfaced and + is not signed. Filed separately; deliberately not fixed here, because + adding an asset to the signing set is a release-path change and this + branch already touches release.yml once. + + A FOURTH SUBSTRATE, found while running the gate and NOT fixed here: + `crates/scry-mcdc/evidence/report.json` and `viz-summary.json` are + TRACKED, and the committed copies record `witness_version: "0.28.0"` + with decisions_total=807 / conditions_proved=234 — a tool version no CI + job installs, produced on someone's local machine. The gate overwrites + both on every run and reads the fresh file, so nothing consumes the + stale copy and no gate rests on it. But CHANGELOG.md:1910 says "the + canonical truth table ships at crates/scry-mcdc/evidence/report.json" + and :1816 says aggregate counts "are committed as + evidence/viz-summary.json" — so two documents describe a stale local + artifact as canonical and shipped. Deliberately left alone here rather + than committing this machine's macOS numbers over it, which would be + equally local and 3.3M lines of churn. Filed separately. + links: + - type: traces-to + target: REQ-010 diff --git a/crates/scry-mcdc/rust-toolchain.toml b/crates/scry-mcdc/rust-toolchain.toml new file mode 100644 index 0000000..e7291f8 --- /dev/null +++ b/crates/scry-mcdc/rust-toolchain.toml @@ -0,0 +1,28 @@ +# scry#215: the MC/DC subject must be compiled by the SAME rustc as the +# component we ship and sign. +# +# `crates/scry-mcdc` is its own workspace (see the `[workspace]` table in +# Cargo.toml), and `mcdc-gate.sh` does `cd "$(dirname "$0")"` before +# building — so this file governs the MC/DC build and nothing else. That +# separation is what makes the pin possible at all: the MAIN workspace +# cannot be pinned here, because wasmtime 48.0.8 declares +# rust-version = 1.95.0 and would not build under 1.91.1. +# +# 1.91.1 is not copied from MODULE.bazel on faith. v3.3.0's signed +# `scry-3.3.0-wasm32-wasip2.wasm` records it directly: +# +# $ wasm-tools metadata show scry-3.3.0-wasm32-wasip2.wasm +# name scry_analyzer_component_wasm_lib_release_wasm_base.wasm +# processed-by rustc [1.91.1 (ed61e7d7e 2025-11-07)] +# +# Before this file, the MC/DC subject was built by whatever +# `dtolnay/rust-toolchain@stable` resolved to on the day the job ran, and +# the skew widened on its own. `tools/check-evidence-substrate.py` asserts +# this channel against MODULE.bazel's `rust.toolchain(versions = [...])`, +# so the two cannot drift apart again silently. +# +# Keep in step with MODULE.bazel on every Bazel rustc bump; the check will +# name the file if you forget. +[toolchain] +channel = "1.91.1" +targets = ["wasm32-wasip1"] diff --git a/tools/check-evidence-substrate.py b/tools/check-evidence-substrate.py new file mode 100755 index 0000000..16ba61b --- /dev/null +++ b/tools/check-evidence-substrate.py @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 +"""scry#215: the evidence and the artifact must be made on the same substrate. + +Two skews, both measured on SHIPPED artifacts rather than read off a config: + + * COMPILER. `bazel build //:scry` builds the released component with a PINNED + rustc; the MC/DC subject (`crates/scry-mcdc`, a separate workspace) was + built by whatever `dtolnay/rust-toolchain@stable` resolved to that day. The + pin is not a hypothesis — v3.3.0's signed `scry-3.3.0-wasm32-wasip2.wasm` + records `processed-by rustc [1.91.1 (ed61e7d7e 2025-11-07)]`, matching + MODULE.bazel's `versions = ["1.91.1"]`. The floating side has no record at + all, and the gap widens on its own. + + * TOOL. The MC/DC gate installs witness at rev 6ff96e75 (= tag v0.29.0); the + release job that produces the RELEASED evidence bundle installs + `--tag v0.30.0` and runs the compliance action at `@v0.30.0`. Read off the + artifact: v3.3.0's evidence manifest says `witness_version: "0.30.0"`. So + the floors were proved by one tool version and the shipped evidence was + produced by another. + +Neither check makes the two binaries identical — they cannot be (wasip1 core +module with DWARF vs wasip2 component, deliberately; see +crates/scry-mcdc/Cargo.toml). They remove skew that nothing currently tracks. + +WHY SHA-ONLY FOR WITNESS: comparing `--rev 6ff96e75...` against `--tag v0.30.0` +textually cannot work, and resolving a tag needs network access from the gate. +Requiring every reference to be a 40-hex commit makes them comparable offline. +That is the same pinning hygiene scry#232 is adding for third-party actions, +arrived at from the verification side rather than the supply-chain side. + +Run with --self-test first; it proves the predicates discriminate in BOTH +directions before they are pointed at the real tree. +""" +import sys, re, os, glob, tempfile + +WITNESS_REPO = "pulseengine/witness" +SHA40 = re.compile(r'^[0-9a-f]{40}$') + +# `REV=` / `WITNESS_REV: ` / `WITNESS_VERSION: v0.30.0` +ASSIGN = re.compile( + r'(?:^|\s)(?:WITNESS_REV|WITNESS_VERSION|REV)\s*[:=]\s*"?([A-Za-z0-9._-]+)"?', + re.M) +# `uses: pulseengine/witness/...@` +USES = re.compile(r'uses:\s*' + re.escape(WITNESS_REPO) + r'(?:/[^@\s]*)?@([A-Za-z0-9._-]+)') +# A bare 40-hex on a line that also mentions witness — e.g. the binary cache +# key `key: witness-bins-${{ runner.os }}-6ff96e75...`. This one is the +# nastiest of the lot: bump the install rev and forget the cache key, and the +# cache serves the OLD binaries while the install step advertises the new rev. +# The gate would then report coverage from a witness it does not pin, which is +# the same wrong-substrate error one level down. +BARE_SHA = re.compile(r'(? [(file, lineno, ref)] for every witness version named in a workflow.""" + found = [] + for path in sorted(glob.glob(os.path.join(workflow_dir, "*.yml"))): + text = open(path, encoding="utf-8").read() + mentions_witness = WITNESS_REPO in text or "witness-mcdc" in text + for i, line in enumerate(text.splitlines(), 1): + for m in USES.finditer(line): + found.append((path, i, m.group(1))) + if not mentions_witness: + continue + matched = False + for m in ASSIGN.finditer(line): + found.append((path, i, m.group(1))) + matched = True + # Only scan for bare SHAs on lines that name witness themselves, + # so an unrelated commit hash elsewhere is not dragged in. + if not matched and "witness" in line.lower(): + for m in BARE_SHA.finditer(line): + found.append((path, i, m.group(1))) + return found + + +def bazel_rustc(path="MODULE.bazel"): + """The rustc the SHIPPED component is built with.""" + text = open(path, encoding="utf-8").read() + m = re.search(r'rust\.toolchain\((.*?)\)', text, re.S) + if not m: + return None + v = re.search(r'versions\s*=\s*\[\s*"([^"]+)"', m.group(1)) + return v.group(1) if v else None + + +def mcdc_rustc(path="crates/scry-mcdc/rust-toolchain.toml"): + """The rustc the MC/DC SUBJECT is built with, or None if it floats.""" + if not os.path.isfile(path): + return None + text = open(path, encoding="utf-8").read() + m = re.search(r'^\s*channel\s*=\s*"([^"]+)"', text, re.M) + return m.group(1) if m else None + + +# ── Known-open skews ──────────────────────────────────────────────────── +# A skew listed here is REPORTED, not failed — so this check can be wired +# into a REQUIRED job today while one half is still open. Every entry names +# the issue that owns it. +# +# The entries are self-cleaning, and that is the point: if a listed skew is +# no longer present, this check FAILS and tells you to delete the entry. So +# the allowlist cannot quietly outlive the defect, and a stale entry is a +# live regression test of the checker itself — the one shape in this repo's +# vacuous-gate history that works in your favour. +EXPECTED_SKEWS = { + "witness-version": + "scry#215 — the MC/DC gate proves the floors with witness v0.29.0 " + "(rev 6ff96e75) while the RELEASED evidence bundle is produced by " + "v0.30.0 (both the `cargo install --tag` and the compliance action, " + "which are coherent with each other). Converging them changes either " + "what proved the floors or an unsimulated release path, so the " + "direction is a deliberate decision and not this change.", +} + + +def verdict(refs, bazel_v, mcdc_v, expected=None): + """-> (fails, reported). `fails` blocks; `reported` is known-open. + + A skew in `expected` moves from `fails` to `reported`. An `expected` key + whose skew is ABSENT becomes a failure, so the allowlist self-cleans.""" + if expected is None: + expected = EXPECTED_SKEWS + fails, reported, seen = [], [], set() + + def skew(key, msg): + seen.add(key) + if key in expected: + reported.append((key, msg, expected[key])) + else: + fails.append(msg) + + # ── tool: every witness reference is one identical 40-hex commit ── + non_sha = [(f, n, r) for (f, n, r) in refs if not SHA40.match(r)] + distinct = sorted({r for (_, _, r) in refs}) + if non_sha or len(distinct) > 1: + by_ref = {} + for f, n, r in refs: + by_ref.setdefault(r, []).append(f"{f}:{n}") + detail = "\n".join(f" {r}{' (not a 40-hex commit)' if not SHA40.match(r) else ''}" + f" <- {', '.join(sites)}" + for r, sites in sorted(by_ref.items())) + skew("witness-version", + f"witness is pinned {len(distinct)} different way(s); the gate that proves " + "the MC/DC floors and the job that produces the RELEASED evidence must be " + "the same tool, and every reference must be a 40-hex commit so they can " + f"be compared without network access:\n{detail}") + if not refs: + # Never allowlistable: losing the subject is how a gate goes vacuous. + fails.append("no witness reference found at all — this check has lost its subject") + + # ── compiler: the MC/DC subject matches the shipped artifact's rustc ── + if bazel_v is None: + fails.append("could not read rust.toolchain versions from MODULE.bazel") + elif mcdc_v is None: + skew("mcdc-rustc", + "crates/scry-mcdc has no rust-toolchain.toml, so the MC/DC subject is " + f"built by whatever @stable resolves to, while the shipped component is " + f"built with rustc {bazel_v}") + elif mcdc_v != bazel_v: + skew("mcdc-rustc", + f"MC/DC subject pins rustc {mcdc_v} but the shipped component is built " + f"with {bazel_v} (MODULE.bazel)") + + # Self-cleaning: an allowlisted skew that is no longer present must go. + for key in sorted(set(expected) - seen): + fails.append( + f"STALE allowlist entry '{key}': this skew is no longer present, so the " + f"entry in EXPECTED_SKEWS is now hiding nothing and must be deleted. " + f"It was recorded as: {expected[key]}") + return fails, reported + + +def self_test(): + ok = True + S1 = "a" * 40 + S2 = "b" * 40 + NO_ALLOW = {} + ALLOW_W = {"witness-version": "owned by scry#215"} + + # (label, refs, bazel, mcdc, allowlist, expect_block) + cases = [ + # ── aligned: nothing to say ───────────────────────────────────── + ("aligned, empty allowlist", + [("ci", 1, S1), ("rel", 2, S1)], "1.91.1", "1.91.1", NO_ALLOW, False), + # ── unexpected skews BLOCK ────────────────────────────────────── + ("two different SHAs, not allowlisted", + [("ci", 1, S1), ("rel", 2, S2)], "1.91.1", "1.91.1", NO_ALLOW, True), + ("a tag instead of a SHA, not allowlisted", + [("ci", 1, S1), ("rel", 2, "v0.30.0")], "1.91.1", "1.91.1", NO_ALLOW, True), + ("floating mcdc toolchain, not allowlisted", + [("ci", 1, S1)], "1.91.1", None, NO_ALLOW, True), + ("mcdc toolchain disagrees with Bazel", + [("ci", 1, S1)], "1.91.1", "1.99.0", NO_ALLOW, True), + # ── allowlisted skews REPORT, do not block ────────────────────── + ("today's real shape: witness skew allowlisted, rustc aligned", + [("ci", 1, S1), ("rel", 2, S1), ("rel", 3, "v0.30.0")], + "1.91.1", "1.91.1", ALLOW_W, False), + # ── the allowlist must NOT cover a different skew ──────────────── + ("witness allowlisted does not excuse a floating toolchain", + [("ci", 1, S1), ("rel", 2, "v0.30.0")], "1.91.1", None, ALLOW_W, True), + # ── SELF-CLEANING: a stale entry blocks ───────────────────────── + ("stale entry: witness now aligned but still allowlisted", + [("ci", 1, S1), ("rel", 2, S1)], "1.91.1", "1.91.1", ALLOW_W, True), + # ── losing the subject is never allowlistable ─────────────────── + ("no witness reference at all, even with the allowlist", + [], "1.91.1", "1.91.1", ALLOW_W, True), + ("MODULE.bazel unreadable is never allowlistable", + [("ci", 1, S1)], None, "1.91.1", ALLOW_W, True), + # ── Added after a mutation check. Both of these SURVIVED first: ── + # M3 (SHA40 accepts anything): the tag case above also had two + # DIFFERENT refs, so `len(distinct) > 1` blocked it whether the regex + # worked or not. The regex was never load-bearing. This row makes every + # ref an IDENTICAL tag — textually comparable, still unresolvable + # without network — so only the regex can catch it. + ("every ref is the same TAG: comparable, but not offline", + [("ci", 1, "v0.30.0"), ("rel", 2, "v0.30.0")], + "1.91.1", "1.91.1", NO_ALLOW, True), + # M5 (lost subject becomes allowlistable): the no-refs case above + # allowlisted `witness-version`, not `no-refs`, so the mutation was + # never tested by its own key. Losing the subject must be unallowlistable + # even when someone names it explicitly. + ("lost subject is not allowlistable even by its own key", + [], "1.91.1", "1.91.1", {"no-refs": "someone tried to silence this"}, True), + ] + for label, refs, bv, mv, allow, expect_block in cases: + fails, _ = verdict(refs, bv, mv, allow) + got_block = bool(fails) + if got_block != expect_block: + print(f" SELF-TEST FAIL ({label}): " + f"{'blocked' if got_block else 'passed'}, expected " + f"{'block' if expect_block else 'pass'}") + ok = False + + # An allowlisted skew must be REPORTED, not silently dropped. + fails, reported = verdict( + [("ci", 1, S1), ("rel", 2, "v0.30.0")], "1.91.1", "1.91.1", ALLOW_W) + if fails or [k for k, _, _ in reported] != ["witness-version"]: + print(f" SELF-TEST FAIL: allowlisted skew not reported: " + f"fails={len(fails)} reported={[k for k,_,_ in reported]}") + ok = False + + # Both outcomes present, or the table passes against a constant predicate. + if not (any(c[5] for c in cases) and any(not c[5] for c in cases)): + print(" SELF-TEST FAIL: table does not exercise both outcomes"); ok = False + + # The parser must find the real tree's refs, or every predicate above is + # being fed an empty list in production. + real = witness_refs() + if len(real) < 5: + print(f" SELF-TEST FAIL: parser found {len(real)} witness refs in " + f".github/workflows, expected at least 5 (ci.yml REV + its cache " + f"key, release.yml WITNESS_VERSION + the compliance action + " + f"WITNESS_REV)"); ok = False + # ── the PARSER, against synthetic workflow text ────────────────── + # Everything above feeds `verdict` pre-built tuples, so none of it + # exercises witness_refs(). Added after a mutation check: dropping the + # `"witness" in line` guard from the bare-SHA scan SURVIVED, because no + # unrelated 40-hex string exists in the workflows TODAY. scry#232 is + # pinning every GitHub Action to a commit SHA, so the day it lands there + # will be dozens, and an unguarded scan would report them as conflicting + # witness versions and fail CI for no reason. + with tempfile.TemporaryDirectory() as td: + OTHER = "c" * 40 + WIT = "d" * 40 + open(os.path.join(td, "probe.yml"), "w", encoding="utf-8").write( + "jobs:\n" + " j:\n" + " steps:\n" + f" - uses: actions/checkout@{OTHER}\n" + " - uses: Swatinem/rust-cache@v2\n" + " - name: Cache build\n" + " uses: actions/cache@v4\n" + " with:\n" + f" key: build-cache-${{{{ runner.os }}}}-{OTHER}\n" + " - name: Cache witness binaries\n" + " uses: actions/cache@v4\n" + " with:\n" + f" key: witness-bins-${{{{ runner.os }}}}-{WIT}\n" + " - run: |\n" + f" REV={WIT}\n" + " cargo install --git $URL --rev \"$REV\" witness-mcdc\n") + got = sorted({r for _, _, r in witness_refs(td)}) + if got != [WIT]: + print(f" SELF-TEST FAIL (parser): picked up {got}, expected only " + f"the witness ref {WIT!r}. A non-witness action or cache key " + f"pinned to a SHA must NOT be read as a witness version " + f"(scry#232 pins every action to a SHA).") + ok = False + + # The binary cache key is a BARE sha on a witness line, not a VAR= + # assignment, so only the bare-SHA scan reaches it. If that scan + # regresses, this row notices — and the hazard it guards is specific: + # bumping the install rev while the cache key keeps the old one makes the + # cache serve binaries the install step no longer names. + cache_key_refs = [ + (f, n) for f, n, _ in real + if open(f, encoding="utf-8").read().splitlines()[n - 1].lstrip().startswith("key:") + ] + if len(cache_key_refs) < 2: + print(f" SELF-TEST FAIL: bare-SHA scan found {len(cache_key_refs)} witness " + f"cache-key refs, expected 2 (ci.yml and release.yml both cache the " + f"built binaries keyed by rev)"); ok = False + + print(" self-test:", "OK — blocks, reports and self-cleans" if ok else "FAILED") + return 0 if ok else 1 + + +def main(): + if "--self-test" in sys.argv: + return self_test() + refs = witness_refs() + bazel_v = bazel_rustc() + mcdc_v = mcdc_rustc() + fails, reported = verdict(refs, bazel_v, mcdc_v) + + print(f" shipped component rustc : {bazel_v} (MODULE.bazel; v3.3.0's signed " + f"wasm records `processed-by rustc [1.91.1 ...]`)") + print(f" MC/DC subject rustc : {mcdc_v or 'FLOATING (@stable)'}") + print(f" witness references : {len(refs)} across " + f"{len({f for f, _, _ in refs})} workflow file(s)") + for f, n, r in refs: + print(f" {f}:{n} {r}") + + for key, msg, why in reported: + print(f"\n::notice title=scry#215 known-open substrate skew " + f"({key})::{msg}\n OWNED BY: {why}") + if fails: + for msg in fails: + print(f"\n::error title=scry#215 evidence substrate::{msg}") + print(f"\n FAILED — {len(fails)} unexpected or stale substrate skew(s)") + return 1 + if reported: + print(f"\n OK — {len(reported)} known-open skew(s), each owned by an issue; " + f"nothing unexpected, nothing stale") + else: + print("\n OK — evidence and artifact share a compiler, and one witness commit") + return 0 + + +if __name__ == "__main__": + sys.exit(main())