From 402b2444af14da47c0583964e9c4cb32f6576a75 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 17 Nov 2026 02:08:02 +0800 Subject: [PATCH 01/13] Hold CI mutation runs until the gates pass A mutation shard rebuilds and runs the whole suite in an unmutated tree, so when the Rust lane fails the shards fail the same way and each pays the whole setup to learn it. The mutants job needed only the plan, so it started beside the gates. Make it need the gates as well, and start only when both the plan and the gates succeeded and the plan found mutants. The plan result is checked explicitly because the plan is skipped on a push or a merge group, where its empty count would otherwise pass the zero guard. The aggregate check job already accepts a skipped mutants job after a successful plan, and a failed gates lane fails it first. Also hold the runs to the lockfile. The Makefile mutants target passes --cargo-arg=--locked so a mutant is never judged against a lockfile cargo quietly updated, but the plan's --list and the shard's --list and run did not, so CI could test against whatever lock cargo produced and the local and CI answers could differ. Pass the flag on all three commands. A new suite pins the job's needs and condition, that each command carries the flag, and that the plan and the shard list select with the same flags so the shard count matches what runs. The development guide says the shards wait for the gate lanes and are skipped when a lane failed. Change-Id: I02d001e8746051adf98b194ddab92f7a7a0fa123 --- .github/workflows/check.yml | 24 ++++++-- docs/development.md | 7 ++- scripts/test.sh | 1 + tests/support.py | 8 ++- tests/test_check_workflow.py | 116 +++++++++++++++++++++++++++++++++++ tests/test_mutants_plan.py | 2 +- 6 files changed, 147 insertions(+), 11 deletions(-) create mode 100644 tests/test_check_workflow.py diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 2cf880325..3e478dc48 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -629,7 +629,8 @@ jobs: set -o pipefail count=0 if [ -s "$RUNNER_TEMP/pr.diff" ]; then - cargo mutants --list --in-diff "$RUNNER_TEMP/pr.diff" >"$RUNNER_TEMP/list.txt" + cargo mutants --list --cargo-arg=--locked \ + --in-diff "$RUNNER_TEMP/pr.diff" >"$RUNNER_TEMP/list.txt" count=$(wc -l <"$RUNNER_TEMP/list.txt" | tr -d ' ') fi width=$(( (count + 14) / 15 )) @@ -673,10 +674,21 @@ jobs: # asks is what a change failed to test, and that is worth answering while the # change can still be edited. mutants: - needs: plan-mutants - # No mutants means no runners. A docs-only or JS-only pull request should + needs: [plan-mutants, gates] + # A shard rebuilds and runs the whole suite in an unmutated tree, so it + # fails the same way the Rust lane does, and pays the whole setup per shard + # to learn it. Waiting for the gates avoids that; the trade-off + # is that shards start after the slowest gates lane rather than beside it. + # + # `plan-mutants.result` is explicit because the plan is skipped on a push + # or a merge group, where its count is empty and `'' != '0'` would pass. + # No mutants means no runners: a docs-only or JS-only pull request should # not start a job to discover it has nothing to do. - if: needs.plan-mutants.outputs.count != '0' + if: >- + ${{ !cancelled() + && needs.plan-mutants.result == 'success' + && needs.gates.result == 'success' + && needs.plan-mutants.outputs.count != '0' }} runs-on: ubuntu-24.04 timeout-minutes: 45 strategy: @@ -739,7 +751,7 @@ jobs: run: | git diff "origin/$BASE_REF...HEAD" -- '*.rs' >"$RUNNER_TEMP/pr.diff" echo "::group::Mutants in shard $SHARD" - cargo mutants --list --in-diff "$RUNNER_TEMP/pr.diff" \ + cargo mutants --list --cargo-arg=--locked --in-diff "$RUNNER_TEMP/pr.diff" \ --shard "$SHARD" \ | tee "$RUNNER_TEMP/shard.txt" echo "::endgroup::" @@ -758,7 +770,7 @@ jobs: # overrun would be swallowed by the construct meant to report it. status=0 timeout --signal=INT --kill-after=60s 35m \ - cargo mutants --in-place --in-diff "$RUNNER_TEMP/pr.diff" \ + cargo mutants --in-place --cargo-arg=--locked --in-diff "$RUNNER_TEMP/pr.diff" \ --shard "$SHARD" \ || status=$? if [ "$status" -eq 124 ]; then diff --git a/docs/development.md b/docs/development.md index 4568edc78..ad71a093d 100644 --- a/docs/development.md +++ b/docs/development.md @@ -126,9 +126,10 @@ before moving selected text. `check` is the only status a branch rule for main should require. It needs every lane and the independent security audit to succeed, and on a pull request the -mutation plan and every shard too; shards may be skipped only when the plan -found no mutants. A push to main or a merge queue entry plans no mutants, so -`check` reports as soon as the lanes finish. The workflow also runs on +mutation plan and every shard too; shards wait for the gate lanes and may be +skipped only when the plan found no mutants or a lane failed, in which case +`check` fails on the lane. A push to main or a merge queue entry plans no +mutants, so `check` reports as soon as the lanes finish. The workflow also runs on `merge_group`, so once main has a merge queue, it tests each pull request on top of the main it will land on; two pull requests that pass against their own bases can still break main together. A merge group saves no caches, since its queue diff --git a/scripts/test.sh b/scripts/test.sh index 0c400f897..377995dcc 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -752,6 +752,7 @@ gate git-hooks "$ROOT/scripts/test-git-hooks.sh" gate commit-reflow-tests unittest_gate "$ROOT/tests/test_commit_reflow.py" gate commit-draft-tests unittest_gate "$ROOT/tests/test_commit_draft.py" gate pr-text-tests unittest_gate "$ROOT/tests/test_check_pr_text.py" +gate check-workflow-tests unittest_gate "$ROOT/tests/test_check_workflow.py" gate shell-syntax shell_syntax gate browser-check-env env BROWSER_CHECK_VALIDATE_ENV_ONLY=1 sh "$ROOT/scripts/browser-check.sh" diff --git a/tests/support.py b/tests/support.py index 7ec52a8a2..5bea165ab 100644 --- a/tests/support.py +++ b/tests/support.py @@ -5,6 +5,7 @@ imports each suite from its own path. """ +import functools import io import re import subprocess @@ -63,9 +64,14 @@ def section(text, start, end, what): return text.split(start, 1)[1].split(end, 1)[0] +@functools.cache +def workflow_text(): + return WORKFLOW.read_text() + + def job(name, text=None): """The body of one job, up to the next job.""" - text = WORKFLOW.read_text() if text is None else text + text = workflow_text() if text is None else text body = section(text, f"\n {name}:\n", "\0", f"job {name}") following = re.search(r"\n [a-z][a-z0-9_-]*:\n", body) return body[: following.start()] if following else body diff --git a/tests/test_check_workflow.py b/tests/test_check_workflow.py new file mode 100644 index 000000000..fadc4843f --- /dev/null +++ b/tests/test_check_workflow.py @@ -0,0 +1,116 @@ +"""The shape of the mutation job and the aggregate status in check.yml.""" + +import importlib.util +from pathlib import Path +import re +import shlex +import unittest + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location( + "check_workflow_support", ROOT / "tests/support.py" +) +support = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(support) + + +def header(body, key): + """The value of a job-level key, with a folded scalar joined to one line.""" + found = re.search(rf"^ {key}: (.*(?:\n .*)*)", body, re.M) + if found is None: + raise AssertionError(f"job no longer sets {key}") + return " ".join(found[1].replace(">-", "").split()) + + +class MutantsJobTests(unittest.TestCase): + def setUp(self): + self.body = support.job("mutants") + + def test_waits_for_the_gates_and_the_plan(self): + needs = set(re.findall(r"[a-z][a-z-]*", header(self.body, "needs"))) + self.assertLessEqual({"gates", "plan-mutants"}, needs) + + def test_starts_only_after_both_succeed_with_work_to_do(self): + condition = header(self.body, "if").strip("${} ") + # Exactly these terms, all required. `!cancelled()` replaces the + # implicit success() that would otherwise skip the job. plan-mutants + # is skipped on push and merge_group, where its count output is empty + # and would pass the != '0' guard on its own. + self.assertEqual( + [ + "!cancelled()", + "needs.plan-mutants.result == 'success'", + "needs.gates.result == 'success'", + "needs.plan-mutants.outputs.count != '0'", + ], + [term.strip() for term in condition.split("&&")], + ) + + +def mutants_invocations(body): + """The argument words of every `cargo mutants` command. + + Continuation lines are joined, comments dropped, and the words end at the + first redirection, pipe or `||`, so a flag only counts when the shell would + pass it. + """ + found = [] + for command in re.findall( + r"^[^#\n]*?cargo mutants\b((?:[^\n]*\\\n)*[^\n]*)", body, re.M + ): + words = [] + for word in shlex.split(command.replace("\\\n", " "), comments=True): + if word[0] in "|>": + break + words.append(word) + found.append(words) + return found + + +class MutantsLockfileTests(unittest.TestCase): + def setUp(self): + self.plan = mutants_invocations(support.job("plan-mutants")) + self.shard = mutants_invocations(support.job("mutants")) + + def test_every_selecting_or_running_command_holds_the_lockfile(self): + # cargo-mutants forwards the flag to every cargo it runs; without it + # a mutant can be judged against a lockfile cargo quietly updated. + commands = self.plan + self.shard + self.assertEqual(3, len(commands)) + for words in commands: + self.assertIn("--cargo-arg=--locked", words, words) + + def test_plan_and_shard_select_the_same_mutants(self): + # The shard count comes from the plan's list, so what is listed has to + # match on both sides: the same flags with the same values. + def selection(words): + kept, skip = [], False + for word in words: + if skip: + skip = False + elif word == "--shard": + skip = True + elif word not in ("--list", "--in-place"): + kept.append(word) + return sorted(kept) + + listings = [w for w in self.plan + self.shard if "--list" in w] + self.assertEqual(2, len(listings)) + self.assertEqual(selection(listings[0]), selection(listings[1])) + run = next(w for w in self.shard if "--list" not in w) + self.assertEqual(selection(listings[1]), selection(run)) + + +class AggregateStatusTests(unittest.TestCase): + def test_check_accepts_mutants_skipped_after_a_plan(self): + body = support.job("check") + self.assertRegex(body, r"success/success \| skipped/success[^)]*\) ;;") + # A failed gates lane fails the job before the mutants table is read. + self.assertLess(body.index('"$GATES" != success'), body.index("case ")) + + def test_release_does_not_wait_for_mutants(self): + self.assertNotIn("mutants", header(support.job("release"), "needs")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mutants_plan.py b/tests/test_mutants_plan.py index ff9b26e47..019fb2438 100644 --- a/tests/test_mutants_plan.py +++ b/tests/test_mutants_plan.py @@ -42,7 +42,7 @@ def setUp(self): set -eu echo "$*" >>"$RUNNER_TEMP/cargo.log" [ "$FAKE_FAIL" = 0 ] || exit 7 -[ "$*" = "mutants --list --in-diff $RUNNER_TEMP/pr.diff" ] || exit 9 +[ "$*" = "mutants --list --cargo-arg=--locked --in-diff $RUNNER_TEMP/pr.diff" ] || exit 9 i=0 while [ "$i" -lt "$FAKE_COUNT" ]; do echo "mutant $i" From 25abfe0299abdbebc3ef58a9b041e3e6aecb73dc Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 17 Nov 2026 02:29:57 +0800 Subject: [PATCH 02/13] Keep the lane tests off the venv setup The lane tests run scripts/test.sh with its gate payload replaced, but the Python lookup above the gates is the real one. In a clone with no instructor tool environment it builds one on the first run, several lane tests running at once, so they see a failed task-tools gate or a half-built environment, and pass only on the second run once it exists. CI names its own environment and never sees it. Give run_lane the running interpreter as PYTHON, when it is 3.11 or later, with the packaging probe answered as ready, unless the test names PYTHON or CODETRIAL_TASK_TOOLS itself. The tests of the lookup and of the setup name one of them and keep the real behavior; the one that named neither now names an empty environment, which the script treats as unset. Change-Id: Idace6b303f73ae9182e538c893838cf3dc98e8da --- tests/test_test_gate.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/test_test_gate.py b/tests/test_test_gate.py index 5fe1840d3..fe966cb30 100644 --- a/tests/test_test_gate.py +++ b/tests/test_test_gate.py @@ -110,6 +110,19 @@ def run_lane( env=None, ): source = SCRIPT.read_text() + named = env or {} + env = {**os.environ, "CXX": "unused", **named} + # Stand in a ready Python for the lookup, which would otherwise build the + # instructor venv on a first run, and answer the packaging probe the + # portable lane asks. A test naming PYTHON or CODETRIAL_TASK_TOOLS + # (empty included) keeps the real lookup. + if ( + "PYTHON" not in named + and "CODETRIAL_TASK_TOOLS" not in named + and sys.version_info >= (3, 11) + ): + env["PYTHON"] = sys.executable + replace = (("\"$1\" -c 'import cryptography'", "true"), *replace) for old, new in replace: self.assertEqual(source.count(old), 1, old) source = source.replace(old, new) @@ -141,7 +154,6 @@ def run_lane( ) if extra_gate: source = source.replace("gate fmt ", "gate future-check true\ngate fmt ", 1) - env = {**os.environ, "CXX": "unused", **(env or {})} if unset_cxx: del env["CXX"] result = subprocess.run( @@ -862,7 +874,9 @@ def test_only_the_portable_lane_sets_up_the_packaging_python(self): ("browser", False), ): with self.subTest(lane=lane): - result, _ = self.run_lane(lane, replace=replace) + # An empty name keeps the lookup this test is about. + env = {"CODETRIAL_TASK_TOOLS": ""} + result, _ = self.run_lane(lane, replace=replace, env=env) self.assertEqual("setup:task-tools" in result.stderr, setup) self.assertEqual( condition(gates_job(), "Place the instructor packaging environment"), From 2369aae45e61c52a9ba50c3d21a69b1fef8bceb7 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 17 Nov 2026 02:09:06 +0800 Subject: [PATCH 03/13] Check the environment before a gate run starts What a gate run needs (the pinned toolchain, Clang, node, optional linters) was learned one failing gate at a time, because the gate script only records a failure and carries on. A machine without the pinned toolchain produced a wall of unrelated failures, and rustup installs the pinned toolchain the first time any rustup command runs in the repository, so even asking which toolchains exist could start a download. Add scripts/env-check.sh, which only looks and never installs. Each profile lists what the matching lane needs, and a missing tool is named with the command that installs it. It asks rustup for the installed toolchains from outside the repository, so no pin applies, before it runs cargo or rustc there, and it reports a broken cargo as a failure instead of passing it over. scripts/test.sh runs it before any gate and stops there when a required tool is missing. The optional tools that are absent come from the same report, so the per-gate probes for ruff, shellcheck, dash, actionlint, cargo-audit and cargo-deny are gone and the skip lines keep their wording. The suites use stub executables, and the system-tool PATH they need is shared in tests/support.py. The development guide gains an Environment check section. The compiler check applies on Linux only, where webrtc-sys needs Clang 21 or later; another platform builds with its own compiler. Change-Id: I77c2c1ce3d4818027dec8d1eb072c16acff452dc --- .claude/skills/codetrial-verify/SKILL.md | 8 +- docs/development.md | 61 ++- scripts/env-check.sh | 443 ++++++++++++++++++++ scripts/test.sh | 62 ++- tests/stubs/env-check/cargo.sh | 30 ++ tests/stubs/env-check/clang.sh | 4 + tests/stubs/env-check/rustc.sh | 20 + tests/stubs/env-check/rustup.sh | 12 + tests/stubs/env-check/tool.sh | 3 + tests/stubs/env-check/uname.sh | 10 + tests/support.py | 18 + tests/test_env_check.py | 502 +++++++++++++++++++++++ tests/test_test_gate.py | 154 +++++++ 13 files changed, 1311 insertions(+), 16 deletions(-) create mode 100755 scripts/env-check.sh create mode 100644 tests/stubs/env-check/cargo.sh create mode 100644 tests/stubs/env-check/clang.sh create mode 100644 tests/stubs/env-check/rustc.sh create mode 100644 tests/stubs/env-check/rustup.sh create mode 100644 tests/stubs/env-check/tool.sh create mode 100644 tests/stubs/env-check/uname.sh create mode 100644 tests/test_env_check.py diff --git a/.claude/skills/codetrial-verify/SKILL.md b/.claude/skills/codetrial-verify/SKILL.md index 3cfddc76c..9d0e0642b 100644 --- a/.claude/skills/codetrial-verify/SKILL.md +++ b/.claude/skills/codetrial-verify/SKILL.md @@ -16,6 +16,11 @@ One gate needs no credentials and is the thing to run: make check # every gate, plus a live Gemini credential check ``` +`scripts/env-check.sh` says whether this machine has what a lane needs, naming +each missing tool with the command that installs it, without starting a build. +`scripts/test.sh` runs it first and stops there, before any gate, when a +required tool is missing. + CI runs one `gates ()` job per lane in parallel: `gates (rust)`, `gates (portable)` and, in three slices, `gates (browser 1/3)`, `gates (browser 2/3)` and `gates (browser 3/3)` @@ -47,7 +52,8 @@ It is not offline. The `fetch-vendor` gate downloads missing assets named by a `actionlint` publishes itself as a container, so that lane may reach for docker. Both are checksum-pinned or version-pinned; neither needs a credential. -A lane whose tool is absent skips instead of failing. Every skip the gate can +A lane whose optional tool is absent skips instead of failing; a missing +required tool stops the run before any gate, as above. Every skip the gate can see, its own, a formatter's in `scripts/indent.sh`, a Node browser test's and a Python case's, appears under the final `not checked by this run:` summary. Read it before claiming coverage; never treat the skip set or its size as diff --git a/docs/development.md b/docs/development.md index ad71a093d..f58a26c28 100644 --- a/docs/development.md +++ b/docs/development.md @@ -166,9 +166,13 @@ composite actions under `.github/actions/` as the lanes, and saves none of them, so the weekly job keeps no second copy. Chromium's system dependencies are still installed on every fresh runner. -The gate picks its own compiler and Python, so running it directly builds with -the same Clang `make` would. On Linux x86_64 with no `CXX` set, it uses the checksum-pinned -Clang that `make` fetches into `target/clang`, fetching it just before the +Before any gate runs, `./scripts/test.sh` asks `scripts/env-check.sh` whether +the machine can run the lane it was given, and stops with the missing tool and +the command that installs it when it cannot; see +[Environment check](#environment-check). The gate picks its own compiler and +Python, so running it directly builds with the same Clang `make` would. On +Linux x86_64 with no `CXX` set, it uses the checksum-pinned Clang that `make` +fetches into `target/clang`, fetching it just before the first lane that compiles. It runs the Python lanes with the first Python 3.11 or newer it finds, because the Rust style checker uses the standard-library TOML parser; set `PYTHON` to name one. An explicitly named older interpreter is @@ -240,6 +244,57 @@ Trimming debug info, which is what buys CI 40% per relink, was measured here and buys nothing: 7.5s against 8.1s on macOS, where the system linker rather than mold does the work. It stays a CI setting for that reason. +## Environment check + +`scripts/env-check.sh` is the one place that says what a run needs. It only +looks: it never downloads or installs anything, and it neither fetches Clang nor +hashes `web/vendor`, because `scripts/test.sh` does both itself, once per run. + +```bash +./scripts/env-check.sh # the local-rust profile +./scripts/env-check.sh --profile portable # also: ci-rust, browser +./scripts/env-check.sh --format json # one JSON object on stdout +``` + +Each profile lists what the matching lane of `scripts/test.sh` runs: + +| Profile | Required | +| --- | --- | +| `local-rust` | the pinned toolchain installed (with rustup), `cargo` and `rustc` at the `rust-toolchain.toml` version, `clippy`, `rustfmt`, no `RUSTUP_TOOLCHAIN`, Clang 21+ through `CXX` (Linux only), `web/vendor`, `node` | +| `ci-rust` | `local-rust`, plus `mold` and `RUSTFLAGS` linking with it | +| `portable` | the pinned toolchain installed (with rustup), `cargo` with `rustfmt` (the lane runs `cargo fmt --check`), `node` | +| `browser` | `node` | + +`scripts/test.sh` picks `local-rust` for the `rust` and `all` lanes, `ci-rust` +for those when `CI=true`, and the lane's own name otherwise. It passes +`--will-provision`, which turns a missing Clang on Linux x86_64 into a note +because the script fetches it after the checks; run by hand, the same state +fails and prints `sh scripts/fetch-clang.sh target/clang`. A `rustc` found +outside `$CARGO_HOME/bin` and `~/.cargo/bin` with the pinned version is a +warning, since distribution and Nix toolchains are legitimate; a different +version fails. Optional tools (`ruff`, `shellcheck`, `dash`, `actionlint`, +`cargo-audit`, `cargo-deny`) that are absent are listed under `skipped`, and +`scripts/test.sh` reports each as the skip it printed before this script existed, +except `dash`, which only chooses the shell that checks syntax and has no skip. + +The exit status is 0 when every required check passed, 1 when one failed, and 2 +for an invalid invocation or a repository the script cannot read. The desktop +app's system packages (the apt list in `check.yml`) are not checked: a missing +one surfaces as that build's own error, except WebKitGTK on Linux, whose absence +makes `scripts/test.sh` skip the desktop gate and say so; CI installs it, and a +skip fails the run there. + +rustup installs the toolchain a `rust-toolchain.toml` pins the first time it +runs in that directory, and any rustup command counts, `rustup toolchain list` +included. So with rustup on PATH the check asks which toolchains are installed +from `/`, where no pin applies, before any `cargo` or `rustc` runs in the +repository. When the pinned toolchain is missing it fails with `rustup +toolchain install --profile minimal -c clippy -c rustfmt` and runs +neither, and a `RUSTUP_TOOLCHAIN` override likewise stops them from running. +It also exports `RUSTUP_AUTO_INSTALL=0`, which a rustup that honours it turns +into a refusal to install; the check does not depend on it. Without rustup (a +distribution or Nix toolchain) the version comparison decides. + ## Formatting `scripts/indent.sh` holds the formatter chain: comment reflow with diff --git a/scripts/env-check.sh b/scripts/env-check.sh new file mode 100755 index 000000000..d1d25ab73 --- /dev/null +++ b/scripts/env-check.sh @@ -0,0 +1,443 @@ +#!/bin/sh + +# The one place that says whether this machine can run the gate, so a missing +# tool is named once, before the run starts, with the command that fixes it, +# instead of surfacing as the same build-script error in every gate that needs +# it. scripts/test.sh runs it as its first step and takes its optional-tool skip +# lines from it. +# +# It only looks. It never downloads or installs anything and never runs +# scripts/fetch-clang.sh or scripts/verify-vendor.sh: scripts/test.sh provisions +# the compiler itself and already hashes the vendored assets once per run, which +# a second pass over twelve megabytes of wasm would only repeat. +# +# Exit status: 0 every required check passed, 1 one failed, 2 a bad invocation +# or a repository this script cannot read. +set -eu + +usage() +{ + echo "usage: ${0##*/} [--profile local-rust|ci-rust|portable|browser]" \ + "[--format text|json] [--will-provision]" >&2 + exit 2 +} + +# Resolved as scripts/fetch-clang.sh does, and entered so that `rustc -V` and +# `cargo -V` see the repository's rust-toolchain.toml exactly as they will when +# scripts/test.sh runs them. +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" + +profile=local-rust +format=text +provision=false +while [ "$#" -gt 0 ]; do + case "$1" in + --profile) + [ "$#" -ge 2 ] || usage + profile=$2 + shift 2 + ;; + --format) + [ "$#" -ge 2 ] || usage + format=$2 + shift 2 + ;; + + # Passed only by scripts/test.sh, which fetches Clang itself after these + # checks; run by hand, a missing compiler is still a failure. + --will-provision) + provision=true + shift + ;; + *) usage ;; + esac +done +case "$profile" in + local-rust | ci-rust | portable | browser) ;; + *) usage ;; +esac +case "$format" in + text | json) ;; + *) usage ;; +esac + +# rustup installs the toolchain a rust-toolchain.toml names the first time any +# rustup command runs inside that directory, `rustup toolchain list` included. +# So whether the pinned toolchain exists is asked from outside the repository, +# before any cargo or rustc runs in it, and nothing runs when it is missing. +# This setting stops the same install on a rustup that honours it; the check +# above does not depend on it. +RUSTUP_AUTO_INSTALL=0 +export RUSTUP_AUTO_INSTALL +toolchain_ready=true + +# The byte whose hex value is $1, written out as is: printf takes it as an octal +# escape in its format, which is the portable way to emit one byte. +raw_byte() +{ + octal=$(printf '%03o' "0x$1") + # shellcheck disable=SC2059 + printf "\\$octal" +} + +# Every JSON string goes through here. Nothing is rejected: a quote, a backslash +# and every control character are escaped, so a hostile environment variable +# cannot break out of the object that reports it. +json_string() +{ + value=$1 + + # The plain case needs two substitutions; only a value that carries a + # control character pays for the byte-by-byte walk. + case $value in + *[[:cntrl:]]*) ;; + *) + printf '"%s"' "$(printf '%s' "$value" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g')" + return 0 + ;; + esac + escaped= + for byte in $(printf '%s' "$value" | od -An -v -tx1); do + case $byte in + 22) piece='\"' ;; + 5c) piece="\\\\" ;; + 0a) piece='\n' ;; + 7f) piece='\u007f' ;; + 09) piece='\t' ;; + 0d) piece='\r' ;; + 0[0-9a-f] | 1[0-9a-f]) piece="\\u00$byte" ;; + + # The byte itself, for printable ASCII and for the bytes of a + # multi-byte character, which JSON carries as they are. + *) piece=$(raw_byte "$byte") ;; + esac + escaped=$escaped$piece + done + printf '"%s"' "$escaped" +} + +kernel=$(uname -s) +failed=false +required_json= +skipped_json= + +say() +{ + [ "$format" = text ] || return 0 + printf '%s\n' "$*" +} + +# required NAME OK FOUND WANT MESSAGE +required() +{ + if [ "$2" = true ]; then + say "ok: $1: $3" + else + failed=true + say "fail: $1: $5" + fi + [ "$format" = json ] || return 0 + entry="{\"name\":$(json_string "$1"),\"ok\":$2,\"found\":$(json_string "$3"),\"want\":$(json_string "$4")}" + required_json=${required_json:+$required_json,}$entry +} + +skipped() +{ + say "skipped: $1: $2" + [ "$format" = json ] || return 0 + entry=$(json_string "$1: $2") + skipped_json=${skipped_json:+$skipped_json,}$entry +} + +have() +{ + command -v "$1" > /dev/null 2>&1 +} + +rust_install_hint() +{ + case "$kernel" in + MINGW* | MSYS* | CYGWIN*) + printf '%s' "download and run rustup-init.exe from https://rustup.rs, then run ./scripts/env-check.sh again" + ;; + *) + printf '%s' "install Rust with: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh, then run ./scripts/env-check.sh again" + ;; + esac +} + +# The pin is read the way scripts/test.sh reads other files, with sed: the +# gate's own Python is not required for the browser lane, and this script should +# not need it either. +pinned_channel() +{ + sed -n '/^[[:space:]]*channel[[:space:]]*=/{s/^[^=]*=[[:space:]]*"\([^"]*\)".*/\1/p;q;}' \ + "$ROOT/rust-toolchain.toml" 2> /dev/null || : +} + +# With rustup on PATH, whether the pinned toolchain is installed, from a +# directory that has no pin so the question itself installs nothing. Without +# rustup (a distribution or Nix toolchain) the version check below decides. +check_pinned_toolchain() +{ + channel=$(pinned_channel) + if [ -z "$channel" ]; then + echo "${0##*/}: no channel in rust-toolchain.toml; cannot tell which toolchain is wanted" >&2 + exit 2 + fi + have rustup || return 0 + if ! installed=$(cd / && rustup toolchain list); then + echo "${0##*/}: rustup toolchain list failed" >&2 + exit 2 + fi + found=false + while read -r name _; do + case $name in + "$channel" | "$channel"-*) found=true ;; + esac + done << LIST +$installed +LIST + if [ "$found" = true ]; then + required toolchain true "$channel is installed" "$channel installed" "" + else + toolchain_ready=false + required toolchain false "$channel is not installed" "$channel installed" \ + "the toolchain $channel that rust-toolchain.toml pins is not installed; run: rustup toolchain install $channel --profile minimal -c clippy -c rustfmt, then run ./scripts/env-check.sh again" + fi +} + +check_cargo() +{ + if ! have cargo; then + required cargo false "not found" "on PATH" "not found; $(rust_install_hint)" + return 0 + fi + if [ -n "${RUSTUP_TOOLCHAIN:-}" ]; then + required RUSTUP_TOOLCHAIN false "$RUSTUP_TOOLCHAIN" unset \ + "set to $RUSTUP_TOOLCHAIN, which overrides the rust-toolchain.toml pin; unset it" + # A rustup shim would act on the override, which may not be installed. + if have rustup; then + toolchain_ready=false + fi + else + required RUSTUP_TOOLCHAIN true "unset" unset "" + fi + [ "$toolchain_ready" = true ] || return 0 + if cargo_line=$(cargo -V 2>&1); then + required cargo true "$cargo_line" "on PATH" "" + else + required cargo false "$cargo_line" "on PATH" \ + "cargo -V failed: $cargo_line" + fi +} + +check_rustc() +{ + have cargo || return 0 + [ "$toolchain_ready" = true ] || return 0 + if ! have rustc; then + required rustc false "not found" "$channel" "not found; $(rust_install_hint)" + return 0 + fi + rustc_path=$(command -v rustc) + # The failure text starts with "rustc", so it must not reach the parse. + rustc_ran=true + if rustc_line=$(rustc -V 2>&1); then + rustc_version=$(printf '%s\n' "$rustc_line" | sed -n '1s/^rustc \([^ ]*\).*/\1/p') + else + rustc_line="rustc -V failed: $rustc_line" + rustc_version= + rustc_ran=false + fi + case $channel in + [0-9]*.[0-9]*.[0-9]*) + if [ "$rustc_version" = "$channel" ]; then + required rustc true "$rustc_version" "$channel" "" + else + required rustc false "${rustc_version:-$rustc_line}" "$channel" \ + "${rustc_version:+rustc }${rustc_version:-$rustc_line} is not the $channel that rust-toolchain.toml pins; run from the repository root, or install it with: rustup toolchain install $channel" + fi + ;; + *) + # A named channel such as nightly has no number to compare. + if [ "$rustc_ran" = true ]; then + required rustc true "${rustc_version:-$rustc_line}" "$channel" "" + else + required rustc false "$rustc_line" "$channel" "$rustc_line" + fi + ;; + esac + + # A distribution or Nix compiler is legitimate when it is the pinned + # version; only the version decides the verdict. + outside=true + for home in "${CARGO_HOME:-}" "${HOME:-}/.cargo"; do + [ -n "$home" ] || continue + case $rustc_path in + "$home"/bin/*) outside=false ;; + esac + done + if [ "$outside" = true ] && [ "$rustc_version" = "$channel" ]; then + say "warning: rustc is $rustc_path, not a rustup shim under \$CARGO_HOME/bin or ~/.cargo/bin; the pin is met by version only" + fi +} + +check_component() +{ + # $1 the cargo subcommand, $2 the component rustup names it by. + have cargo || return 0 + [ "$toolchain_ready" = true ] || return 0 + if cargo "$1" --version > /dev/null 2>&1; then + required "$2" true "cargo $1 --version" "installed" "" + else + required "$2" false "missing" "installed" \ + "cargo $1 is not available; run: rustup component add clippy rustfmt" + fi +} + +check_compiler() +{ + # webrtc-sys needs Clang 21 or later on Linux only; another platform builds + # with its own compiler, whose version number says nothing here. + [ "$kernel" = Linux ] || return 0 + cxx=${CXX:-} + cxx_fetched=false + if [ -z "$cxx" ]; then + if [ "$(uname -m)" = x86_64 ]; then + if [ -x "$ROOT/target/clang/bin/clang++" ]; then + cxx=$ROOT/target/clang/bin/clang++ + cxx_fetched=true + elif [ "$provision" = true ]; then + required CXX true "not set; scripts/test.sh fetches Clang into target/clang" "Clang 21+" "" + say "note: Clang 21+ is not on disk yet; scripts/test.sh fetches it before the build" + return 0 + else + required CXX false "not set" "Clang 21+" \ + "no Clang 21+; run: sh scripts/fetch-clang.sh target/clang" + return 0 + fi + else + required CXX false "not set" "Clang 21+" \ + "only Linux x86_64 has a fetched Clang; set CXX to a Clang 21+ instead" + return 0 + fi + fi + + # A CXX the caller set may carry arguments (a launcher in front of the + # compiler), so it is split on purpose; the path found under the checkout is + # one word, whatever the checkout's directory is called. + if [ "$cxx_fetched" = true ]; then + cxx_line=$("$cxx" --version 2> /dev/null | sed -n 1p) || cxx_line= + else + # shellcheck disable=SC2086 + cxx_line=$($cxx --version 2> /dev/null | sed -n 1p) || cxx_line= + fi + cxx_major=$(printf '%s\n' "$cxx_line" | sed -n 's/.*clang version \([0-9][0-9]*\).*/\1/p') + if [ -n "$cxx_major" ] && [ "$cxx_major" -ge 21 ]; then + required CXX true "$cxx: clang $cxx_major" "Clang 21+" "" + else + required CXX false "$cxx: ${cxx_line:-no version reported}" "Clang 21+" \ + "$cxx is not Clang 21+ (${cxx_line:-no version reported}); set CXX to one, or unset it on Linux x86_64 to use sh scripts/fetch-clang.sh target/clang" + fi +} + +check_vendor() +{ + if [ -d "$ROOT/web/vendor" ]; then + required web/vendor true present present "" + else + required web/vendor false missing present \ + "web/vendor is missing, and the server embeds it; restore it from the checkout" + fi +} + +check_node() +{ + if have node; then + required node true "$(command -v node)" "on PATH" "" + else + required node false "not found" "on PATH" "not found; install Node.js, which runs the generated-file and browser checks" + fi +} + +check_ci_linker() +{ + if have mold; then + required mold true "$(command -v mold)" "on PATH" "" + else + required mold false "not found" "on PATH" "not found; install mold, which the workflow's RUSTFLAGS links with" + fi + case ${RUSTFLAGS:-} in + *-fuse-ld=mold*) required RUSTFLAGS true "${RUSTFLAGS:-}" "-fuse-ld=mold" "" ;; + *) required RUSTFLAGS false "${RUSTFLAGS:-}" "-fuse-ld=mold" "does not link with mold; the workflow sets -Clink-arg=-fuse-ld=mold" ;; + esac +} + +# `docker info` and not just the client: a client with no daemon behind it +# cannot run the actionlint image, as scripts/test.sh's own fallback decides. +docker_usable() +{ + have docker && docker info > /dev/null 2>&1 +} + +# The wording is what scripts/test.sh used to print when it probed these tools +# itself, so a skipped check reads the same. +check_optional() +{ + have ruff || skipped ruff "installing it lints the Python under scripts/, tests/ and the skills" + if ! have shellcheck; then + set -- "$ROOT"/scripts/*.sh "$ROOT"/.claude/skills/*/*.sh "$ROOT"/tests/stubs/*/*.sh + skipped shellcheck "installing it checks what $# shell scripts mean, not just that they parse" + fi + have dash || skipped dash "\`sh -n\` runs under the system sh, which may accept what dash refuses" + if ! have actionlint && ! docker_usable; then + skipped actionlint "installing it, or docker, checks .github/workflows" + fi + have cargo-audit || skipped cargo-audit "\`cargo install cargo-audit\` audits the dependency tree" + have cargo-deny || skipped cargo-deny "\`cargo install cargo-deny\` checks dependency licenses and sources" +} + +# What each lane of scripts/test.sh really needs, read off its gate list. Node +# runs generated-file checks in every lane. The portable lane also checks Rust +# formatting, so it needs rustfmt but neither clippy nor a compiler. +case "$profile" in + local-rust | ci-rust) + check_pinned_toolchain + check_cargo + check_rustc + check_component clippy clippy + check_component fmt rustfmt + check_compiler + check_vendor + check_node + if [ "$profile" = ci-rust ]; then + check_ci_linker + fi + ;; + portable) + check_pinned_toolchain + check_cargo + check_component fmt rustfmt + check_node + ;; + browser) + check_node + ;; +esac +check_optional + +if [ "$failed" = true ]; then + ok=false +else + ok=true +fi +if [ "$format" = json ]; then + printf '{"profile":%s,"ok":%s,"required":[%s],"skipped":[%s]}\n' \ + "$(json_string "$profile")" "$ok" "$required_json" "$skipped_json" +else + say "env-check: profile $profile: $([ "$ok" = true ] && echo ok || echo failed)" +fi +[ "$ok" = true ] || exit 1 +exit 0 diff --git a/scripts/test.sh b/scripts/test.sh index 377995dcc..f633b6d03 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -19,6 +19,31 @@ esac ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +# What this machine can run is settled before anything else. `gate` only records +# a failure and carries on, so this is a plain step: a missing toolchain stops +# the run here, naming the tool and the command that installs it, instead of +# failing every gate that needs it. It runs before the Python lookup below, +# which can set up an environment, and before the Clang fetch, which is why it +# is told that fetch will happen. The report it prints for a passing machine +# also carries the optional tools that are absent; the gates below read their +# skip lines from it. +case "$lane" in + rust | all) + env_profile=local-rust + [ "${CI:-}" != true ] || env_profile=ci-rust + ;; + portable) env_profile=portable ;; + browser) env_profile=browser ;; +esac +env_check=$ROOT/scripts/env-check.sh +env_report=$("$env_check" --profile "$env_profile" --will-provision) || { + env_status=$? + printf '%s\n' "$env_report" >&2 + exit "$env_status" +} +# A passing report is otherwise discarded, and with it the warnings and notes. +printf '%s\n' "$env_report" | grep -E '^(warning|note): ' >&2 + # The tree this run judges, taken before any gate runs: an edit made while a # long run is in progress then fails to match the receipt, instead of being # stamped as what the run saw. Empty outside a git work tree. @@ -103,6 +128,21 @@ skip() $1" } +# env_absent TOOL: whether scripts/env-check.sh found TOOL missing. +env_absent() +{ + printf '%s\n' "$env_report" | grep -q "^skipped: $1: " +} + +# env_skip TOOL: record the skip with the reason scripts/env-check.sh gave, and +# succeed only when TOOL was missing. +env_skip() +{ + reason=$(printf '%s\n' "$env_report" | sed -n "s/^skipped: $1: //p") + [ -n "$reason" ] || return 1 + skip "$1: $reason" +} + # Run a command that reports its own skips on stderr, and add each one the sed # script $2 extracts to this run's summary under label $1. Its stderr is shown # once the command finishes. @@ -305,6 +345,10 @@ eslint_gate() # Both absent is still a skip, on the same contract as eslint above. actionlint_gate() { + if env_skip actionlint; then + return 0 + fi + if command -v actionlint > /dev/null 2>&1; then (cd "$ROOT" && actionlint) return "$?" @@ -340,8 +384,7 @@ release_msrv_matches() cargo_audit_gate() { - if ! command -v cargo-audit > /dev/null 2>&1; then - skip "cargo-audit: \`cargo install cargo-audit\` audits the dependency tree" + if env_skip cargo-audit; then return 0 fi @@ -352,8 +395,7 @@ cargo_audit_gate() cargo_deny_gate() { - if ! command -v cargo-deny > /dev/null 2>&1; then - skip "cargo-deny: \`cargo install cargo-deny\` checks dependency licenses and sources" + if env_skip cargo-deny; then return 0 fi @@ -409,8 +451,7 @@ vendor_patches_gate() # until the script runs. Optional the way shellcheck is, for the same reason. ruff_gate() { - if ! command -v ruff > /dev/null 2>&1; then - skip "ruff: installing it lints the Python under scripts/, tests/ and the skills" + if env_skip ruff; then return 0 fi @@ -435,20 +476,16 @@ ruff_gate() shell_syntax() { status=0 - scripts=0 posix_sh="sh" - if command -v dash > /dev/null 2>&1; then + if ! env_absent dash; then posix_sh=dash fi for script in "$ROOT"/scripts/*.sh "$ROOT"/.claude/skills/*/*.sh "$ROOT"/tests/stubs/*/*.sh; do - scripts=$((scripts + 1)) "$posix_sh" -n "$script" || status=1 done - if command -v shellcheck > /dev/null 2>&1; then + if ! env_skip shellcheck; then (cd "$ROOT" && shellcheck scripts/*.sh .claude/skills/*/*.sh tests/stubs/*/*.sh) || status=1 - else - skip "shellcheck: installing it checks what $scripts shell scripts mean, not just that they parse" fi return "$status" @@ -626,6 +663,7 @@ gate audit-tests unittest_gate "$ROOT/tests/test_audit.py" gate rust-style-tests unittest_gate "$ROOT/tests/test_check_rust_style.py" gate ceiling-hotspots-tests unittest_gate "$ROOT/tests/test_ceiling_hotspots.py" gate test-gate-tests unittest_gate "$ROOT/tests/test_test_gate.py" +gate env-check-tests unittest_gate "$ROOT/tests/test_env_check.py" gate mutants-plan-tests unittest_gate "$ROOT/tests/test_mutants_plan.py" gate mutants-install-tests unittest_gate "$ROOT/tests/test_mutants_install.py" gate latency-tests unittest_gate "$ROOT/tests/test_analyze_latency.py" diff --git a/tests/stubs/env-check/cargo.sh b/tests/stubs/env-check/cargo.sh new file mode 100644 index 000000000..cd50af416 --- /dev/null +++ b/tests/stubs/env-check/cargo.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# FAKE_FORBID_RUN names a file this stub appends to and then fails, for a case +# that must prove the tool was never run. +[ -z "${FAKE_FORBID_RUN:-}" ] || { + echo ran >> "$FAKE_FORBID_RUN" + exit 99 +} + +# Stands in for cargo. FAKE_CARGO_VERSION is the number `-V` reports and +# FAKE_CARGO_MISSING names the subcommands whose `--version` fails, the way a +# toolchain installed without those components does. FAKE_CARGO_BROKEN makes +# `-V` itself fail. +case "$1" in + -V) + [ -z "${FAKE_CARGO_BROKEN:-}" ] || exit 1 + echo "cargo ${FAKE_CARGO_VERSION:-1.100.0} (stub)" + ;; + clippy | fmt) + case " ${FAKE_CARGO_MISSING:-} " in + *" $1 "*) + echo "error: no such command: \`$1\`" >&2 + exit 101 + ;; + esac + echo "cargo-$1 ${FAKE_CARGO_VERSION:-1.100.0} (stub)" + ;; + *) + exit 101 + ;; +esac diff --git a/tests/stubs/env-check/clang.sh b/tests/stubs/env-check/clang.sh new file mode 100644 index 000000000..901af3135 --- /dev/null +++ b/tests/stubs/env-check/clang.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# Stands in for a C++ compiler; FAKE_CLANG_VERSION is the major version it +# reports, in the shape both Clang and Chromium's build print. +echo "clang version ${FAKE_CLANG_VERSION:-21}.1.0 (stub)" diff --git a/tests/stubs/env-check/rustc.sh b/tests/stubs/env-check/rustc.sh new file mode 100644 index 000000000..b49627ac2 --- /dev/null +++ b/tests/stubs/env-check/rustc.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# FAKE_FORBID_RUN names a file this stub appends to and then fails, for a case +# that must prove the tool was never run. +[ -z "${FAKE_FORBID_RUN:-}" ] || { + echo ran >> "$FAKE_FORBID_RUN" + exit 99 +} + +# Stands in for rustc; FAKE_RUSTC_VERSION is the number `-V` reports and +# FAKE_RUSTC_BROKEN makes `-V` fail and FAKE_RUSTC_ODD makes it succeed with a +# line that has no version. +[ -z "${FAKE_RUSTC_BROKEN:-}" ] || { + echo "rustc is broken" >&2 + exit 1 +} +[ -z "${FAKE_RUSTC_ODD:-}" ] || { + echo "a wrapper banner" + exit 0 +} +echo "rustc ${FAKE_RUSTC_VERSION:-1.100.0} (0123abcd 2026-01-01)" diff --git a/tests/stubs/env-check/rustup.sh b/tests/stubs/env-check/rustup.sh new file mode 100644 index 000000000..38f6a0c23 --- /dev/null +++ b/tests/stubs/env-check/rustup.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Stands in for rustup. Only `toolchain list` is understood, and like the real +# one it must not be asked from a directory with a pin, where rustup would try +# to install the pinned toolchain. FAKE_RUSTUP_LIST is what it prints and +# FAKE_RUSTUP_BROKEN makes it fail. +[ "$*" = "toolchain list" ] || exit 2 +if [ -f rust-toolchain.toml ]; then + echo "rustup was run inside a directory with a pin" >&2 + exit 1 +fi +[ -z "${FAKE_RUSTUP_BROKEN:-}" ] || exit 1 +printf '%s\n' "${FAKE_RUSTUP_LIST:-}" diff --git a/tests/stubs/env-check/tool.sh b/tests/stubs/env-check/tool.sh new file mode 100644 index 000000000..b3942558e --- /dev/null +++ b/tests/stubs/env-check/tool.sh @@ -0,0 +1,3 @@ +#!/bin/sh +# Stands in for a tool whose only job here is to be found on PATH. +exit 0 diff --git a/tests/stubs/env-check/uname.sh b/tests/stubs/env-check/uname.sh new file mode 100644 index 000000000..4210ea1ca --- /dev/null +++ b/tests/stubs/env-check/uname.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Stands in for uname on a host the suite chooses, so a case does not depend on +# the machine it runs on. +kernel=${FAKE_UNAME_S:-Linux} +machine=${FAKE_UNAME_M:-x86_64} +case "${1:-}" in + -s) echo "$kernel" ;; + -m) echo "$machine" ;; + *) echo "$kernel $machine" ;; +esac diff --git a/tests/support.py b/tests/support.py index 5bea165ab..c0e7337a0 100644 --- a/tests/support.py +++ b/tests/support.py @@ -8,6 +8,7 @@ import functools import io import re +import shutil import subprocess from pathlib import Path @@ -16,6 +17,23 @@ STUBS = ROOT / "tests/stubs" +# The tools a script under test needs from the system, and nothing else: a PATH +# that holds only these cannot reach whatever toolchain the machine has. +CHECKER_TOOLS = ("sh", "sed", "grep", "tr", "od", "dirname", "cat", "wc", "uname") + + +def link_system_tools(directory, names=CHECKER_TOOLS): + """Symlink each named system tool into directory; return what is missing.""" + missing = [] + for name in names: + found = shutil.which(name) + if found is None: + missing.append(name) + else: + (Path(directory) / name).symlink_to(found) + return missing + + def stub(name, interpreter=None): """The source of a fake executable kept under tests/stubs/. diff --git a/tests/test_env_check.py b/tests/test_env_check.py new file mode 100644 index 000000000..cec9f8ae4 --- /dev/null +++ b/tests/test_env_check.py @@ -0,0 +1,502 @@ +"""Exercise scripts/env-check.sh against stub toolchains. + +Every case runs the script copied into a scratch repository, with a PATH that +holds only stubs and the few system tools the script itself needs, so the +verdicts never depend on the Rust, Node or Clang installed on the machine that +runs the suite. +""" + +import importlib.util +import json +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +SPEC = importlib.util.spec_from_file_location( + "support", Path(__file__).with_name("support.py") +) +support = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(support) + +SCRIPT = support.ROOT / "scripts/env-check.sh" +# The system tools the script may use; nothing that could pull the real +# toolchain in is linked. +SYSTEM_TOOLS = ("sed", "grep", "tr", "od", "dirname", "cat", "wc") +OPTIONAL = ("ruff", "shellcheck", "dash", "actionlint", "cargo-audit", "cargo-deny") +INSTALL_UNIX = "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh" +AGAIN = "./scripts/env-check.sh" + + +class Sandbox: + """A scratch repository, a home directory and a PATH of stubs.""" + + def __init__(self, tmp): + self.tmp = Path(tmp) + self.repo = self.tmp / "repo" + self.home = self.tmp / "home" + self.cargo_bin = self.home / ".cargo/bin" + self.system = self.tmp / "system" + for directory in ( + self.repo / "scripts", + self.repo / "web/vendor", + self.cargo_bin, + self.system, + ): + directory.mkdir(parents=True) + shutil.copy(SCRIPT, self.repo / "scripts/env-check.sh") + self.pin("1.100.0") + missing = support.link_system_tools(self.system, SYSTEM_TOOLS) + if missing: + raise unittest.SkipTest(f"{missing} not installed") + self.stub("uname", "uname.sh", self.system) + self.stub("cargo", "cargo.sh", self.cargo_bin) + self.stub("rustc", "rustc.sh", self.cargo_bin) + self.stub("node", "tool.sh", self.system) + self.clang = self.tmp / "clang/bin/clang++" + self.clang.parent.mkdir(parents=True) + self.stub_at(self.clang, "clang.sh") + + def pin(self, channel): + (self.repo / "rust-toolchain.toml").write_text( + f'[toolchain]\nchannel = "{channel}"\ncomponents = ["clippy", "rustfmt"]\n' + ) + + def stub_at(self, path, source): + support.write_executable(path, support.stub(f"env-check/{source}")) + + def stub(self, name, source, directory): + self.stub_at(directory / name, source) + + def remove(self, name, directory=None): + (directory or self.system).joinpath(name).unlink() + + def run(self, *args, path=None, env=None, remove_cxx=False): + entries = path if path is not None else [self.cargo_bin, self.system] + environment = { + "PATH": os.pathsep.join(str(entry) for entry in entries), + "HOME": str(self.home), + "CXX": str(self.clang), + } + if remove_cxx: + del environment["CXX"] + environment.update(env or {}) + return subprocess.run( + ["/bin/sh", str(self.repo / "scripts/env-check.sh"), *args], + cwd=self.repo, + env=environment, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + + +class EnvCheckCase(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.box = Sandbox(directory.name) + + def assertFailed(self, result, *needles): + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + for needle in needles: + self.assertIn(needle, result.stdout) + + def lines(self, result, prefix): + return [line for line in result.stdout.splitlines() if line.startswith(prefix)] + + +class LocalRust(EnvCheckCase): + def test_a_complete_toolchain_passes(self): + result = self.box.run() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.lines(result, "fail:"), []) + + def test_missing_cargo_prints_the_install_command(self): + self.box.remove("cargo", self.box.cargo_bin) + result = self.box.run() + self.assertFailed(result, "fail: cargo", INSTALL_UNIX, AGAIN) + + def test_windows_points_at_rustup_init(self): + self.box.remove("cargo", self.box.cargo_bin) + result = self.box.run(env={"FAKE_UNAME_S": "MINGW64_NT-10.0"}) + self.assertFailed(result, "fail: cargo", "rustup-init.exe", AGAIN) + self.assertNotIn("sh.rustup.rs", result.stdout) + + def test_a_version_other_than_the_pin_fails_with_both_numbers(self): + result = self.box.run(env={"FAKE_RUSTC_VERSION": "1.99.0"}) + self.assertFailed(result, "fail: rustc", "1.99.0", "1.100.0") + + def test_a_failing_rustc_fails_for_a_named_pin_too(self): + self.box.pin("nightly") + result = self.box.run(env={"FAKE_RUSTC_BROKEN": "1"}) + self.assertFailed(result, "fail: rustc", "rustc -V failed") + + def test_a_named_pin_accepts_a_rustc_line_with_no_number(self): + self.box.pin("nightly") + result = self.box.run(env={"FAKE_RUSTC_ODD": "1"}) + self.assertEqual(result.returncode, 0, result.stdout) + self.assertIn("ok: rustc", result.stdout) + + def test_rustup_toolchain_overrides_the_pin(self): + result = self.box.run(env={"RUSTUP_TOOLCHAIN": "nightly"}) + self.assertFailed(result, "fail: RUSTUP_TOOLCHAIN", "nightly") + + def test_a_matching_rustc_outside_cargo_home_only_warns(self): + self.box.remove("rustc", self.box.cargo_bin) + elsewhere = self.box.tmp / "distro" + elsewhere.mkdir() + self.box.stub("rustc", "rustc.sh", elsewhere) + path = [self.box.cargo_bin, elsewhere, self.box.system] + result = self.box.run(path=path) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(len(self.lines(result, "warning:")), 1) + self.assertIn("rustc", self.lines(result, "warning:")[0]) + + def test_a_mismatched_rustc_outside_cargo_home_still_fails(self): + self.box.remove("rustc", self.box.cargo_bin) + elsewhere = self.box.tmp / "distro" + elsewhere.mkdir() + self.box.stub("rustc", "rustc.sh", elsewhere) + path = [self.box.cargo_bin, elsewhere, self.box.system] + result = self.box.run(path=path, env={"FAKE_RUSTC_VERSION": "1.88.0"}) + self.assertFailed(result, "fail: rustc", "1.88.0") + + def test_cargo_home_names_where_the_shims_live(self): + shims = self.box.tmp / "elsewhere/bin" + shims.parent.mkdir() + self.box.cargo_bin.rename(shims) + result = self.box.run( + path=[shims, self.box.system], + env={"CARGO_HOME": str(shims.parent)}, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.lines(result, "warning:"), []) + + def test_a_missing_component_prints_the_rustup_command(self): + for missing in ("clippy", "fmt"): + with self.subTest(missing=missing): + result = self.box.run(env={"FAKE_CARGO_MISSING": missing}) + self.assertFailed(result, "rustup component add clippy rustfmt") + + def test_a_cargo_that_cannot_report_its_version_fails(self): + result = self.box.run(env={"FAKE_CARGO_BROKEN": "1"}) + self.assertFailed(result, "fail: cargo") + + def test_a_missing_pin_file_is_an_indeterminate_result(self): + (self.box.repo / "rust-toolchain.toml").unlink() + result = self.box.run() + self.assertEqual(result.returncode, 2, result.stdout) + self.assertIn("rust-toolchain.toml", result.stderr) + + def test_vendored_assets_must_be_present(self): + shutil.rmtree(self.box.repo / "web/vendor") + result = self.box.run() + self.assertFailed(result, "fail: web/vendor") + + +class Rustup(EnvCheckCase): + """With rustup on PATH nothing may run in the repository before the pinned + toolchain is known to be installed, since rustup installs it on first use.""" + + INSTALLED = ( + "stable-x86_64-unknown-linux-gnu (default)\n" + "1.99.1-x86_64-unknown-linux-gnu\n" + "1.100.0-x86_64-unknown-linux-gnu (active)" + ) + INSTALL = "rustup toolchain install 1.100.0 --profile minimal -c clippy -c rustfmt" + + def setUp(self): + super().setUp() + self.box.stub("rustup", "rustup.sh", self.box.cargo_bin) + self.marker = self.box.tmp / "ran" + + def run_with(self, listing, *args, env=None): + environment = { + "FAKE_RUSTUP_LIST": listing, + "FAKE_FORBID_RUN": str(self.marker), + } + return self.box.run(*args, env=environment | (env or {})) + + def test_an_installed_pin_lets_the_checks_run(self): + result = self.box.run(env={"FAKE_RUSTUP_LIST": self.INSTALLED}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("ok: toolchain: 1.100.0 is installed", result.stdout) + self.assertIn("ok: rustc: 1.100.0", result.stdout) + + def test_a_missing_pin_fails_without_running_cargo_or_rustc(self): + for profile in ("local-rust", "portable"): + with self.subTest(profile=profile): + listing = "stable-x86_64-unknown-linux-gnu (default)\n1.99.1-x86_64" + result = self.run_with(listing, "--profile", profile) + self.assertFailed(result, "fail: toolchain", self.INSTALL, AGAIN) + self.assertFalse(self.marker.exists()) + + def test_a_missing_pin_is_reported_in_json_too(self): + result = self.run_with("stable-x86_64-unknown-linux-gnu", "--format", "json") + self.assertEqual(result.returncode, 1) + report = json.loads(result.stdout) + entry = next(e for e in report["required"] if e["name"] == "toolchain") + self.assertIs(entry["ok"], False) + self.assertFalse(self.marker.exists()) + + def test_a_longer_name_does_not_stand_in_for_the_pin(self): + result = self.run_with("1.100.01-x86_64-unknown-linux-gnu\n11.100.0-x86_64") + self.assertFailed(result, "fail: toolchain") + self.assertFalse(self.marker.exists()) + + def test_a_named_channel_matches_by_its_full_name(self): + self.box.pin("nightly-2026-01-01") + listing = "nightly-2026-01-01-x86_64-unknown-linux-gnu" + found = self.box.run("--profile", "portable", env={"FAKE_RUSTUP_LIST": listing}) + self.assertEqual(found.returncode, 0, found.stdout) + self.assertFailed( + self.run_with("nightly-x86_64-unknown-linux-gnu", "--profile", "portable"), + "fail: toolchain", + ) + + def test_an_override_stops_the_shims_from_running(self): + result = self.run_with(self.INSTALLED, env={"RUSTUP_TOOLCHAIN": "nightly"}) + self.assertFailed(result, "fail: RUSTUP_TOOLCHAIN") + self.assertFalse(self.marker.exists()) + + def test_a_failing_rustup_is_an_indeterminate_result(self): + result = self.box.run(env={"FAKE_RUSTUP_BROKEN": "1"}) + self.assertEqual(result.returncode, 2, result.stdout) + self.assertTrue(result.stderr.strip()) + + def test_the_browser_profile_never_asks_rustup(self): + result = self.run_with("", "--profile", "browser") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertFalse(self.marker.exists()) + + +class Compiler(EnvCheckCase): + def test_clang_21_is_accepted_and_20_is_not(self): + self.assertEqual(self.box.run().returncode, 0) + result = self.box.run(env={"FAKE_CLANG_VERSION": "20"}) + self.assertFailed(result, "fail: CXX", "20") + + def test_a_cxx_that_is_not_clang_is_refused(self): + self.box.stub_at(self.box.clang, "tool.sh") + self.assertFailed(self.box.run(), "fail: CXX") + + def test_a_linux_host_without_a_fetched_clang_must_name_a_compiler(self): + machine = {"FAKE_UNAME_S": "Linux", "FAKE_UNAME_M": "aarch64"} + result = self.box.run(env=machine, remove_cxx=True) + self.assertFailed(result, "set CXX to a Clang 21+") + + def test_another_platform_builds_with_its_own_compiler(self): + # webrtc-sys asks for Clang 21 only on Linux; macOS builds with the + # system compiler, whatever version number it reports. + machine = {"FAKE_UNAME_S": "Darwin", "FAKE_UNAME_M": "arm64"} + result = self.box.run(env=machine, remove_cxx=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("CXX", result.stdout) + + def test_linux_without_a_compiler_prints_the_fetch_command(self): + result = self.box.run(remove_cxx=True) + self.assertFailed(result, "sh scripts/fetch-clang.sh target/clang") + self.assertFalse((self.box.repo / "target").exists()) + + def test_will_provision_turns_the_missing_compiler_into_a_note(self): + result = self.box.run("--will-provision", remove_cxx=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(len(self.lines(result, "note:")), 1) + self.assertFalse((self.box.repo / "target").exists()) + + def test_will_provision_does_not_excuse_a_linux_host_that_cannot_fetch(self): + result = self.box.run( + "--will-provision", + env={"FAKE_UNAME_S": "Linux", "FAKE_UNAME_M": "aarch64"}, + remove_cxx=True, + ) + self.assertFailed(result, "set CXX to a Clang 21+") + + def test_a_checkout_path_with_a_space_still_finds_the_fetched_compiler(self): + spaced = self.box.tmp / "Code Trial" + shutil.copytree(self.box.repo, spaced) + fetched = spaced / "target/clang/bin" + fetched.mkdir(parents=True) + self.box.stub_at(fetched / "clang++", "clang.sh") + path = os.pathsep.join((str(self.box.cargo_bin), str(self.box.system))) + result = subprocess.run( + ["/bin/sh", str(spaced / "scripts/env-check.sh")], + cwd=spaced, + env={"PATH": path, "HOME": str(self.box.home)}, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_the_fetched_compiler_is_used_when_cxx_is_unset(self): + fetched = self.box.repo / "target/clang/bin" + fetched.mkdir(parents=True) + self.box.stub_at(fetched / "clang++", "clang.sh") + result = self.box.run(remove_cxx=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.lines(result, "note:"), []) + + +class Optional(EnvCheckCase): + def test_absent_tools_are_skipped_not_failed(self): + result = self.box.run() + self.assertEqual(result.returncode, 0) + skipped = self.lines(result, "skipped:") + for tool in OPTIONAL: + with self.subTest(tool=tool): + self.assertEqual( + len([line for line in skipped if f": {tool}: " in line]), 1 + ) + + def test_present_tools_are_not_listed(self): + for tool in OPTIONAL: + self.box.stub(tool, "tool.sh", self.box.system) + self.box.stub("docker", "tool.sh", self.box.system) + result = self.box.run() + self.assertEqual(result.returncode, 0) + self.assertEqual(self.lines(result, "skipped:"), []) + + def test_dash_alone_being_absent_is_reported_and_does_not_fail(self): + for tool in OPTIONAL: + if tool != "dash": + self.box.stub(tool, "tool.sh", self.box.system) + self.box.stub("docker", "tool.sh", self.box.system) + result = self.box.run() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual( + self.lines(result, "skipped:"), + [ + "skipped: dash: `sh -n` runs under the system sh, which may " + "accept what dash refuses" + ], + ) + + def test_the_skip_reasons_keep_the_wording_the_gate_printed(self): + result = self.box.run() + self.assertIn( + "skipped: ruff: installing it lints the Python under scripts/, tests/ " + "and the skills", + result.stdout.splitlines(), + ) + self.assertIn( + "skipped: cargo-audit: `cargo install cargo-audit` audits the " + "dependency tree", + result.stdout.splitlines(), + ) + + +class Profiles(EnvCheckCase): + def test_portable_needs_rustfmt_and_node_but_no_clippy_or_compiler(self): + result = self.box.run( + "--profile", + "portable", + env={"FAKE_CARGO_MISSING": "clippy", "FAKE_RUSTC_VERSION": "1.1.0"}, + remove_cxx=True, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + result = self.box.run( + "--profile", "portable", env={"FAKE_CARGO_MISSING": "fmt"} + ) + self.assertFailed(result, "rustup component add") + + def test_browser_needs_node_and_nothing_else(self): + self.box.remove("cargo", self.box.cargo_bin) + self.box.remove("rustc", self.box.cargo_bin) + shutil.rmtree(self.box.repo / "web/vendor") + result = self.box.run("--profile", "browser", remove_cxx=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.box.remove("node") + self.assertFailed(self.box.run("--profile", "browser"), "fail: node") + + def test_ci_rust_needs_mold_and_its_linker_flag(self): + flags = "-Clink-arg=-fuse-ld=mold" + self.box.stub("mold", "tool.sh", self.box.system) + ok = self.box.run("--profile", "ci-rust", env={"RUSTFLAGS": flags}) + self.assertEqual(ok.returncode, 0, ok.stdout + ok.stderr) + self.assertFailed(self.box.run("--profile", "ci-rust"), "fail: RUSTFLAGS") + self.box.remove("mold") + self.assertFailed( + self.box.run("--profile", "ci-rust", env={"RUSTFLAGS": flags}), + "fail: mold", + ) + + def test_local_rust_does_not_ask_for_mold(self): + self.assertEqual(self.box.run("--profile", "local-rust").returncode, 0) + + def test_unknown_profile_flag_and_format_are_invalid_invocations(self): + for args in ( + ("--profile", "gpu"), + ("--profile",), + ("--format", "xml"), + ("--nope",), + ): + with self.subTest(args=args): + result = self.box.run(*args) + self.assertEqual(result.returncode, 2, result.stdout) + self.assertTrue(result.stderr.strip()) + + +class Json(EnvCheckCase): + def report(self, *args, env=None, **options): + result = self.box.run("--format", "json", *args, env=env, **options) + return result, json.loads(result.stdout) + + def test_stdout_is_one_object_and_nothing_else(self): + result, report = self.report() + self.assertEqual(result.returncode, 0) + self.assertEqual(len(result.stdout.splitlines()), 1) + self.assertEqual(set(report), {"profile", "ok", "required", "skipped"}) + self.assertEqual(report["profile"], "local-rust") + self.assertIs(report["ok"], True) + for entry in report["required"]: + self.assertEqual(set(entry), {"name", "ok", "found", "want"}) + names = [entry["name"] for entry in report["required"]] + self.assertIn("cargo", names) + self.assertTrue(any(item.startswith("ruff: ") for item in report["skipped"])) + + def test_a_failure_is_reported_and_exits_one(self): + self.box.remove("cargo", self.box.cargo_bin) + result, report = self.report() + self.assertEqual(result.returncode, 1) + self.assertIs(report["ok"], False) + failed = [entry for entry in report["required"] if not entry["ok"]] + self.assertIn("cargo", [entry["name"] for entry in failed]) + + def test_hostile_values_come_out_as_escaped_json(self): + for hostile in ( + 'x"\n{', + "tab\there", + "ctl\x01\x02\x1f\x7f end", + "back\\slash éα", + "carriage\rreturn", + 'slow \\ " path \x01 é', + ): + with self.subTest(hostile=hostile): + result, report = self.report(env={"RUSTUP_TOOLCHAIN": hostile}) + self.assertEqual(result.returncode, 1) + entry = next( + item + for item in report["required"] + if item["name"] == "RUSTUP_TOOLCHAIN" + ) + self.assertEqual(entry["found"], hostile) + self.assertIs(entry["ok"], False) + # Every control character is escaped, DEL included, not only + # the ones JSON insists on. + raw = result.stdout.rstrip("\n") + self.assertFalse([c for c in raw if ord(c) < 32 or ord(c) == 127]) + + def test_a_hostile_rustc_version_is_escaped_too(self): + result, report = self.report(env={"FAKE_RUSTC_VERSION": 'v"\x01'}) + self.assertEqual(result.returncode, 1) + found = [e["found"] for e in report["required"] if e["name"] == "rustc"] + self.assertTrue(found and 'v"\x01' in found[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_test_gate.py b/tests/test_test_gate.py index fe966cb30..8ee38c409 100644 --- a/tests/test_test_gate.py +++ b/tests/test_test_gate.py @@ -108,6 +108,7 @@ def run_lane( unset_cxx=False, replace=(), env=None, + real_env_check=False, ): source = SCRIPT.read_text() named = env or {} @@ -123,6 +124,12 @@ def run_lane( ): env["PYTHON"] = sys.executable replace = (("\"$1\" -c 'import cryptography'", "true"), *replace) + if not real_env_check: + # The lane dispatch is under test here, not this machine's toolchain. + replace = ( + ("env_check=$ROOT/scripts/env-check.sh", "env_check=true"), + *replace, + ) for old, new in replace: self.assertEqual(source.count(old), 1, old) source = source.replace(old, new) @@ -275,6 +282,153 @@ def test_new_gate_is_included_in_portable_lane(self): self.assertIn("future-check", self.run_lane("portable", extra_gate=True)[1]) self.assertNotIn("future-check", self.run_lane("rust", extra_gate=True)[1]) + def test_a_missing_cargo_stops_a_rust_run_before_any_gate(self): + # `gate` only records a failure and carries on, so the environment check + # is a plain step: a machine with no toolchain must not first sit through + # every gate that would each fail on the same missing tool. + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(support.link_system_tools(tmp), []) + result, gates = self.run_lane( + "rust", env={"PATH": tmp}, real_env_check=True + ) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(gates, []) + self.assertIn("cargo", result.stderr) + self.assertNotIn("failed gates:", result.stderr) + + def test_the_pre_step_picks_a_profile_per_lane(self): + for lane, ci, profile in ( + ("rust", False, "local-rust"), + ("all", False, "local-rust"), + ("rust", True, "ci-rust"), + ("portable", True, "portable"), + ("portable", False, "portable"), + ("browser", False, "browser"), + ): + with self.subTest(lane=lane, ci=ci): + self.assertEqual( + self.env_check_arguments(lane, ci), + f"--profile {profile} --will-provision\n", + ) + + def env_check_arguments(self, lane, ci): + """What test.sh passes the checker, read off a stub that records it.""" + with tempfile.TemporaryDirectory() as tmp: + checker = Path(tmp) / "env-check.sh" + support.write_executable( + checker, '#!/bin/sh\nprintf "%s\\n" "$*" > "$OUT"\n' + ) + out = Path(tmp) / "out" + result, _ = self.run_lane( + lane, + ci=ci, + env={"OUT": str(out)}, + replace=(("env_check=true", f"env_check={checker}"),), + ) + self.assertEqual(result.returncode, 0, result.stderr) + return out.read_text() + + def test_a_missing_optional_tool_skips_with_the_checkers_wording(self): + report = ( + "skipped: ruff: installing it lints the Python under scripts/, " + "tests/ and the skills\n" + "skipped: cargo-deny: `cargo install cargo-deny` checks dependency " + "licenses and sources" + ) + script = "\n".join( + [ + "ROOT=/nonexistent", + 'skip() { echo "skipping $1" >&2; }', + f"env_report={shlex.quote(report)}", + *( + f"{name}() {{\n{function_body(name)}\n}}" + for name in ( + "env_absent", + "env_skip", + "ruff_gate", + "cargo_deny_gate", + ) + ), + "ruff_gate; cargo_deny_gate", + "env_absent dash && echo no-dash", + "env_absent ruff && echo no-ruff", + ] + ) + result = subprocess.run( + ["sh", "-c", script], capture_output=True, text=True, check=False + ) + self.assertEqual( + result.stderr.splitlines(), + [ + "skipping ruff: installing it lints the Python under scripts/, " + "tests/ and the skills", + "skipping cargo-deny: `cargo install cargo-deny` checks " + "dependency licenses and sources", + ], + ) + self.assertEqual(result.stdout.split(), ["no-ruff"]) + + def test_every_absent_optional_tool_skips_with_its_original_wording(self): + # The real checker's report, on a PATH with none of the optional tools, + # fed to the real gate functions: what the gate printed before the + # probes moved into scripts/env-check.sh, word for word. + shell_scripts = ( + len(list(ROOT.glob("scripts/*.sh"))) + + len(list(ROOT.glob(".claude/skills/*/*.sh"))) + + len(list(ROOT.glob("tests/stubs/*/*.sh"))) + ) + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(support.link_system_tools(tmp), []) + bare = {"PATH": tmp, "HOME": tmp} + report = subprocess.run( + ["/bin/sh", str(ROOT / "scripts/env-check.sh"), "--profile", "browser"], + env=bare, + capture_output=True, + text=True, + check=False, + ).stdout + names = ( + "env_absent", + "env_skip", + "ruff_gate", + "cargo_audit_gate", + "cargo_deny_gate", + "actionlint_gate", + "shell_syntax", + ) + script = "\n".join( + [ + f"ROOT={shlex.quote(str(ROOT))}", + 'skip() { echo "skipping $1" >&2; }', + f"env_report={shlex.quote(report)}", + *(f"{name}() {{\n{function_body(name)}\n}}" for name in names), + "ruff_gate; cargo_audit_gate; cargo_deny_gate", + "actionlint_gate; shell_syntax", + ] + ) + result = subprocess.run( + ["/bin/sh", "-c", script], + env=bare, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual( + result.stderr.splitlines(), + [ + "skipping ruff: installing it lints the Python under scripts/, " + "tests/ and the skills", + "skipping cargo-audit: `cargo install cargo-audit` audits the " + "dependency tree", + "skipping cargo-deny: `cargo install cargo-deny` checks " + "dependency licenses and sources", + "skipping actionlint: installing it, or docker, checks " + ".github/workflows", + f"skipping shellcheck: installing it checks what {shell_scripts} " + "shell scripts mean, not just that they parse", + ], + ) + def test_failure_does_not_hide_later_gates(self): for lane, failures in ( ("rust", "clippy cargo-test"), From 8c1f07965f5f9778ca55d5e5be6d86413378a36b Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 17 Nov 2026 02:09:35 +0800 Subject: [PATCH 04/13] Name the tree a gate receipt's run would become A receipt carried only a fingerprint that changes on every rebase or amend, so nothing could tell whether a later commit holds what the run saw, and the next run on any branch overwrote the lane file. Add worktree_tree, the tree id a commit of the same content gets. It is computed in a scratch index, so the real index and partial staging are never touched, and it is taken at the start and the end of the run and kept only when both agree, so a tree edited during the run claims nothing, though an edit made and then undone cannot be seen. The receipt also records rustc, cargo and the environment report, and a copy is kept per tree, newest 20, because the lane file is overwritten by the next run, possibly on another branch. A tree folder written within the last hour is never pruned, since another run in the clone may have just written it and nothing locks the folder. The development guide describes each receipt field. The extractor the suites use to load a shell function is shared in tests/support.py, and a closeout check test pins that the new fields and the per-tree folder do not break reading older receipts. Change-Id: I3351b78479dc59aca590898343e88765e1d4139f --- docs/development.md | 23 ++ scripts/test.sh | 114 +++++++++- tests/support.py | 28 +++ tests/test_agent_workflow_hooks.py | 25 +++ tests/test_gate_receipt.py | 346 +++++++++++++++++++++++++++++ tests/test_test_gate.py | 55 +---- 6 files changed, 543 insertions(+), 48 deletions(-) create mode 100644 tests/test_gate_receipt.py diff --git a/docs/development.md b/docs/development.md index f58a26c28..5fbcda328 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1007,6 +1007,29 @@ count and `scripts/tree-fingerprint.sh` of the tree, live under `.git/codetrial-review/gate-receipts/`. A receipt shows that a run happened on this tree, not that it was complete; CI recomputes the verdict. +A receipt is one JSON line: `lane`, `passed`, `failed`, `skipped`, `tree` (the +fingerprint above, which changes on every rebase or amend), `worktree_tree`, +`rustc` and `cargo` (their `-V` output, empty for the browser lane, which needs +no Rust), `env` (the report of `scripts/env-check.sh`, captured once at the +start; `null` when it could not be) and `time`. + +`worktree_tree` is the id a commit of the same content gets. It is computed in +a scratch index (the real one, and any partial staging in it, is never read): +`HEAD` plus everything not ignored, as `git add -A` would stage it, with the +repository's attributes and the contributor's filters applied as a commit +applies them. A run on a dirty tree therefore matches the commit made from it +afterwards, provided that commit stages everything the run saw; a commit of +part of the tree has another id. The value is taken when the run starts and +again when it ends and is written only when both agree, so a tree edited during +the run claims nothing and the field is empty (it is empty too outside a git +work tree or on an unborn `HEAD`). An edit made and then undone during the run +cannot be seen. When it is set, the receipt is also kept as +`gate-receipts/by-tree//.json`, because the lane file is +overwritten by the next run, possibly on another branch; the 20 most recently +written trees are kept and older ones are removed, except that a tree written +within the last hour is never removed, because another run in the same clone may +have just written it and nothing locks the folder. + ## Review memory and the review skill `/codetrial-review` in Claude Code, and `$codetrial-review` in Codex (which diff --git a/scripts/test.sh b/scripts/test.sh index f633b6d03..cfbbc4b43 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -49,6 +49,50 @@ printf '%s\n' "$env_report" | grep -E '^(warning|note): ' >&2 # stamped as what the run saw. Empty outside a git work tree. receipt_tree=$("$ROOT/scripts/tree-fingerprint.sh" 2> /dev/null) || receipt_tree= +# worktree_tree: the id of the tree a commit would get if everything the work +# tree holds that git does not ignore were staged, which is what lets evidence +# keyed by tree id be matched to a commit. It is built in a scratch index so the +# real one, and with it any partial staging, is never read or changed. The index +# path is exported once, absolute: an assignment in front of `a && b` applies to +# `a` alone. Prints nothing and fails outside a git work tree, on an unborn +# HEAD, or when any step fails. +worktree_tree() +{ + ( + cd "$ROOT" || exit 1 + + # A relative TMPDIR would resolve under ROOT, inside the tree being + # named. + tmp=${TMPDIR:-/tmp} + case $tmp in /*) ;; *) tmp=/tmp ;; esac + scratch=$(mktemp -d "$tmp/codetrial-index.XXXXXX") || exit 1 + trap 'rm -f "$scratch"/*; rmdir "$scratch"' EXIT + trap 'exit 1' HUP INT TERM + GIT_INDEX_FILE=$scratch/index + export GIT_INDEX_FILE + git -c core.fsmonitor=false read-tree HEAD > /dev/null 2>&1 \ + && git -c core.fsmonitor=false add -A > /dev/null 2>&1 \ + && git write-tree 2> /dev/null + ) +} + +# What a receipt records about the run, taken once here and not at the end: the +# tree id (compared with the one at the end), the toolchain and the environment +# report. The browser lane needs no Rust, and rustup would install a pinned +# toolchain on the first rustc it runs, so that lane asks for neither. +worktree_start= +env_json= +rustc_version= +cargo_version= +if [ -n "$receipt_tree" ]; then + worktree_start=$(worktree_tree) || worktree_start= + env_json=$("$env_check" --profile "$env_profile" --will-provision --format json 2> /dev/null) || env_json= + if [ "$env_profile" != browser ]; then + rustc_version=$(cd "$ROOT" && RUSTUP_AUTO_INSTALL=0 rustc -V 2> /dev/null) || rustc_version= + cargo_version=$(cd "$ROOT" && RUSTUP_AUTO_INSTALL=0 cargo -V 2> /dev/null) || cargo_version= + fi +fi + # The gate needs Python 3.11 for the Rust checker's standard-library TOML # parser. A distribution's python3 can be older than that while a newer one sits # beside it, and running the old one fails every Python gate on the interpreter @@ -664,6 +708,7 @@ gate rust-style-tests unittest_gate "$ROOT/tests/test_check_rust_style.py" gate ceiling-hotspots-tests unittest_gate "$ROOT/tests/test_ceiling_hotspots.py" gate test-gate-tests unittest_gate "$ROOT/tests/test_test_gate.py" gate env-check-tests unittest_gate "$ROOT/tests/test_env_check.py" +gate gate-receipt-tests unittest_gate "$ROOT/tests/test_gate_receipt.py" gate mutants-plan-tests unittest_gate "$ROOT/tests/test_mutants_plan.py" gate mutants-install-tests unittest_gate "$ROOT/tests/test_mutants_install.py" gate latency-tests unittest_gate "$ROOT/tests/test_analyze_latency.py" @@ -810,6 +855,42 @@ gate report-parity-fixtures env REPORT_PARITY_CHECK_VALIDATE_FIXTURES_ONLY=1 sh fi } +# json_escape TEXT: TEXT as the inside of a JSON string. Control characters go +# and the quote and the backslash are escaped; a version string comes from a +# tool the receipt does not control. +json_escape() +{ + printf '%s' "$1" | tr -d '[:cntrl:]' | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' +} + +# prune_by_tree DIR CURRENT: keeps the 20 newest tree folders of DIR. CURRENT is +# never removed, whatever the clock says, and neither is a folder written within +# the last hour: another run in this clone may have just written it, and there +# is no lock to tell. Fresh folders still count toward the 20. Only a name that +# is a hex object id is counted or removed, so nothing else that sits there is +# touched, and only the receipt files inside are deleted. +prune_by_tree() +{ + kept=1 + + # Newest first needs the modification time, which a glob cannot order by; + # each name is checked below before it is used. + # shellcheck disable=SC2045 + for name in $(ls -t "$1" 2> /dev/null); do + case $name in *[!0-9a-f]*) continue ;; esac + [ "$name" != "$2" ] || continue + [ "${#name}" -eq 40 ] || [ "${#name}" -eq 64 ] || continue + # A link is never followed: its target is not ours to empty. + [ -d "$1/$name" ] || continue + [ ! -L "$1/$name" ] || continue + kept=$((kept + 1)) + [ "$kept" -gt 20 ] || continue + [ -z "$(find "$1/$name" -prune -mmin -60 2> /dev/null)" ] || continue + rm -f "$1/$name"/*.json 2> /dev/null + rmdir "$1/$name" 2> /dev/null || true + done +} + # A receipt names the lane, the verdict and the tree it was run on, so a claim # that "the gate passed" can be matched to the tree it is about instead of being # taken on trust: a closeout (experience-close.sh) and the Stop hook @@ -817,6 +898,12 @@ gate report-parity-fixtures env REPORT_PARITY_CHECK_VALIDATE_FIXTURES_ONLY=1 sh # the next run, kept in the clone's git directory where it is never committed. A # tree that cannot be fingerprinted writes none. Skipped checks are counted, not # hidden: a run that skipped something is a weaker receipt. +# +# `worktree_tree` is the id a commit of the same content gets. It is taken at +# the start and again here and kept only when the two agree, so a tree edited +# during the run claims nothing; an edit made and reverted in between is not +# seen. A copy is kept per tree, because the lane file is overwritten by the +# next run, possibly on another branch. write_receipt() { [ -n "$receipt_tree" ] || return 0 @@ -825,9 +912,30 @@ write_receipt() dir=$dir/codetrial-review/gate-receipts mkdir -p "$dir" 2> /dev/null || return 0 skips=$(printf '%s\n' "$skipped" | grep -c '[^[:space:]]') - printf '{"lane":"%s","passed":%s,"failed":"%s","skipped":%s,"tree":"%s","time":"%s"}\n' \ - "$lane" "$1" "${failed# }" "$skips" "$receipt_tree" \ - "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$dir/$lane.json" 2> /dev/null || true + tree_now=$(worktree_tree) || tree_now= + case $tree_now in *[!0-9a-f]*) tree_now= ;; esac + [ "$tree_now" = "$worktree_start" ] || tree_now= + case $env_json in + *[[:cntrl:]]*) env_json=null ;; + '{'*'}') ;; + *) env_json=null ;; + esac + # A byte that is not UTF-8 would make the whole line unreadable as JSON. + printf '%s' "$env_json" | iconv -f UTF-8 -t UTF-8 > /dev/null 2>&1 || env_json=null + line=$(printf '{"lane":"%s","passed":%s,"failed":"%s","skipped":%s,"tree":"%s","worktree_tree":"%s","rustc":"%s","cargo":"%s","env":%s,"time":"%s"}' \ + "$lane" "$1" "$(json_escape "${failed# }")" "$skips" "$receipt_tree" \ + "$tree_now" "$(json_escape "$rustc_version")" "$(json_escape "$cargo_version")" \ + "$env_json" "$(date -u +%Y-%m-%dT%H:%M:%SZ)") + printf '%s\n' "$line" > "$dir/$lane.json" 2> /dev/null || true + [ -n "$tree_now" ] || return 0 + by_tree=$dir/by-tree + mkdir -p "$by_tree/$tree_now" 2> /dev/null || return 0 + printf '%s\n' "$line" > "$by_tree/$tree_now/$lane.json" 2> /dev/null || true + + # Rewriting a file in place leaves the folder's age alone, and the age is + # what decides which trees are the newest. + touch "$by_tree/$tree_now" 2> /dev/null || true + prune_by_tree "$by_tree" "$tree_now" } [ -z "$failed" ] || { diff --git a/tests/support.py b/tests/support.py index c0e7337a0..65bc9df62 100644 --- a/tests/support.py +++ b/tests/support.py @@ -7,6 +7,7 @@ import functools import io +import os import re import shutil import subprocess @@ -16,6 +17,27 @@ WORKFLOW = ROOT / ".github/workflows/check.yml" STUBS = ROOT / "tests/stubs" +# A scratch repository must not read the developer's git configuration, nor be +# redirected by a GIT_DIR or GIT_INDEX_FILE inherited from a hook. +GIT_ENV = { + **{k: v for k, v in os.environ.items() if not k.startswith("GIT_")}, + "GIT_CONFIG_GLOBAL": "/dev/null", + "GIT_CONFIG_SYSTEM": "/dev/null", +} +COMMIT = "git -c user.name=t -c user.email=t@t commit -q" + + +def sh(script, cwd): + """Run `script` under sh in `cwd` with the isolated git environment.""" + return subprocess.run( + ["sh", "-c", script], + cwd=cwd, + capture_output=True, + text=True, + env=GIT_ENV, + check=False, + ) + # The tools a script under test needs from the system, and nothing else: a PATH # that holds only these cannot reach whatever toolchain the machine has. @@ -82,6 +104,12 @@ def section(text, start, end, what): return text.split(start, 1)[1].split(end, 1)[0] +def function_source(text, name): + """A whole shell function of a script, as the script defines it.""" + body = section(text, f"\n{name}()\n{{\n", "\n}\n", f"function {name}") + return f"{name}()\n{{\n{body}\n}}\n" + + @functools.cache def workflow_text(): return WORKFLOW.read_text() diff --git a/tests/test_agent_workflow_hooks.py b/tests/test_agent_workflow_hooks.py index 6ad48b763..c97045d73 100644 --- a/tests/test_agent_workflow_hooks.py +++ b/tests/test_agent_workflow_hooks.py @@ -165,6 +165,31 @@ def test_status_requires_every_lane_on_this_exact_tree(self): self.assertTrue(covered) self.assertIn("all (3 skipped)", text) + def test_receipts_with_the_tree_id_and_toolchain_fields_are_still_read(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + folder = Path(tmp.name) + receipt = ran( + "all", + worktree_tree="a" * 40, + rustc="rustc 1.0.0", + cargo="cargo 1.0.0", + env={"profile": "local-rust", "ok": True, "required": [], "skipped": []}, + ) + (folder / "all.json").write_text(json.dumps(receipt)) + # The per-tree copies sit beside the lane files and are not lane receipts. + (folder / "by-tree" / ("a" * 40)).mkdir(parents=True) + (folder / "by-tree" / ("a" * 40) / "all.json").write_text(json.dumps(receipt)) + # A private copy: the other cases patch the shared module's globals while + # they run in parallel with this one. + own = load("agent_closeout_check_files", "agent-closeout-check.py") + with ( + mock.patch.object(own, "tree", return_value="T1"), + mock.patch.object(own, "receipts_folder", return_value=folder), + ): + self.assertEqual(own.receipts(), [receipt]) + self.assertTrue(own.status()[0]) + def test_a_later_lane_failure_beats_an_earlier_all_pass(self): covered, text = self.status( [ diff --git a/tests/test_gate_receipt.py b/tests/test_gate_receipt.py new file mode 100644 index 000000000..9441546c3 --- /dev/null +++ b/tests/test_gate_receipt.py @@ -0,0 +1,346 @@ +"""The gate receipt's tree id and toolchain fields, run as scripts/test.sh defines them.""" + +import importlib.util +import json +import os +import shlex +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +SPEC = importlib.util.spec_from_file_location( + "support", Path(__file__).with_name("support.py") +) +support = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(support) + +ROOT = support.ROOT +SOURCE = (ROOT / "scripts/test.sh").read_text() +ENV_JSON = '{"profile":"portable","ok":true,"required":[],"skipped":[]}' + + +def function_source(name): + return support.function_source(SOURCE, name) + + +class ReceiptTests(unittest.TestCase): + def repo(self, files=None): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + root = Path(tmp.name) + for name, body in {"a.txt": "a\n", **(files or {})}.items(): + (root / name).write_text(body) + done = support.sh( + f"git init -q -b main . && git add -A && {support.COMMIT} -m x", root + ) + self.assertEqual(done.returncode, 0, done.stderr) + return root + + def run_gate(self, root, during="", verdict="true", **setup): + """Take the start tree, run `during`, then write the receipt, in `set -u` sh.""" + defaults = { + "lane": "rust", + "failed": "", + "skipped": "", + "receipt_tree": "T1", + "rustc_version": "rustc 1.0.0", + "cargo_version": "cargo 1.0.0", + "env_json": ENV_JSON, + } + assigns = "; ".join( + f"{k}={shlex.quote(v)}" for k, v in {**defaults, **setup}.items() + ) + functions = "".join( + function_source(name) + for name in ( + "worktree_tree", + "prune_by_tree", + "json_escape", + "write_receipt", + ) + ) + script = ( + f"set -u; ROOT={shlex.quote(str(root))}; {assigns}\n{functions}\n" + "worktree_start=$(worktree_tree) || worktree_start=\n" + f"{during}\nwrite_receipt {verdict}\n" + ) + done = support.sh(script, root) + self.assertEqual(done.returncode, 0, done.stderr) + return done + + def folder(self, root): + return root / ".git/codetrial-review/gate-receipts" + + def receipt(self, root, lane="rust"): + return json.loads((self.folder(root) / f"{lane}.json").read_text()) + + def head_tree(self, root): + return support.sh("git rev-parse 'HEAD^{tree}'", root).stdout.strip() + + def test_a_clean_tree_is_the_head_tree(self): + root = self.repo() + self.run_gate(root) + self.assertEqual(self.receipt(root)["worktree_tree"], self.head_tree(root)) + + def test_a_dirty_tree_committed_unchanged_matches_the_commit(self): + root = self.repo() + (root / "a.txt").write_text("changed\n") + (root / "new.txt").write_text("new\n") + self.run_gate(root) + seen = self.receipt(root)["worktree_tree"] + self.assertNotEqual(seen, self.head_tree(root)) + support.sh(f"git add -A && {support.COMMIT} -m y", root) + self.assertEqual(seen, self.head_tree(root)) + + def test_partial_staging_does_not_change_it(self): + root = self.repo() + (root / "a.txt").write_text("changed\n") + (root / "new.txt").write_text("new\n") + self.run_gate(root) + plain = self.receipt(root)["worktree_tree"] + support.sh("git add new.txt", root) + self.run_gate(root) + self.assertEqual(self.receipt(root)["worktree_tree"], plain) + + def test_the_real_index_is_left_alone(self): + root = self.repo() + (root / "new.txt").write_text("new\n") + before = support.sh("git status --porcelain=v1 && git ls-files -s", root).stdout + self.run_gate(root) + self.assertEqual( + support.sh("git status --porcelain=v1 && git ls-files -s", root).stdout, + before, + ) + self.assertIn("?? new.txt", before) + + def test_untracked_counts_and_ignored_does_not(self): + root = self.repo({".gitignore": "skip.txt\n"}) + self.run_gate(root) + base = self.receipt(root)["worktree_tree"] + (root / "skip.txt").write_text("ignored\n") + self.run_gate(root) + self.assertEqual(self.receipt(root)["worktree_tree"], base) + (root / "kept.txt").write_text("untracked\n") + self.run_gate(root) + self.assertNotEqual(self.receipt(root)["worktree_tree"], base) + + def test_a_change_during_the_run_leaves_it_empty(self): + root = self.repo() + self.run_gate(root, during="echo edited >> a.txt") + receipt = self.receipt(root) + self.assertEqual(receipt["worktree_tree"], "") + self.assertEqual(receipt["tree"], "T1") + self.assertFalse((self.folder(root) / "by-tree").exists()) + + def test_the_old_keys_stay_and_the_toolchain_is_carried(self): + root = self.repo() + self.run_gate(root, verdict="false", failed=" fmt clippy", skipped="\n x\n y") + text = (self.folder(root) / "rust.json").read_text() + self.assertEqual(len(text.splitlines()), 1) + receipt = json.loads(text) + self.assertEqual( + {k: receipt[k] for k in ("lane", "passed", "failed", "skipped", "tree")}, + { + "lane": "rust", + "passed": False, + "failed": "fmt clippy", + "skipped": 2, + "tree": "T1", + }, + ) + self.assertRegex(receipt["time"], r"^\d{4}-\d\d-\d\dT[\d:]+Z$") + self.assertEqual( + (receipt["rustc"], receipt["cargo"]), ("rustc 1.0.0", "cargo 1.0.0") + ) + self.assertEqual(receipt["env"], json.loads(ENV_JSON)) + + def test_version_strings_are_escaped(self): + root = self.repo() + self.run_gate(root, rustc_version='rustc "x" \\ y\tz', env_json="not json") + receipt = self.receipt(root) + self.assertEqual(receipt["rustc"], 'rustc "x" \\ yz') + self.assertIsNone(receipt["env"]) + + def test_a_report_that_is_not_utf8_is_not_embedded(self): + # A byte no JSON reader accepts would make the whole receipt unreadable, + # so a passing run would leave no evidence at all. + root = self.repo() + bad = 'env_json="{\\"a\\":\\"$(printf \'\\377\')\\"}"' + self.run_gate(root, during=bad) + self.assertIsNone(self.receipt(root)["env"]) + + def test_a_utf8_report_is_embedded(self): + root = self.repo() + self.run_gate(root, env_json='{"found":"/home/\u00e9"}') + self.assertEqual(self.receipt(root)["env"], {"found": "/home/\u00e9"}) + + def test_each_receipt_is_kept_by_tree(self): + root = self.repo() + self.run_gate(root) + self.run_gate(root, lane="portable") + tree = self.receipt(root)["worktree_tree"] + keep = self.folder(root) / "by-tree" / tree + self.assertEqual( + sorted(p.name for p in keep.iterdir()), ["portable.json", "rust.json"] + ) + self.assertEqual( + json.loads((keep / "rust.json").read_text()), self.receipt(root) + ) + + def test_only_the_newest_twenty_trees_are_kept(self): + root = self.repo() + by_tree = self.folder(root) / "by-tree" + by_tree.mkdir(parents=True) + for index in range(25): + old = by_tree / f"{index:040x}" + old.mkdir() + (old / "rust.json").write_text("{}") + os.utime(old, (1_000_000 + index, 1_000_000 + index)) + # Not a tree id: left alone, and not counted. + (by_tree / "keep-me").mkdir() + (by_tree / "keep-me" / "x").write_text("x") + self.run_gate(root) + names = sorted(p.name for p in by_tree.iterdir()) + trees = [n for n in names if n != "keep-me"] + self.assertEqual(len(trees), 20) + self.assertIn("keep-me", names) + self.assertNotIn(f"{0:040x}", trees) + self.assertIn(f"{24:040x}", trees) + self.assertIn(self.receipt(root)["worktree_tree"], trees) + + def prune(self, ages, current): + """Run prune_by_tree over one tree folder per entry of `ages` (its mtime, + or None for now), named by index, and return the indexes that survive.""" + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + by_tree = Path(tmp.name) + for index, age in enumerate(ages): + folder = by_tree / f"{index:040x}" + folder.mkdir() + (folder / "rust.json").write_text("{}") + if age is not None: + os.utime(folder, (age, age)) + script = ( + f"set -u\n{function_source('prune_by_tree')}\n" + f"prune_by_tree {shlex.quote(str(by_tree))} {current:040x}\n" + ) + done = support.sh(script, by_tree) + self.assertEqual(done.returncode, 0, done.stderr) + return {int(p.name, 16) for p in by_tree.iterdir()} + + def test_fresh_trees_and_the_newest_old_ones_survive(self): + # Three folders written now and 22 old ones, older with the lower index; + # the current tree is a fresh one. + kept = self.prune([None] * 3 + [1_000_000 + i for i in range(22)], 0) + self.assertEqual(kept, {0, 1, 2} | set(range(8, 25))) + + def test_a_folder_written_in_the_last_hour_is_never_pruned(self): + # Another run may have just written it, so even a 23rd stays. + self.assertEqual(len(self.prune([None] * 23, 0)), 23) + + def test_the_current_tree_survives_when_every_folder_has_the_same_age(self): + # The current tree sorts last among equals, so only its own rule saves it. + kept = self.prune([1_000_000] * 26, 25) + self.assertIn(25, kept) + self.assertEqual(len(kept), 20) + + def test_a_rerun_on_an_old_tree_keeps_it_from_the_next_prune(self): + root = self.repo() + by_tree = self.folder(root) / "by-tree" + self.run_gate(root) + again = by_tree / self.receipt(root)["worktree_tree"] + # Long ago, with 25 folders newer than it. + os.utime(again, (1_000_000, 1_000_000)) + for index in range(25): + newer = by_tree / f"{index:040x}" + newer.mkdir() + os.utime(newer, (2_000_000 + index, 2_000_000 + index)) + # Rewriting the receipt files in place leaves a folder's mtime alone, so + # only the writer's own refresh makes this tree look recent again. + self.run_gate(root) + (root / "other.txt").write_text("another tree\n") + self.run_gate(root) + self.assertNotEqual(self.receipt(root)["worktree_tree"], again.name) + self.assertTrue(again.exists()) + + def test_a_link_among_the_trees_is_not_followed(self): + root = self.repo() + by_tree = self.folder(root) / "by-tree" + by_tree.mkdir(parents=True) + outside = Path(tempfile.mkdtemp(dir=root.parent)) + self.addCleanup( + lambda: support.sh(f"rm -rf {shlex.quote(str(outside))}", root.parent) + ) + (outside / "example.json").write_text("{}") + link = by_tree / f"{0:040x}" + link.symlink_to(outside) + os.utime(link, (1_000_000, 1_000_000), follow_symlinks=False) + for index in range(1, 22): + folder = by_tree / f"{index:040x}" + folder.mkdir() + os.utime(folder, (1_000_000 + index, 1_000_000 + index)) + self.run_gate(root) + self.assertTrue((outside / "example.json").exists()) + + def test_a_relative_tmpdir_leaves_nothing_in_the_work_tree(self): + root = self.repo() + script = ( + f"set -u; ROOT={shlex.quote(str(root))}; TMPDIR=.\n" + f"{function_source('worktree_tree')}\nworktree_tree\n" + ) + done = support.sh(script, root) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual(support.sh("git status --porcelain", root).stdout, "") + # An index kept under the work tree would be part of the tree it names. + self.assertEqual(done.stdout.strip(), self.head_tree(root)) + + def test_outside_a_work_tree_there_is_no_receipt_and_no_error(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.run_gate(Path(tmp.name), receipt_tree="") + self.assertEqual(list(Path(tmp.name).iterdir()), []) + + def test_a_work_tree_without_a_head_writes_no_tree_id(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + root = Path(tmp.name) + support.sh("git init -q -b main .", root) + self.run_gate(root, receipt_tree="T1") + self.assertEqual(self.receipt(root)["worktree_tree"], "") + + +class SupportEnvironmentTests(unittest.TestCase): + def test_an_inherited_git_variable_never_reaches_a_scratch_repository(self): + # A hook runs with GIT_DIR and GIT_INDEX_FILE set; a suite started from + # one must still work on its own scratch repository. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "scratch").mkdir() + decoy = root / "decoy.git" + os.mkdir(decoy) + probe = ( + "import importlib.util, sys;" + "spec = importlib.util.spec_from_file_location('support', sys.argv[1]);" + "support = importlib.util.module_from_spec(spec);" + "spec.loader.exec_module(support);" + "done = support.sh('git init -q . && git rev-parse --git-dir', sys.argv[2]);" + "sys.stdout.write(done.stdout)" + ) + done = subprocess.run( + [sys.executable, "-c", probe, str(SPEC.origin), str(root / "scratch")], + env={ + **os.environ, + "GIT_DIR": str(decoy), + "GIT_INDEX_FILE": str(root / "decoy-index"), + }, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(done.stdout.strip(), ".git", done.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_test_gate.py b/tests/test_test_gate.py index 8ee38c409..64f72a387 100644 --- a/tests/test_test_gate.py +++ b/tests/test_test_gate.py @@ -1,7 +1,6 @@ """Exercise CI lane selection without executing the expensive gate commands.""" import importlib.util -import json import os import re import shlex @@ -316,7 +315,7 @@ def env_check_arguments(self, lane, ci): with tempfile.TemporaryDirectory() as tmp: checker = Path(tmp) / "env-check.sh" support.write_executable( - checker, '#!/bin/sh\nprintf "%s\\n" "$*" > "$OUT"\n' + checker, '#!/bin/sh\nprintf "%s\\n" "$*" >> "$OUT"\n' ) out = Path(tmp) / "out" result, _ = self.run_lane( @@ -326,7 +325,10 @@ def env_check_arguments(self, lane, ci): replace=(("env_check=true", f"env_check={checker}"),), ) self.assertEqual(result.returncode, 0, result.stderr) - return out.read_text() + # The report the gates read, then the JSON the receipt embeds. + first, second = out.read_text().splitlines() + self.assertEqual(second, f"{first} --format json") + return first + "\n" def test_a_missing_optional_tool_skips_with_the_checkers_wording(self): report = ( @@ -1203,22 +1205,13 @@ def test_unknown_lane_fails_before_running_any_gate(self): def function_source(name): """A whole shell function of scripts/test.sh, as the gate defines it.""" - return f"{name}()\n{{\n{function_body(name)}\n}}\n" + return support.function_source(SCRIPT.read_text(), name) class GateHelperTests(unittest.TestCase): - """The receipt writer and the exclusion notice, run as the gate defines them.""" + """The exclusion notice, run as the gate defines it.""" - env = { - **os.environ, - "GIT_CONFIG_GLOBAL": "/dev/null", - "GIT_CONFIG_SYSTEM": "/dev/null", - } - - def sh(self, script, cwd): - return subprocess.run( - ["sh", "-c", script], cwd=cwd, capture_output=True, text=True, env=self.env - ) + sh = staticmethod(support.sh) def repo(self, files=None): tmp = tempfile.TemporaryDirectory() @@ -1227,41 +1220,13 @@ def repo(self, files=None): for name, body in (files or {}).items(): (root / name).parent.mkdir(parents=True, exist_ok=True) (root / name).write_text(body) - commit = "git -c user.name=t -c user.email=t@t commit -q" done = self.sh( - f"git init -q -b main . && git add -A && {commit} --allow-empty -m x", root + f"git init -q -b main . && git add -A && {support.COMMIT} --allow-empty -m x", + root, ) self.assertEqual(done.returncode, 0, done.stderr) return root - def write_receipt(self, root, verdict, failed="", tree="T1", skipped=""): - script = ( - f'ROOT={shlex.quote(str(root))}; lane=portable; failed="{failed}"; receipt_tree={tree}; ' - f'skipped="{skipped}"; {function_source("write_receipt")}\nwrite_receipt {verdict}' - ) - done = self.sh(script, root) - self.assertEqual(done.returncode, 0, done.stderr) - return root / ".git/codetrial-review/gate-receipts/portable.json" - - def test_a_pass_and_a_failure_are_recorded_with_the_start_tree(self): - root = self.repo() - won = json.loads( - self.write_receipt(root, "true", skipped="\n a\n b").read_text() - ) - self.assertEqual( - (won["lane"], won["passed"], won["failed"], won["skipped"], won["tree"]), - ("portable", True, "", 2, "T1"), - ) - lost = json.loads( - self.write_receipt(root, "false", failed=" fmt clippy").read_text() - ) - self.assertEqual((lost["passed"], lost["failed"]), (False, "fmt clippy")) - - def test_no_tree_means_no_receipt(self): - root = self.repo() - path = self.write_receipt(root, "true", tree="") - self.assertFalse(path.exists()) - def exclusions(self, base, head): def config(entries): body = "".join(f' "{e}",\n' for e in entries) From 6964db771eb75511417bf0504c804dc453804e38 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 17 Nov 2026 02:09:54 +0800 Subject: [PATCH 05/13] Add a Rust-scope helper for gate evidence Which paths the Rust gate covers is decided nowhere, so every caller that needs to know whether a change requires a Rust receipt would copy its own list and the lists would drift. check-gate-evidence.py --scope-only reads paths from stdin (NUL-separated with -z, since a path can hold a newline) and prints in or out. A .github/ path is never in scope. Line mode refuses a path that ends in a carriage return, which would otherwise stop an exact name such as Cargo.toml from matching. The scope is src/, desktop/, vendor/, problem-bank/, scripts/hook-check/, .cargo/, the root manifests and toolchain files, the clippy, rustfmt and deny configs, and tests/ except the browser lane, the stubs and the top-level Python suites. web/ stays out through one named constant, apart from the few files a Rust source names. A list written from the directory layout misses files that Rust tests read from outside it. A test therefore scans the tracked Rust sources for include macros, folder attributes, CARGO_MANIFEST_DIR joins and string literals that are tracked files, and fails when the scope omits one. It tokenizes left to right so a quote inside a comment or a character literal cannot shift the literals after it, and a failing git raises instead of passing as an empty listing. Change-Id: I1343874fe42a09b12b6090bf03e9deede587b5e8 --- scripts/check-gate-evidence.py | 135 +++++++++++ scripts/test.sh | 1 + tests/test_check_gate_evidence.py | 356 ++++++++++++++++++++++++++++++ 3 files changed, 492 insertions(+) create mode 100644 scripts/check-gate-evidence.py create mode 100644 tests/test_check_gate_evidence.py diff --git a/scripts/check-gate-evidence.py b/scripts/check-gate-evidence.py new file mode 100644 index 000000000..f135fd3de --- /dev/null +++ b/scripts/check-gate-evidence.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Decide whether a change touches what the Rust gate covers. + + check-gate-evidence.py --scope-only [-z] < paths + +Paths are repository-relative, one per line, or NUL-separated with -z because +a path may hold a newline. --scope-only prints `in` when any path is in the +Rust scope and `out` otherwise; an empty list is `out`. A path under .github/ +is never in scope. It exits 0 on an answer and 2 on bad usage or a path that is +absolute, empty (a blank line is one), ends in a carriage return in line mode, +or holds a `.` or `..` component: those can name something other than what they +look like, so they are refused rather than classified. +""" + +import argparse +import sys + +# Whether web/ is in the Rust scope. The binary embeds web/ at build time, so a +# web-only change can break the Rust lane; it stays out by default so that +# JavaScript contributors need no toolchain. Flip it here and nowhere else. +WEB_IN_RUST_SCOPE = False + +# vendor/ holds the path dependencies the root manifest excludes from its +# workspace, and problem-bank/ is read by include_str! at compile time; the +# three .toml files configure the lint and license gates of the Rust lane. +RUST_PREFIXES = ( + "src/", + "desktop/", + "scripts/hook-check/", + ".cargo/", + "vendor/", + "problem-bank/", +) +RUST_FILES = frozenset( + ( + "build.rs", + "Cargo.toml", + "Cargo.lock", + "rust-toolchain.toml", + "clippy.toml", + "rustfmt.toml", + "deny.toml", + ) +) +# Files a Rust source names by literal: an include_str!, a path a test reads, a +# script it runs. tests/test_check_gate_evidence.py derives the list from the +# tracked sources, so it grows only when that test says a file is missing. The +# web/ entries are in whatever WEB_IN_RUST_SCOPE says: the constant decides +# only the embedded folder as a whole, and a change to a file a test reads can +# fail that test either way. +RUST_READ_FILES = frozenset( + ( + "docs/interview-contract-versions.md", + "docs/recording-contract.md", + "docs/task-mode.md", + "scripts/browser-check.cjs", + "scripts/browser-check.sh", + "scripts/install-git-hooks.sh", + "scripts/interview-run-budget.py", + "scripts/livekit-room-service.cjs", + "scripts/recording-cleanup.sh", + "scripts/recording-integration.sh", + "scripts/recording-provision-check.sh", + "scripts/server-check.sh", + "scripts/soak-judge.cjs", + "web/document-grounding.js", + "web/index.html", + "web/interview.html", + "web/interview.js", + "web/lib.js", + "web/problem-pages.json", + "web/runners.js", + "web/styles.css", + "web/vendor/livekit-client.js", + ) +) +# Under tests/ everything is in scope except the browser lane, the executable +# stubs and the top-level Python files. +TESTS_OUT_PREFIXES = ("tests/browser/", "tests/stubs/") + + +def is_top_level_python(name): + # test_*.py and the helper they share, support.py, which no Rust source names. + return "/" not in name and name.endswith(".py") + + +def in_rust_scope(path): + if path in RUST_FILES or path in RUST_READ_FILES or path.startswith(RUST_PREFIXES): + return True + if WEB_IN_RUST_SCOPE and path.startswith("web/"): + return True + if path.startswith("tests/"): + return not ( + path.startswith(TESTS_OUT_PREFIXES) + or is_top_level_python(path[len("tests/") :]) + ) + return False + + +def parse_paths(data, nul): + """The paths in `data`, or ValueError naming the first one refused.""" + try: + text = data.decode("utf-8") + except UnicodeDecodeError as err: + raise ValueError("input is not UTF-8") from err + parts = text.split("\0" if nul else "\n") + if parts[-1] == "": + parts.pop() + for path in parts: + if ( + "\0" in path + or (not nul and path.endswith("\r")) + or path.startswith("/") + or {"", ".", ".."} & set(path.split("/")) + ): + raise ValueError(f"not a relative repository path: {path!r}") + return parts + + +def main(argv): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--scope-only", action="store_true", required=True) + parser.add_argument("-z", dest="nul", action="store_true") + args = parser.parse_args(argv) + try: + paths = parse_paths(sys.stdin.buffer.read(), args.nul) + except ValueError as err: + print(f"check-gate-evidence: {err}", file=sys.stderr) + return 2 + print("in" if any(in_rust_scope(p) for p in paths) else "out") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/test.sh b/scripts/test.sh index cfbbc4b43..79639ffb7 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -704,6 +704,7 @@ gate calibration-fixtures "$PYTHON" "$ROOT/scripts/gen-calibration-fixtures.py" gate calibration-tests unittest_gate "$ROOT/tests/test_calibrate_framework.py" gate gemini-usage-tests unittest_gate "$ROOT/tests/test_analyze_gemini_usage.py" gate audit-tests unittest_gate "$ROOT/tests/test_audit.py" +gate gate-evidence-tests unittest_gate "$ROOT/tests/test_check_gate_evidence.py" gate rust-style-tests unittest_gate "$ROOT/tests/test_check_rust_style.py" gate ceiling-hotspots-tests unittest_gate "$ROOT/tests/test_ceiling_hotspots.py" gate test-gate-tests unittest_gate "$ROOT/tests/test_test_gate.py" diff --git a/tests/test_check_gate_evidence.py b/tests/test_check_gate_evidence.py new file mode 100644 index 000000000..6ab8d3141 --- /dev/null +++ b/tests/test_check_gate_evidence.py @@ -0,0 +1,356 @@ +"""Check the Rust-scope helper of scripts/check-gate-evidence.py.""" + +import importlib.util +from pathlib import Path +import posixpath +import re +import subprocess +import sys +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts/check-gate-evidence.py" + +_spec = importlib.util.spec_from_file_location("check_gate_evidence", SCRIPT) +gate_evidence = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(gate_evidence) + + +def run(args, data): + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + input=data, + capture_output=True, + check=False, + ) + + +def scope(*paths): + return run(["--scope-only"], ("\n".join(paths) + "\n").encode()) + + +# One pass over Rust text, left to right, so a quote inside a comment or a +# character literal cannot pair with a later one; only string literals are kept. +RUST_TOKEN = re.compile( + r"""//[^\n]* + |/\*.*?\*/ + |(?\#*)"(?P.*?)"(?P=hashes) + |b?"(?P(?:[^"\\]|\\.)*)" + |'(?:\\[^']+|[^\\'])'""", + re.DOTALL | re.VERBOSE, +) +JOIN_CHAIN = r"((?:\s*\.\s*join\(\s*\"[^\"]*\"\s*\))+)" +ENV_CALL = re.compile(r'env!\(\s*"[^"]*"\s*\)') +INCLUDE = re.compile(r"\binclude(?:_str|_bytes)?!\s*\(") +FOLDER = re.compile(r'#\[folder\s*=\s*"([^"]*)"') +MANIFEST_BINDING = re.compile( + r"let\s+(?:mut\s+)?(\w+)\s*=[^;]*CARGO_MANIFEST_DIR[^;]*;" +) +MANIFEST_CHAIN = re.compile(r'CARGO_MANIFEST_DIR\s*"?\)*' + JOIN_CHAIN) +REPO_LITERAL = re.compile( + r'(?:Path::new|read_to_string|fs::read|File::open|read_dir)\(\s*"([^"]*)"' +) +RUST_SOURCE_PREFIXES = ("src/", "tests/", "desktop/", "scripts/hook-check/", "vendor/") + + +def string_literals(text): + return [ + m.group("raw") if m.group("raw") is not None else m.group("plain") + for m in RUST_TOKEN.finditer(text) + if m.group("raw") is not None or m.group("plain") is not None + ] + + +def tracked_files(): + """The tracked paths, or [] only where there is no checkout to ask. + + A failing git inside a checkout raises, so the coverage tests cannot be + skipped into passing by a broken repository. + """ + if not (ROOT / ".git").exists(): + return [] + listing = subprocess.run( + ["git", "-C", str(ROOT), "ls-files", "-z"], capture_output=True, check=True + ) + return [p.decode() for p in listing.stdout.split(b"\0") if p] + + +def balanced(text, open_at): + """The text between the parenthesis at `open_at` and its partner.""" + depth = 0 + for at in range(open_at, len(text)): + depth += {"(": 1, ")": -1}.get(text[at], 0) + if depth == 0: + return text[open_at + 1 : at] + return None + + +def under(base, relative): + """`relative` joined to `base` as a repository path, None when it leaves the tree.""" + path = posixpath.normpath(posixpath.join(base, relative)) + return None if path.startswith("..") else path + + +def referenced_paths(source, text, crate_root): + """Repository paths a Rust source names by literal, and what it names otherwise. + + Resolved statically from include_str!, include_bytes! and include! (relative + to the file, or to the crate root through CARGO_MANIFEST_DIR), #[folder], + `.join("...")` chains on CARGO_MANIFEST_DIR or on a variable bound to it, and + path literals handed to Path::new or a file-reading call, which count only + where they resolve against the crate root. The second result lists the + references a regular expression cannot resolve, so they are reported rather + than guessed. + """ + found, unresolved = set(), [] + for match in INCLUDE.finditer(text): + args = balanced(text, match.end() - 1) or "" + literals = string_literals(ENV_CALL.sub("", args)) + if "OUT_DIR" in args: + continue + if "CARGO_MANIFEST_DIR" in args: + found.add(under(crate_root, "".join(literals).lstrip("/"))) + elif len(literals) == 1 and args.strip().startswith('"'): + found.add(under(posixpath.dirname(source), literals[0])) + else: + unresolved.append(f"{source}: include of {args.strip()[:60]!r}") + for match in FOLDER.finditer(text): + found.add(under(crate_root, match.group(1))) + chains = [m.group(1) for m in MANIFEST_CHAIN.finditer(text)] + for name in {m.group(1) for m in MANIFEST_BINDING.finditer(text)}: + chains += re.findall(r"\b" + name + JOIN_CHAIN, text) + for chain in chains: + found.add(under(crate_root, "/".join(string_literals(chain)))) + if "CARGO_MANIFEST_DIR" in text and not chains: + unresolved.append(f"{source}: CARGO_MANIFEST_DIR without a literal join") + for match in REPO_LITERAL.finditer(text): + found.add(under(crate_root, match.group(1))) + return found, unresolved + + +def mentioned_paths(text, crate_root): + """Every string literal, read against the crate root and the repository root. + + Broader than referenced_paths and used only where a literal is exactly a + tracked file: a file handed to a command or joined onto a variable names no + function the scan could follow, and a message that merely mentions a file + costs the same toolchain requirement as a read would. + """ + bases = {crate_root, ""} + return {under(base, lit) for lit in string_literals(text) for base in bases} + + +class ScopeFunction(unittest.TestCase): + def test_rust_paths_are_in(self): + for path in ( + "src/main.rs", + "src/web/mod.rs", + "desktop/src/main.rs", + "desktop/Cargo.toml", + "scripts/hook-check/lib.rs", + "build.rs", + "Cargo.toml", + "Cargo.lock", + ".cargo/config.toml", + "rust-toolchain.toml", + "clippy.toml", + "rustfmt.toml", + "deny.toml", + "vendor/livekit-net/src/lib.rs", + "problem-bank/task-defaults.json", + "tests/cli.rs", + "tests/fixtures/sample.json", + "tests/unit/web/tasks.rs", + ): + with self.subTest(path=path): + self.assertTrue(gate_evidence.in_rust_scope(path)) + + def test_non_rust_paths_are_out(self): + for path in ( + "README.md", + "docs/contributing-notes.md", + "scripts/test.sh", + "scripts/check-pr-text.py", + "tests/test_check_pr_text.py", + "tests/support.py", + "tests/browser/app.spec.js", + "tests/stubs/codex", + ".github/workflows/check.yml", + "package.json", + "Makefile", + "srcs/x.rs", + "build.rs.orig", + "Cargo.toml.bak", + ): + with self.subTest(path=path): + self.assertFalse(gate_evidence.in_rust_scope(path)) + + def test_web_follows_the_one_named_constant(self): + self.assertFalse(gate_evidence.WEB_IN_RUST_SCOPE) + self.assertFalse(gate_evidence.in_rust_scope("web/app.js")) + with mock.patch.object(gate_evidence, "WEB_IN_RUST_SCOPE", True): + self.assertTrue(gate_evidence.in_rust_scope("web/app.js")) + + def test_files_rust_reads_are_in_even_under_an_out_directory(self): + for path in ("docs/task-mode.md", "scripts/soak-judge.cjs", "web/lib.js"): + with self.subTest(path=path): + self.assertTrue(gate_evidence.in_rust_scope(path)) + + def test_github_never_counts_as_in_scope(self): + self.assertFalse(gate_evidence.in_rust_scope(".github/workflows/x.yml")) + + def test_every_tracked_rust_file_is_in(self): + tracked = tracked_files() + if not tracked: + self.skipTest("not a git checkout") + rust = [ + p + for p in tracked + if p.endswith(".rs") or Path(p).name in ("Cargo.toml", "Cargo.lock") + ] + self.assertTrue(rust) + missed = [p for p in rust if not gate_evidence.in_rust_scope(p)] + self.assertEqual(missed, [], "Rust files outside the scope list") + + +class RustReferences(unittest.TestCase): + maxDiff = None + + def test_the_scan_resolves_each_reference_form(self): + text = """ + const A: &str = include_str!("../docs/a.md"); + const B: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/docs/b.md")); + #[folder = "web/"] + struct Assets; + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let c = root.join("scripts").join("c.sh"); + let d = Path::new(env!("CARGO_MANIFEST_DIR")).join("docs/d.md"); + let e = std::fs::read_to_string("docs/e.md"); + const F: &str = include_str!(concat!(env!("OUT_DIR"), "/gen.rs")); + const G: &str = include_str!(PATH); + """ + found, unresolved = referenced_paths("tests/unit/x.rs", text, "") + self.assertEqual( + found, + { + "tests/docs/a.md", + "docs/b.md", + "web", + "scripts/c.sh", + "docs/d.md", + "docs/e.md", + }, + ) + self.assertEqual(len(unresolved), 1) + self.assertIn("PATH", unresolved[0]) + + def test_quotes_in_comments_and_char_literals_do_not_shift_literals(self): + text = """ + // it's "quoted" here + let quote = '"'; + /* "block" */ + let raw = r#"docs/raw.md"#; + let path = Path::new("docs/after.md"); + """ + self.assertEqual(string_literals(text), ["docs/raw.md", "docs/after.md"]) + + def test_a_literal_is_read_against_both_roots(self): + self.assertEqual( + mentioned_paths('run("docs/f.md")', "desktop"), + {"desktop/docs/f.md", "docs/f.md"}, + ) + + def test_every_path_a_rust_source_names_is_in_scope(self): + tracked = tracked_files() + if not tracked: + self.skipTest("not a git checkout") + files = set(tracked) + crate_roots = sorted( + ( + posixpath.dirname(p) + for p in tracked + if posixpath.basename(p) == "Cargo.toml" + ), + key=len, + reverse=True, + ) + missed = {} + members_of = {} + for source in tracked: + if not source.endswith(".rs") or not ( + source.startswith(RUST_SOURCE_PREFIXES) or source == "build.rs" + ): + continue + crate_root = next( + r for r in crate_roots if r == "" or source.startswith(r + "/") + ) + text = (ROOT / source).read_text(encoding="utf-8") + named = referenced_paths(source, text, crate_root)[0] + for path in (named | mentioned_paths(text, crate_root)) - {None, "."}: + if path in files: + members = [path] + elif path in named: + # A directory the Rust code walks or embeds. web/ is decided + # by WEB_IN_RUST_SCOPE alone; single web/ files named by a + # literal include are in scope whatever it says. + if path not in members_of: + members_of[path] = [ + f + for f in tracked + if f.startswith(path + "/") and not f.startswith("web/") + ] + members = members_of[path] + else: + continue + for member in members: + if not gate_evidence.in_rust_scope(member): + missed.setdefault(member, source) + self.assertEqual(missed, {}, "paths Rust names that the scope omits") + + +class ScopeOnly(unittest.TestCase): + def test_in_when_any_path_is_in(self): + result = scope("docs/a.md", "src/lib.rs") + self.assertEqual((result.returncode, result.stdout), (0, b"in\n")) + + def test_out_when_none_is(self): + result = scope("docs/a.md", "web/app.js", ".github/workflows/x.yml") + self.assertEqual((result.returncode, result.stdout), (0, b"out\n")) + + def test_an_empty_list_is_out(self): + result = run(["--scope-only"], b"") + self.assertEqual((result.returncode, result.stdout), (0, b"out\n")) + + def test_nul_separated_paths_may_hold_newlines(self): + result = run(["--scope-only", "-z"], b"docs/a\nsrc/b.md\0") + self.assertEqual(result.stdout, b"out\n") + result = run(["--scope-only", "-z"], b"docs/a.md\0src/x\ny.rs\0") + self.assertEqual(result.stdout, b"in\n") + + def test_invalid_input_exits_2(self): + for data in ( + b"\n", + b"Cargo.toml\r\n", + b"/etc/passwd\n", + b"src/../docs/a.md\n", + b"..\n", + b"./src/a.rs\n", + b"docs//a.md\n", + b"docs/a.md\n\nsrc/a.rs\n", + b"\xff\xfe\n", + b"docs/a\0src/main.rs\n", + ): + with self.subTest(data=data): + result = run(["--scope-only"], data) + self.assertEqual((result.returncode, result.stdout), (2, b"")) + self.assertTrue(result.stderr) + + def test_bad_usage_exits_2(self): + for args in ([], ["-z"], ["--needs-maintainer"]): + with self.subTest(args=args): + self.assertEqual(run(args, b"").returncode, 2) + + +if __name__ == "__main__": + unittest.main() From 463f58a3a4a8bf2264715e64b2e2bd79f1d4efbc Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 17 Nov 2026 02:10:09 +0800 Subject: [PATCH 06/13] Check a push's gate receipt against its tree Approving a held run needed a person to read a contributor's claim that the Rust gate had passed, and nothing compared it with what is pushed. The evidence check turns that into a mechanical answer from data files alone: a change with no Rust-scope path owes no receipt; otherwise some receipt must be for lane rust or all, passed, name the head commit's tree, and carry the rustc that the base rust-toolchain.toml pins (the head's when the change edits that file). A bot author is exempt, a .github/ path is never in scope, and a truncated changed-file list or an unreadable pin is indeterminate rather than a pass. The receipt is untrusted: it is size-checked before it is parsed, parsed strictly (no duplicate keys, NaN, trailing data or bool-as-number), and none of its text reaches a path, a command or the output. It is a hint a dishonest author can forge, not proof the gate ran. Change-Id: I2b86f020d9716447c6c117d9761bf5a17177f3c9 --- scripts/check-gate-evidence.py | 265 +++++++++++++++++++++++++- tests/test_check_gate_evidence.py | 301 ++++++++++++++++++++++++++++++ 2 files changed, 564 insertions(+), 2 deletions(-) diff --git a/scripts/check-gate-evidence.py b/scripts/check-gate-evidence.py index f135fd3de..e8e931b21 100644 --- a/scripts/check-gate-evidence.py +++ b/scripts/check-gate-evidence.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 -"""Decide whether a change touches what the Rust gate covers. +"""Decide whether a change needs the Rust gate, and whether it carries proof. check-gate-evidence.py --scope-only [-z] < paths + check-gate-evidence.py --head-tree ID --changed-files-file FILE + --author-type TYPE --base-toolchain-file FILE + [--receipt-file FILE] [--head-toolchain-file FILE] Paths are repository-relative, one per line, or NUL-separated with -z because a path may hold a newline. --scope-only prints `in` when any path is in the @@ -10,11 +13,36 @@ absolute, empty (a blank line is one), ends in a carriage return in line mode, or holds a `.` or `..` component: those can name something other than what they look like, so they are refused rather than classified. + +Without a mode the script checks evidence. Every input is a file of data the +caller fetched (the head commit's tree id, the gate note on it, the changed +paths NUL-separated, the two rust-toolchain.toml files); it reads those files +and nothing else, runs nothing, and never places a receipt field in a path or a +command. A change with no path in the Rust scope needs no receipt. Otherwise +some receipt must be for lane rust or all, passed, and name the head commit's +tree, and its rustc must be the channel pinned by the base's +rust-toolchain.toml, or by the head's when the change edits that file; the +toolchain files are read only when a receipt is owed. The +receipt is a hint a dishonest author can forge; it makes the approval decision +mechanical and does not prove the gate ran. A receipt that counts optional-tool +skips still qualifies: a local run tolerates them. + +Exit 0 means the evidence holds or no receipt is owed (a Bot author is exempt). +Exit 1 means it is missing or wrong, with the reason and the commands to run. +Exit 2 means the answer cannot be given: a cut path list, a toolchain file that +cannot be read, bad usage. """ import argparse +import json +import re import sys +try: + import tomllib +except ImportError: # Python before 3.11; only the evidence check needs it + tomllib = None + # Whether web/ is in the Rust scope. The binary embeds web/ at build time, so a # web-only change can break the Rust lane; it stays out by default so that # JavaScript contributors need no toolchain. Flip it here and nowhere else. @@ -117,11 +145,244 @@ def parse_paths(data, nul): return parts +RECEIPT_LIMIT = 16384 +TOOLCHAIN_LIMIT = 65536 +PATHS_LIMIT = 1 << 24 +MAX_RECEIPTS = 8 +LANES = frozenset(("rust", "all", "portable", "browser")) +# The lanes that run the Rust gates; the others say nothing about a Rust change. +COVERING_LANES = frozenset(("rust", "all")) +TREE_ID = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})") +EXACT_CHANNEL = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+") +RUSTC_VERSION = re.compile(r"rustc ([0-9]+\.[0-9]+\.[0-9]+) \(.+\)") +TOOLCHAIN_FILE = "rust-toolchain.toml" +PREFIX = "check-gate-evidence: " +NO_NOTE = "no receipt: the pushed head carries no gate note" + +GUIDANCE = ( + "Run ./scripts/env-check.sh, then ./scripts/test.sh, then push again.", + "A receipt names the tree the gate saw, so a partial commit needs the gate " + "re-run on what is committed.", +) + + +class Indeterminate(Exception): + """The answer cannot be given from these inputs (exit 2).""" + + +class Refused(Exception): + """The evidence is missing or wrong (exit 1).""" + + +def read_limited(path, limit): + """The file's bytes, or None when it is larger than `limit`.""" + with open(path, "rb") as handle: + data = handle.read(limit + 1) + return data if len(data) <= limit else None + + +def reject_duplicates(pairs): + keys = [key for key, _ in pairs] + if len(set(keys)) != len(keys): + raise ValueError("duplicate key") + return dict(pairs) + + +def reject_constant(name): + raise ValueError(name) + + +def is_count(value): + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def check_receipt_shape(one): + if not isinstance(one, dict): + raise Refused("receipt: an entry is not an object") + lane = one.get("lane") + if not isinstance(lane, str) or lane not in LANES: + raise Refused("receipt: an entry has an unknown lane") + if not isinstance(one.get("passed"), bool): + raise Refused("receipt: an entry has a non-boolean passed") + if not is_count(one.get("skipped")): + raise Refused("receipt: an entry has a bad skipped count") + for key, limit in (("time", 32), ("rustc", 200), ("cargo", 200)): + value = one.get(key) + if not isinstance(value, str) or len(value) > limit: + raise Refused(f"receipt: an entry has a bad {key}") + + +def parse_receipt(data, head_tree): + """The receipts of a well-formed note for `head_tree`, else Refused.""" + if data is None: + raise Refused(f"receipt: over {RECEIPT_LIMIT} bytes") + try: + body = json.loads( + data.decode("utf-8"), + object_pairs_hook=reject_duplicates, + parse_constant=reject_constant, + ) + except (UnicodeDecodeError, ValueError, RecursionError) as err: + raise Refused("receipt: not a single JSON value") from err + if not isinstance(body, dict): + raise Refused("receipt: not a JSON object") + version = body.get("version") + # 1.0 and True compare equal to 1, so the type is checked too. + if type(version) is not int or version != 1: + raise Refused("receipt: unsupported version") + tree = body.get("worktree_tree") + if not isinstance(tree, str) or not TREE_ID.fullmatch(tree): + raise Refused("receipt: bad worktree_tree") + receipts = body.get("receipts") + if not isinstance(receipts, list) or not 1 <= len(receipts) <= MAX_RECEIPTS: + raise Refused(f"receipt: needs 1 to {MAX_RECEIPTS} entries") + for one in receipts: + check_receipt_shape(one) + if tree != head_tree: + raise Refused("receipt is for another tree than the pushed head") + return receipts + + +def pinned_channel(data): + """The exact `channel` of a rust-toolchain.toml, else Indeterminate. + + A channel that is not X.Y.Z (stable, 1.100) names whatever the toolchain + manager resolves it to, so no receipt string can be compared to it. + """ + if tomllib is None: + raise Indeterminate("the evidence check needs Python 3.11+ (tomllib)") + try: + table = tomllib.loads(data.decode("utf-8")) + except (UnicodeDecodeError, tomllib.TOMLDecodeError) as err: + raise Indeterminate(f"{TOOLCHAIN_FILE} is not readable TOML") from err + toolchain = table.get("toolchain") + channel = toolchain.get("channel") if isinstance(toolchain, dict) else None + if not isinstance(channel, str) or not EXACT_CHANNEL.fullmatch(channel): + raise Indeterminate(f"{TOOLCHAIN_FILE} does not pin an exact X.Y.Z channel") + return channel + + +def toolchain_channel(path): + if path is None: + raise Indeterminate(f"no {TOOLCHAIN_FILE} was given to read the pin from") + try: + data = read_limited(path, TOOLCHAIN_LIMIT) + except OSError as err: + raise Indeterminate( + f"cannot read {TOOLCHAIN_FILE}: {err.strerror or err}" + ) from err + if data is None: + raise Indeterminate(f"{TOOLCHAIN_FILE} is over {TOOLCHAIN_LIMIT} bytes") + return pinned_channel(data) + + +def read_receipt(path): + if path is None: + raise Refused(NO_NOTE) + try: + return read_limited(path, RECEIPT_LIMIT) + except FileNotFoundError: + raise Refused(NO_NOTE) from None + except OSError as err: + raise Indeterminate(f"cannot read the receipt: {err.strerror or err}") from err + + +def check_receipt(args, paths): + receipts = parse_receipt(read_receipt(args.receipt_file), args.head_tree) + lane_ok = [r for r in receipts if r["lane"] in COVERING_LANES] + if not lane_ok: + raise Refused("no receipt for lane rust or all") + passed = [r for r in lane_ok if r["passed"]] + if not passed: + raise Refused("the rust or all lane did not pass") + changes_pin = TOOLCHAIN_FILE in paths + channel = toolchain_channel( + args.head_toolchain_file if changes_pin else args.base_toolchain_file + ) + for one in passed: + found = RUSTC_VERSION.fullmatch(one["rustc"]) + if found and found.group(1) == channel: + return + raise Refused( + f"no passing receipt was made with the pinned rustc {channel}; " + "if the base branch moved the pin, rebase onto it and run the gate again" + ) + + +def read_changed_paths(path): + try: + data = read_limited(path, PATHS_LIMIT) + if data is None: + raise ValueError("list is too large") + if data and not data.endswith(b"\0"): + raise ValueError("the last path is not terminated, so the list may be cut") + return parse_paths(data, True) + except (OSError, ValueError) as err: + raise Indeterminate( + f"cannot use the changed-file list: {str(err)[:200]}" + ) from err + + +def evidence(args): + """Print what holds; raise Refused or Indeterminate when it does not.""" + if not TREE_ID.fullmatch(args.head_tree): + raise Indeterminate("--head-tree is not a tree id") + paths = read_changed_paths(args.changed_files_file) + if args.author_type == "Bot": + print("exempt: the author type is Bot, so no receipt is owed") + elif not any(in_rust_scope(p) for p in paths): + print("no Rust-scope path changed, so no receipt is owed") + else: + check_receipt(args, paths) + print("gate evidence holds") + + +def run_evidence(args): + try: + evidence(args) + except Indeterminate as err: + print(PREFIX + str(err), file=sys.stderr) + return 2 + except Refused as err: + print(PREFIX + str(err), file=sys.stderr) + print(*GUIDANCE, sep="\n", file=sys.stderr) + return 1 + return 0 + + +# The evidence check's value-taking options, by attribute name. +CHECK_OPTIONS = ( + "head_tree", + "receipt_file", + "changed_files_file", + "base_toolchain_file", + "head_toolchain_file", + "author_type", +) + + def main(argv): parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - parser.add_argument("--scope-only", action="store_true", required=True) + parser.add_argument("--scope-only", action="store_true") parser.add_argument("-z", dest="nul", action="store_true") + check = parser.add_argument_group("evidence check") + check.add_argument("--head-tree") + check.add_argument("--receipt-file") + check.add_argument("--changed-files-file") + check.add_argument("--base-toolchain-file") + check.add_argument("--head-toolchain-file") + check.add_argument("--author-type") args = parser.parse_args(argv) + if not args.scope_only: + required = (args.head_tree, args.changed_files_file, args.author_type) + if args.nul or None in required: + parser.error( + "the evidence check needs --head-tree, --changed-files-file and --author-type" + ) + return run_evidence(args) + given = vars(args) + if any(given[name] is not None for name in CHECK_OPTIONS): + parser.error("--scope-only takes paths on stdin only") try: paths = parse_paths(sys.stdin.buffer.read(), args.nul) except ValueError as err: diff --git a/tests/test_check_gate_evidence.py b/tests/test_check_gate_evidence.py index 6ab8d3141..8bca8426a 100644 --- a/tests/test_check_gate_evidence.py +++ b/tests/test_check_gate_evidence.py @@ -1,11 +1,13 @@ """Check the Rust-scope helper of scripts/check-gate-evidence.py.""" import importlib.util +import json from pathlib import Path import posixpath import re import subprocess import sys +import tempfile import unittest from unittest import mock @@ -352,5 +354,304 @@ def test_bad_usage_exits_2(self): self.assertEqual(run(args, b"").returncode, 2) +TREE = "a" * 40 +OTHER_TREE = "b" * 40 +PINNED = '[toolchain]\nchannel = "1.100.0"\n' + + +def receipt(**over): + one = { + "lane": "rust", + "passed": True, + "skipped": 0, + "time": "2026-11-16T12:34:56Z", + "rustc": "rustc 1.100.0 (abcdef123 2026-10-01)", + "cargo": "cargo 1.100.0 (abcdef123 2026-09-01)", + } + one.update(over) + return one + + +def note(*receipts, **over): + body = { + "version": 1, + "worktree_tree": TREE, + "receipts": list(receipts) or [receipt()], + } + body.update(over) + return body + + +class EvidenceCheck(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.dir = Path(tmp.name) + + def put(self, name, data): + path = self.dir / name + if isinstance(data, (dict, list)): + data = json.dumps(data) + path.write_bytes(data if isinstance(data, bytes) else data.encode()) + return str(path) + + def args(self, paths=("src/main.rs",), tree=TREE, author="User", base=PINNED): + args = [ + "--head-tree", tree, + "--changed-files-file", self.put("changed", "".join(p + "\0" for p in paths)), + "--author-type", author, + ] # fmt: skip + if base is not None: + args += ["--base-toolchain-file", self.put("base.toml", base)] + return args + + def check(self, body=None, head=None, **kwargs): + args = self.args(**kwargs) + if body is not None: + args += ["--receipt-file", self.put("receipt", body)] + if head is not None: + args += ["--head-toolchain-file", self.put("head.toml", head)] + return run(args, b"") + + def fails(self, result, reason=None): + self.assertEqual(result.returncode, 1, result.stderr + result.stdout) + lines = result.stderr.decode().splitlines() + self.assertTrue(lines[0].startswith("check-gate-evidence: "), lines) + if reason: + self.assertIn(reason, lines[0]) + text = "\n".join(lines[1:]) + self.assertIn("./scripts/env-check.sh", text) + self.assertIn("./scripts/test.sh", text) + self.assertIn("partial commit", text) + return lines[0] + + def test_receipt_for_the_head_tree_passes(self): + result = self.check(note()) + self.assertEqual((result.returncode, result.stderr), (0, b"")) + + def test_skips_do_not_disqualify(self): + self.assertEqual(self.check(note(receipt(skipped=3))).returncode, 0) + + def test_all_lane_satisfies(self): + self.assertEqual(self.check(note(receipt(lane="all"))).returncode, 0) + + def test_one_good_receipt_among_others_passes(self): + body = note(receipt(lane="portable"), receipt(passed=False), receipt()) + self.assertEqual(self.check(body).returncode, 0) + + def test_unknown_keys_are_ignored(self): + body = note(receipt(extra=[1]), extra="x") + self.assertEqual(self.check(body).returncode, 0) + + def test_missing_receipt_fails(self): + self.fails(self.check(None), "no receipt") + + def test_missing_receipt_file_fails(self): + args = self.args() + ["--receipt-file", str(self.dir / "absent")] + self.fails(run(args, b""), "no receipt") + + def test_unreadable_receipt_is_indeterminate(self): + args = self.args() + ["--receipt-file", str(self.dir)] + self.assertEqual(run(args, b"").returncode, 2) + + def test_malformed_receipts_fail_with_a_one_line_reason(self): + valid = json.dumps(note()).encode() + cases = { + "not json": b"not json", + "empty": b"", + "not utf-8": b"\xff\xfe", + "not an object": b"[]", + "trailing data": valid + b" {}", + "over 16 KiB": valid + b" " * (16385 - len(valid)), + "duplicate key": b'{"version":1,"version":1}', + "nan": b'{"version": NaN}', + "deep": b"[" * 16000, + "wrong version": note(version=2), + "version bool": note(version=True), + "version missing": {"worktree_tree": TREE, "receipts": [receipt()]}, + "bad tree": note(worktree_tree="A" * 40), + "short tree": note(worktree_tree="a" * 39), + "tree not str": note(worktree_tree=7), + "no receipts": note(receipts=[]), + "receipts not list": note(receipts={}), + "nine receipts": note(*[receipt()] * 9), + "receipt not object": note(receipts=[1]), + "unknown lane": note(receipt(lane="mutants")), + "lane list": note(receipt(lane=[])), + "lane object": note(receipt(lane={})), + "version float": note(version=1.0), + "version exponent": json.dumps(note()).replace('"version": 1,', '"version": 1e0,').encode(), + "passed not bool": note(receipt(passed="yes")), + "passed int": note(receipt(passed=1)), + "skipped bool": note(receipt(skipped=True)), + "skipped negative": note(receipt(skipped=-1)), + "skipped float": note(receipt(skipped=0.0)), + "time long": note(receipt(time="x" * 33)), + "time not str": note(receipt(time=5)), + "rustc long": note(receipt(rustc="r" * 201)), + "cargo long": note(receipt(cargo="c" * 201)), + "cargo not str": note(receipt(cargo=None)), + "key missing": note({"lane": "rust", "passed": True}), + } # fmt: skip + for name, body in cases.items(): + with self.subTest(name): + reason = self.fails(self.check(body)) + self.assertIn("receipt", reason) + + def test_eight_receipts_are_allowed(self): + self.assertEqual(self.check(note(*[receipt()] * 8)).returncode, 0) + + def test_empty_strings_are_allowed(self): + body = note(receipt(rustc="", cargo=""), receipt()) + self.assertEqual(self.check(body).returncode, 0) + + def test_receipt_for_another_tree_fails(self): + self.fails(self.check(note(worktree_tree=OTHER_TREE)), "another tree") + + def test_sha256_tree_ids_compare_exactly(self): + tree = "c" * 64 + self.assertEqual(self.check(note(worktree_tree=tree), tree=tree).returncode, 0) + + def test_bad_head_tree_is_indeterminate(self): + for tree in ("", "A" * 40, "a" * 41, "../x"): + with self.subTest(tree=tree): + self.assertEqual(self.check(note(), tree=tree).returncode, 2) + + def test_wrong_lane_fails(self): + for lane in ("portable", "browser"): + with self.subTest(lane=lane): + self.fails(self.check(note(receipt(lane=lane))), "rust or all") + + def test_failed_gate_fails(self): + self.fails(self.check(note(receipt(passed=False))), "did not pass") + + def test_a_toolchain_mismatch_says_to_rebase_onto_the_base(self): + # A branch cut before the pin moved produces the same old rustc on every + # rerun, so running the gate again cannot clear the refusal. + old = receipt(rustc="rustc 1.99.0 (abcdef123 2026-10-01)") + self.fails(self.check(note(old)), "rebase") + + def test_rustc_must_match_the_base_channel(self): + old = receipt(rustc="rustc 1.99.0 (abcdef123 2026-10-01)") + self.fails(self.check(note(old)), "1.100.0") + for bad in ( + "", + "cargo 1.100.0", + "rustc 1.100.01 (x)", + "rustc 1.100.0-nightly (x)", + "rustc 1.100.0", + "rustc 1.100.0 (", + "rustc 1.100.0 ()", + "rustc 1.100.\u0660 (x)", + ): + with self.subTest(rustc=bad): + self.fails(self.check(note(receipt(rustc=bad))), "1.100.0") + + def test_receipt_text_is_never_echoed(self): + result = self.check(note(receipt(rustc="rustc 9.9.9 EVIL"))) + self.fails(result) + self.assertNotIn(b"EVIL", result.stderr + result.stdout) + + def test_toolchain_change_compares_to_the_head(self): + paths = ("rust-toolchain.toml", "src/a.rs") + new = receipt(rustc="rustc 1.101.0 (abcdef123 2026-11-01)") + head = '[toolchain]\nchannel = "1.101.0"\n' + self.assertEqual(self.check(note(new), paths=paths, head=head).returncode, 0) + self.fails(self.check(note(receipt()), paths=paths, head=head), "1.101.0") + + def test_head_toolchain_is_ignored_when_the_diff_leaves_it_alone(self): + new = receipt(rustc="rustc 1.101.0 (abcdef123 2026-11-01)") + head = '[toolchain]\nchannel = "1.101.0"\n' + self.fails(self.check(note(new), head=head), "1.100.0") + + def test_toolchain_that_cannot_be_read_is_indeterminate(self): + bad = ( + "", + "channel = ", + 'channel = "1.100.0"\n', + "[toolchain]\nchannel = 7\n", + '[toolchain]\nchannel = "1.100"\n', + '[toolchain]\nchannel = "stable"\n', + '[toolchain]\nchannel = "1.100.٠"\n', + PINNED + "#" * 70000, + ) + for text in bad: + with self.subTest(text=text[:30]): + self.assertEqual(self.check(note(), base=text).returncode, 2) + self.assertEqual(self.check(note(), base=None).returncode, 2) + removed = self.check(note(), paths=("rust-toolchain.toml",)) + self.assertEqual(removed.returncode, 2) + + def test_without_tomllib_the_check_is_indeterminate(self): + with mock.patch.object(gate_evidence, "tomllib", None): + with self.assertRaises(gate_evidence.Indeterminate): + gate_evidence.pinned_channel(b'[toolchain]\nchannel = "1.100.0"') + + def test_out_of_scope_paths_pass_without_a_receipt(self): + paths = ("docs/a.md", "web/app.js", "tests/test_x.py") + result = self.check(None, paths=paths) + self.assertEqual((result.returncode, result.stderr), (0, b"")) + self.assertIn(b"no Rust-scope", result.stdout) + + def test_no_changed_files_pass(self): + self.assertEqual(self.check(None, paths=()).returncode, 0) + + def test_a_github_path_owes_no_receipt_by_itself(self): + workflow = (".github/workflows/check.yml",) + self.assertEqual(self.check(None, paths=workflow).returncode, 0) + self.assertEqual( + self.check(note(), paths=workflow + ("src/a.rs",)).returncode, 0 + ) + self.assertEqual(self.check(None, paths=workflow + ("src/a.rs",)).returncode, 1) + + def test_bot_is_exempt_and_named(self): + result = self.check(None, author="Bot") + self.assertEqual(result.returncode, 0) + self.assertIn(b"exempt", result.stdout) + self.assertIn(b"Bot", result.stdout) + + def test_other_authors_are_not_exempt(self): + for author in ("User", "Organization", "bot", "Mannequin", "x"): + with self.subTest(author=author): + self.assertEqual(self.check(None, author=author).returncode, 1) + + def test_bad_changed_path_is_indeterminate(self): + for paths in (("../x",), ("/etc/passwd",), ("a//b",)): + with self.subTest(paths=paths): + self.assertEqual(self.check(note(), paths=paths).returncode, 2) + + def test_changed_list_without_its_final_nul_is_indeterminate(self): + args = self.args() + args[args.index("--changed-files-file") + 1] = self.put("cut", "docs/a.md") + result = run(args, b"") + self.assertEqual(result.returncode, 2) + self.assertIn(b"changed-file list", result.stderr) + + def test_a_refused_path_is_not_echoed_in_full(self): + result = self.check(note(), paths=("../" + "x" * 5000,)) + self.assertEqual(result.returncode, 2) + self.assertLess(len(result.stderr), 500) + + def test_changed_paths_may_hold_a_newline(self): + result = self.check(None, paths=("docs/a\nsrc/b.md",)) + self.assertEqual(result.returncode, 0) + + def test_bad_usage_is_indeterminate(self): + self.assertEqual(run([], b"").returncode, 2) + self.assertEqual(run(["--head-tree", TREE], b"").returncode, 2) + for flag in ("--head-tree", "--changed-files-file", "--author-type"): + args = self.args() + at = args.index(flag) + with self.subTest(flag=flag): + self.assertEqual(run(args[:at] + args[at + 2 :], b"").returncode, 2) + mixed = ["--scope-only", "--head-tree", TREE] + self.assertEqual(run(mixed, b"").returncode, 2) + + def test_missing_changed_files_file_is_indeterminate(self): + args = self.args() + args[args.index("--changed-files-file") + 1] = str(self.dir / "absent") + self.assertEqual(run(args, b"").returncode, 2) + + if __name__ == "__main__": unittest.main() From 19bec5168ad4945a827d779d55077d6106e8bdb8 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 17 Nov 2026 02:11:54 +0800 Subject: [PATCH 07/13] Send gate evidence with a push A contributor with no toolchain can open a pull request that only CI compiles, and nothing on the push says whether the gate was run. The pre-push hook now asks, for a topic branch that changes what the Rust gate covers, for a passing rust or all receipt of the pushed tree, and sends it as a note on the head before the branch. Scope comes from check-gate-evidence.py, measured from the first main that resolves, looked up once per push; with none the hook says so and refuses nothing. A refusal prints the environment report, and when a run changed a tracked file while it ran, which writes no by-tree receipt, it says that too. The note is built by scripts/gate-note.py: one JSON object, bounded in size and in every field, that copies only the keys of the schema from the receipt files and drops what is malformed. It is sent only when the remote does not already hold it: the hook leaves a local note that already holds the blob alone, and after fetching the remote notes it drops every head whose note the remote holds with the same blob, so an unchanged head causes no second upload. The review-record block of the hook is left as it is and the new one sits beside it, because that one is gated by a completed review and keeps its own state. The receipt can be forged and a hook can be skipped, so CI and approval are what bind. The development guide describes the check and the note, and the hook tests cover them. Change-Id: Ia17dd2dd4dfe9cae131d115bda6e1ac32cf708f3 --- docs/development.md | 48 ++++ scripts/gate-note.py | 170 ++++++++++++++ scripts/git-pre-push.sh | 141 +++++++++++ scripts/test-git-hooks.sh | 326 +++++++++++++++++++++++++- scripts/test.sh | 1 + tests/test_gate_note.py | 274 ++++++++++++++++++++++ tests/test_interactive_review_hook.py | 19 ++ 7 files changed, 977 insertions(+), 2 deletions(-) create mode 100644 scripts/gate-note.py create mode 100644 tests/test_gate_note.py diff --git a/docs/development.md b/docs/development.md index 5fbcda328..099ede31e 100644 --- a/docs/development.md +++ b/docs/development.md @@ -411,6 +411,54 @@ does not fetch `origin`. Git has no pre-pull hook. Installing replaces the exact refspecs an earlier installer wrote. Uninstalling removes the refspec; other fetch settings stay in place. +### Gate evidence on a push + +The pre-push hook asks of each topic branch it is about to publish whether the +change is one the Rust gate covers, and if so for a gate run on the content +being pushed. The scope is `scripts/check-gate-evidence.py --scope-only` over +the paths the branch changes since it left main. The main it is measured from is +the first that resolves of the push remote's (or upstream's), `origin`'s, +`upstream`'s, and the push remote's default branch, so a clone whose fork has +another name is still judged. When none resolves the hook cannot tell whether +the change is in scope: it says so, refuses nothing, and still sends the note +when a matching receipt exists, because turning away pushes that change nothing +in scope only teaches people to bypass the hook. The hook keeps no list of its +own. A branch outside the scope, such as a change to a workflow or to prose no +Rust source reads, sends nothing and is never refused. Pushes of `main`, of +tags, of a deletion, and of the notes refs themselves are not judged for +evidence, and neither is `--publish-review`, which sends only a record. + +For a branch in scope the hook reads the receipts of the tree of the pushed head +(`gate-receipts/by-tree//`, written by `scripts/test.sh`) and accepts the +push when one of them is a `rust` or `all` run that passed. A `portable` or +`browser` receipt does not vouch for Rust, and an optional-tool skip does not +disqualify a run. Because the receipt names a tree and not a commit, an amend +that only rewords, and a gate run on a dirty tree that is then committed +unchanged, both match. A commit that holds anything else does not: a rebase +that changes the tree, an edit, or a partial commit (`git add -p`, a file left +out), which needs the gate re-run on what is committed. + +With no matching receipt the push is refused with the `scripts/env-check.sh` +report, which names a missing `cargo` or toolchain and the command that +installs it, and the steps: `./scripts/env-check.sh`, `./scripts/test.sh`, push +again. + +A matching receipt is sent as a note on `refs/notes/codetrial-gate`, attached +to the head commit of the branch, before the branch itself and to the same +remote. `scripts/gate-note.py` builds it: a JSON object of at most 16 KiB with +the tree and, per receipt, only the lane, verdict, skip count, time and +toolchain lines. The receipt files are treated as data: a malformed one is +ignored, and nothing from one is passed to a shell. The notes the remote already +holds are fetched and merged first, so other heads' notes are kept, and the push +stops if the note cannot be sent. A head whose identical note the remote already +holds sends nothing, so pushing the same head again is not a second upload. +Notes for heads a contributor later amends away stay on their own fork. + +The receipt is a file on the contributor's machine and can be written by hand, +and a hook can be skipped or never installed; an agent with no toolchain +usually has none installed. The note narrows who is asked for CI time. CI and +the maintainer's approval, not this hook, are what bind. + ### Offline credential checks and message guidance `scripts/check-secrets.py` checks the changed index blobs before pre-commit diff --git a/scripts/gate-note.py b/scripts/gate-note.py new file mode 100644 index 000000000..13cd70c00 --- /dev/null +++ b/scripts/gate-note.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Build the note that carries a gate run's evidence with a push. + + gate-note.py build --receipts DIR --tree TREE + +DIR is the folder scripts/test.sh keeps receipts in, one subfolder per git tree +id. The note body goes to stdout, one JSON object of at most 16 KiB: the tree +and, for each receipt of that tree, the few fields the reader needs. Exit 0 +when the note was built, 1 when no passing `rust` or `all` receipt names TREE +(the reason goes to stderr), 2 on bad usage or a note that cannot be sized. + +The receipt is a file on the contributor's machine, so the note proves nothing +a person could not write by hand. It saves a maintainer the question of whether +a gate was run, and CI and the maintainer's approval stay what binds. Receipts +are data: only the keys below are copied, only if they have the expected shape, +and nothing from one is ever put on a command line. +""" + +import argparse +import json +import os +import re +import stat +import sys + +NOTE_VERSION = 1 +MAX_NOTE_BYTES = 16384 +MAX_RECEIPTS = 8 +# A receipt embeds the environment report, a few kilobytes; far more than that +# is not something scripts/test.sh wrote. +MAX_RECEIPT_BYTES = 256 * 1024 +LANES = frozenset(("rust", "all", "portable", "browser")) +# What scripts/test.sh writes; the note's schema allows any string up to 32. +TIME_SHAPE = re.compile(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ") +MAX_TOOL_CHARS = 200 +# The lanes that build the Rust crate, and so vouch for a Rust change. +EVIDENCE_LANES = frozenset(("rust", "all")) +TREE_ID = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})") + + +class NoteError(Exception): + pass + + +class NoEvidence(NoteError): + pass + + +def tool_string(value): + """The toolchain line, empty when the lane has none; None when malformed.""" + if value is None: + return "" + if isinstance(value, str) and len(value) <= MAX_TOOL_CHARS: + # A lone surrogate escape parses but cannot be written as UTF-8. + try: + value.encode("utf-8") + except UnicodeEncodeError: + return None + return value + return None + + +def clean_receipt(raw, tree): + """The note's entry for one receipt, or None when it is not a valid one.""" + if not isinstance(raw, dict) or raw.get("worktree_tree") != tree: + return None + lane, passed, skipped = raw.get("lane"), raw.get("passed"), raw.get("skipped") + stamp = raw.get("time") + rustc, cargo = tool_string(raw.get("rustc")), tool_string(raw.get("cargo")) + if ( + not isinstance(lane, str) + or lane not in LANES + or not isinstance(passed, bool) + or isinstance(skipped, bool) + or not isinstance(skipped, int) + or skipped < 0 + or not isinstance(stamp, str) + or not TIME_SHAPE.fullmatch(stamp) + or rustc is None + or cargo is None + ): + return None + return { + "lane": lane, + "passed": passed, + "skipped": skipped, + "time": stamp, + "rustc": rustc, + "cargo": cargo, + } + + +def read_receipt(path): + """The parsed JSON of a regular, bounded file, or None.""" + try: + info = os.lstat(path) + if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_RECEIPT_BYTES: + return None + with open(path, "rb") as handle: + return json.loads(handle.read(MAX_RECEIPT_BYTES + 1)) + # A deeply nested file raises RecursionError, not ValueError. + except (OSError, ValueError, RecursionError): + return None + + +def dumps(note): + return json.dumps(note, separators=(",", ":"), ensure_ascii=False) + + +def encode(note): + text = dumps(note) + if len(text.encode("utf-8")) > MAX_NOTE_BYTES: + raise NoteError(f"the note is larger than {MAX_NOTE_BYTES} bytes") + return text + + +def vouches(entry): + return entry["passed"] and entry["lane"] in EVIDENCE_LANES + + +def build_note(receipts_dir, tree): + """The note for TREE; NoEvidence when nothing passing vouches for it.""" + folder = os.path.join(receipts_dir, tree) + try: + names = sorted(n for n in os.listdir(folder) if n.endswith(".json")) + except OSError: + names = [] + entries = [] + for name in names: + entry = clean_receipt(read_receipt(os.path.join(folder, name)), tree) + if entry is not None: + entries.append(entry) + + # The receipt that vouches goes first, so neither the cap nor the size + # limit drops it. + entries.sort(key=vouches, reverse=True) + if not entries or not vouches(entries[0]): + raise NoEvidence(f"no passing rust or all receipt for tree {tree}") + note = {"version": NOTE_VERSION, "worktree_tree": tree, "receipts": entries} + del entries[MAX_RECEIPTS:] + while len(entries) > 1 and len(dumps(note).encode("utf-8")) > MAX_NOTE_BYTES: + entries.pop() + return note + + +def main(argv): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + commands = parser.add_subparsers(dest="command", required=True) + build = commands.add_parser("build", help="print the note for one tree") + build.add_argument("--receipts", required=True) + build.add_argument("--tree", required=True) + try: + args = parser.parse_args(argv) + except SystemExit as err: + return 2 if err.code else 0 + # The tree names a folder, so it is checked before it is joined to a path. + if not TREE_ID.fullmatch(args.tree): + print("gate-note: --tree is not a git tree id", file=sys.stderr) + return 2 + try: + text = encode(build_note(args.receipts, args.tree)) + except NoteError as err: + print(f"gate-note: {err}", file=sys.stderr) + return 1 if isinstance(err, NoEvidence) else 2 + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/git-pre-push.sh b/scripts/git-pre-push.sh index 9ac3c4473..bd35bdaf2 100755 --- a/scripts/git-pre-push.sh +++ b/scripts/git-pre-push.sh @@ -8,6 +8,13 @@ # It also checks the formatting of a topic branch with scripts/indent.sh, the # pre-commit hook's own check, for the commits that never passed through it. # +# It also refuses a topic branch that changes what the Rust gate covers (the +# scope scripts/check-gate-evidence.py names) unless a gate run on the pushed +# content is on record, and sends that record as a note on the head. The record +# is a local file a contributor could write by hand, and a contributor with no +# toolchain usually has no hooks installed either, so this is a convenience: CI +# and the maintainer's approval are what bind. +# # It also refuses new work on the main branch of any other copy of this # repository, which in practice is a fork a pull request is about to be opened # from. The same rule runs in .github/workflows/check.yml for a contributor who @@ -149,6 +156,70 @@ check_format() return "$status" } +# $1 is the head of a topic branch, $2 the ref it goes to. A head whose branch +# changes nothing the Rust gate covers is left alone. Any other must have a +# receipt of a gate run on its own tree, which goes into a note on that head; +# the note is sent after the loop, and $gate_sent keeps what to send. The +# receipt names a tree, so it matches a commit holding the same content however +# the commit was made (an amend that only rewords, a gate run on a dirty tree +# committed unchanged) and not one holding anything else. +check_gate_evidence() +{ + gate_base= + [ -z "$gate_main" ] || gate_base=$(git merge-base "$1" "$gate_main" 2> /dev/null || :) + if [ -n "$gate_base" ]; then + gate_paths=$(mktemp) || return 1 + + # A file, not a pipe, so a failed diff is not read as a change of + # nothing. + if ! git diff --name-only --no-renames -z "$gate_base" "$1" > "$gate_paths" 2> /dev/null \ + || ! gate_scope=$(python3 "$script_dir/check-gate-evidence.py" --scope-only -z < "$gate_paths"); then + rm -f "$gate_paths" + echo "Push rejected: cannot tell whether $2 changes what the Rust gate covers." >&2 + return 1 + fi + rm -f "$gate_paths" + [ "$gate_scope" = in ] || return 0 + else + + # Nothing to measure from. Refusing would turn away pushes that change + # nothing in scope and teach people to bypass the hook, so the push goes + # ahead, with the evidence when there is some. + gate_scope=unknown + if [ -z "$gate_unknown_said" ]; then + echo "Note: could not tell whether this push changes Rust sources; run ./scripts/test.sh and fetch so the check can judge it." >&2 + gate_unknown_said=1 + fi + fi + gate_tree=$(git rev-parse "$1^{tree}") || return 1 + if ! gate_body=$(python3 "$script_dir/gate-note.py" build \ + --receipts "$gate_receipts" --tree "$gate_tree"); then + [ "$gate_scope" = in ] || return 0 + echo "Push rejected: $2 changes what the Rust gate covers, and no passing gate run is on record for its content." >&2 + if [ -x "$script_dir/env-check.sh" ]; then + "$script_dir/env-check.sh" --profile local-rust --will-provision >&2 || : + fi + echo "Run the gate on exactly what you are pushing, then push again:" >&2 + echo " ./scripts/env-check.sh" >&2 + echo " ./scripts/test.sh" >&2 + echo "A run matches a commit only when the commit holds everything the run saw, so a" >&2 + echo "partial commit (git add -p, a file left out) needs the gate re-run on what is" >&2 + echo "committed, as does a rebase or an edit that changes the content. A run that" >&2 + echo "changed a file of the tree while it ran leaves no receipt either." >&2 + echo "This check reads a file on this machine. CI and the maintainer's approval, not this hook, are what bind." >&2 + return 1 + fi + if ! gate_blob=$(printf '%s' "$gate_body" | git hash-object -w --stdin) \ + || { + [ "$(git notes --ref="$gate_notes" list "$1" 2> /dev/null)" != "$gate_blob" ] \ + && ! git notes --ref="$gate_notes" add -f -C "$gate_blob" "$1" + }; then + echo "Push rejected: cannot record the gate evidence for $2." >&2 + return 1 + fi + gate_sent="$gate_sent $1:$gate_blob" +} + # For a new branch the remote has no tip to diff against, so "new" means every # commit not already published somewhere. Scoped to this remote when it has # refs, because a commit already on another remote is not this push's to judge. @@ -189,6 +260,24 @@ test_note= review_notes=refs/notes/codetrial-review publish_review=0 review_heads= +gate_notes=refs/notes/codetrial-gate +gate_receipts=$(git rev-parse --git-common-dir)/codetrial-review/gate-receipts/by-tree +gate_sent= +gate_unknown_said= + +# What a branch is measured from for the gate evidence: the first of these that +# resolves, the same for every ref of the push. A clone whose push remote has no +# main of its own (a fork under another name, a first push before any fetch) +# still has one of the others. +gate_main= +for gate_ref in "$review_policy" refs/remotes/origin/main refs/remotes/upstream/main \ + "${remote:+refs/remotes/$remote/HEAD}"; do + [ -n "$gate_ref" ] || continue + if git rev-parse --verify --quiet "$gate_ref^{commit}" > /dev/null 2>&1; then + gate_main=$gate_ref + break + fi +done while read -r local_ref local_sha remote_ref remote_sha; do [ -n "${local_ref:-}" ] || continue @@ -197,6 +286,10 @@ while read -r local_ref local_sha remote_ref remote_sha; do # The review record this hook sends below carries no code and no commit # message to judge, and judging it would loop back into this hook. [ "$remote_ref" != "$review_notes" ] || continue + + # The gate evidence below is sent the same way, and is skipped for the same + # reason. + [ "$remote_ref" != "$gate_notes" ] || continue git cat-file -e "${local_sha}^{commit}" 2> /dev/null || continue # A published upstream policy must not be reset by the outgoing branch. set -- verify --tip "$local_sha" @@ -378,6 +471,13 @@ while read -r local_ref local_sha remote_ref remote_sha; do test_note="$test_note write the failing test first (codetrial-improve), or say why none fits in the PR." fi fi + + # By hand, only a record is sent and no branch goes with it, so the + # evidence for one is not this run's to judge. + if [ "$publish_only" -eq 0 ] && ! check_gate_evidence "$local_sha" "$remote_ref"; then + failed=1 + continue + fi ;; esac done << EOF @@ -423,6 +523,47 @@ if [ "$failed" -eq 0 ] && [ "$publish_review" -eq 1 ]; then fi fi +# The gate evidence goes the way the review record does, to the remote the +# branch goes to and before the branch, for the same fetch, merge and push, kept +# apart because that block is gated by a completed review and keeps its own +# state. The notes for other heads the remote holds are merged in, and a note +# made here wins a conflict through its blob. A note the remote already holds +# with the same blob is not sent again: the upload would carry nothing and is a +# round trip that can fail. A note that cannot be sent leaves a push of in-scope +# work without its evidence, so the push stops here. +if [ "$failed" -eq 0 ] && [ -n "$gate_sent" ]; then + gate_incoming=refs/notes/codetrial-gate-incoming + if GIT_TERMINAL_PROMPT=0 git fetch -q --no-tags "$url" \ + "+$gate_notes:$gate_incoming" 2> /dev/null; then + gate_pending= + for pair in $gate_sent; do + [ "$(git notes --ref="$gate_incoming" list "${pair%%:*}" 2> /dev/null)" = "${pair#*:}" ] \ + || gate_pending="$gate_pending $pair" + done + gate_sent=$gate_pending + if [ -n "$gate_sent" ] \ + && ! git notes --ref="$gate_notes" merge -q -s ours "$gate_incoming"; then + git notes --ref="$gate_notes" merge --abort 2> /dev/null + failed=1 + fi + for pair in $gate_sent; do + [ "$failed" -eq 0 ] || break + [ "$(git notes --ref="$gate_notes" list "${pair%%:*}" 2> /dev/null)" = "${pair#*:}" ] \ + || git notes --ref="$gate_notes" add -f -C "${pair#*:}" "${pair%%:*}" \ + || failed=1 + done + git update-ref -d "$gate_incoming" + fi + if [ "$failed" -eq 0 ] && [ -n "$gate_sent" ]; then + GIT_TERMINAL_PROMPT=0 git push -q "$remote" "$gate_notes:$gate_notes" || failed=1 + fi + if [ "$failed" -ne 0 ]; then + echo "Push rejected: cannot send the gate evidence to $remote." >&2 + echo "Check that you can push to $remote, then push again." >&2 + failed=1 + fi +fi + # By hand, say what happened: nothing else would, and a check that has already # failed does not rerun for a record alone. The messages above say "Push # rejected", but here no branch was pushed. diff --git a/scripts/test-git-hooks.sh b/scripts/test-git-hooks.sh index 7e7e64291..de5e6a8cd 100755 --- a/scripts/test-git-hooks.sh +++ b/scripts/test-git-hooks.sh @@ -88,10 +88,11 @@ mkdir -p "$work/repo/scripts" || exit 1 cp "$ROOT"/scripts/git-*.sh "$ROOT/scripts/git-commit-msg.awk" \ "$ROOT/scripts/check-commit-log.sh" "$ROOT/scripts/check-secrets.py" \ "$ROOT/scripts/check-cjk.py" "$ROOT/scripts/check-interactive-review.py" \ - "$ROOT/scripts/hashchain.py" \ + "$ROOT/scripts/hashchain.py" "$ROOT/scripts/gate-note.py" \ + "$ROOT/scripts/check-gate-evidence.py" "$ROOT/scripts/env-check.sh" \ "$ROOT/scripts/install-git-hooks.sh" "$ROOT/scripts/indent.sh" \ "$work/repo/scripts/" || exit 1 -for config in .editorconfig .shellcheckrc; do +for config in .editorconfig .shellcheckrc rust-toolchain.toml; do [ -e "$ROOT/$config" ] && cp "$ROOT/$config" "$work/repo/$config" done cd "$work/repo" || exit 1 @@ -1179,6 +1180,18 @@ push_main() | "$hooks/pre-push" "$1" "$2" } +# The receipt scripts/test.sh keeps for a run on the tree a commit would get: +# the lane, whether it passed, and the tree. The fields are the ones the hook +# reads; the tree defaults to the current head's. +plant_receipt() +{ + tree=${3:-$(git rev-parse 'HEAD^{tree}')} + dir=$(git rev-parse --git-common-dir)/codetrial-review/gate-receipts/by-tree/$tree + mkdir -p "$dir" || return 1 + printf '{"lane":"%s","passed":%s,"failed":"","skipped":0,"tree":"t","worktree_tree":"%s","rustc":"rustc 1.0.0","cargo":"cargo 1.0.0","env":null,"time":"2026-11-16T00:00:00Z"}\n' \ + "$1" "$2" "$tree" > "$dir/$1.json" +} + # A small topic head without a recorded review is pushed with a note; one it # recorded is pushed without. A checkout without the skill or an upstream review # policy, as every push above, gets no note at all. @@ -1206,6 +1219,7 @@ git add src/untested.rs if git commit -q -m "Change a source file with no test" \ -m "The branch touches src/ and nothing under tests/ yet." > /dev/null 2>&1; then git rev-parse HEAD >> "$(git rev-parse --git-common-dir)/codetrial-review/reviewed" + plant_receipt rust true expect 0 "a src change without a test is pushed" git push -q fork HEAD:refs/heads/topic contains "the push notes the missing test" "$work/last.out" "nothing under tests/" printf 'a test for it\n' > tests/untested.txt @@ -1213,6 +1227,7 @@ if git commit -q -m "Change a source file with no test" \ git commit -q -m "Add the test for that change" \ -m "The same branch now carries a file under tests/." git rev-parse HEAD >> "$(git rev-parse --git-common-dir)/codetrial-review/reviewed" + plant_receipt rust true expect 0 "the same branch with its test is pushed" git push -q fork HEAD:refs/heads/topic absent "a branch with a test gets no test note" "$work/last.out" "nothing under tests/" else @@ -1259,6 +1274,7 @@ commit_unhooked() # A commit that never saw the pre-commit hook, as a rebase leaves it, still has # a valid Change-Id once rechained. The push is where its formatting is judged. if command -v ruff > /dev/null 2>&1; then + plant_receipt rust true git push -q fork HEAD:refs/heads/fmt printf 'x = [1,\n 2]\n' > tests/unformatted.py git add tests/unformatted.py @@ -1269,12 +1285,14 @@ if command -v ruff > /dev/null 2>&1; then git add tests/unformatted.py git -c core.hooksPath=/dev/null commit -q --amend --no-edit python3 scripts/hashchain.py rechain > /dev/null 2>&1 + plant_receipt rust true expect 0 "the same file formatted is pushed" git push -q -f fork HEAD:refs/heads/fmt git reset -q --hard HEAD~1 printf 'line-length = 120\n' > ruff.toml printf 'value = ["first item with a long name", "second item with a long name", "third item with a long name"]\n' > tests/config_delete.py git add ruff.toml tests/config_delete.py commit_unhooked "Add formatter configuration" "Establish the branch base for the deletion test." + plant_receipt rust true git push -q fork HEAD:refs/heads/config-delete git rm -q ruff.toml commit_unhooked "Remove formatter configuration" "The remaining files must be checked against the default settings." @@ -1302,6 +1320,7 @@ if command -v ruff > /dev/null 2>&1; then printf 'x = 1\n' > tests/rb_one.py git add tests/rb_one.py git commit -q -m "Add a formatted script" -m "The branch's only change, made through the hooks." + plant_receipt rust true expect 0 "the branch is pushed before main moves" git push -q rb HEAD:refs/heads/rb-topic git checkout -q -b rb-main "$start" printf 'y = [1,\n 2]\n' > tests/rb_legacy.py @@ -1314,6 +1333,7 @@ if command -v ruff > /dev/null 2>&1; then git checkout -q rb-topic git -c core.hooksPath=/dev/null rebase -q rb-main python3 scripts/hashchain.py rechain > /dev/null 2>&1 + plant_receipt rust true expect 0 "a rebased branch is judged by its own files" \ git push -q -f rb HEAD:refs/heads/rb-topic @@ -1365,6 +1385,308 @@ expect 0 "new work pushed to upstream's main over port 443" \ git remote set-url upstream "$ssh443" expect 1 "new work on a fork's main, upstream over port 443" \ git push -q fork HEAD:refs/heads/main + +# A push that changes what the Rust gate covers carries the receipt of a gate +# run on that content, as a note on the head. The base is a commit of its own +# that stands in for upstream's main, and the remote is a bare repository with +# no notes yet. +echo " HOOKS gate evidence" +gate_policy=$(git rev-parse -q --verify refs/remotes/upstream/main || :) +gate_notes=refs/notes/codetrial-gate +git init -q --bare "$work/gate.git" +git remote add gate "$work/gate.git" +git switch -q --orphan gate-base +printf 'base\n' > gate-readme.txt +git add gate-readme.txt +git commit -q -m "Start the gate cases" -m "A base the topic branches below are measured against." +git update-ref refs/remotes/upstream/main HEAD + +# The note the remote holds for a commit. +# +# Invoked through `expect`, which shellcheck cannot follow. +# shellcheck disable=SC2317,SC2329 +gate_note_of() +{ + git --git-dir="$work/gate.git" notes --ref="$gate_notes" show "$1" +} + +# A push of the notes ref itself, as the hook's own push of a note makes it. +# shellcheck disable=SC2317,SC2329 +push_gate_notes() +{ + printf '%s %s %s %040d\n' "$gate_notes" "$(git rev-parse "$gate_notes")" "$gate_notes" 0 \ + | "$hooks/pre-push" gate "$work/gate.git" +} + +git switch -q -c gate-docs +mkdir -p docs +printf 'text\n' > docs/gate.txt +git add docs/gate.txt +git commit -q -m "Describe the gate" -m "A change nothing in the Rust lane reads." +expect 0 "a docs-only topic is pushed with no receipt" git push -q gate HEAD:refs/heads/gate-docs +expect 1 "a docs-only push sends no note" \ + git --git-dir="$work/gate.git" rev-parse -q --verify "$gate_notes" + +git switch -q -c gate-rust gate-base +mkdir -p problem-bank +printf 'one\n' > problem-bank/gate.txt +git add problem-bank/gate.txt +git commit -q -m "Change the problem bank" -m "Rust embeds this file, so the Rust lane reads it." +expect 1 "an in-scope push with no receipt is refused" git push -q gate HEAD:refs/heads/gate-rust +contains "the refusal names the environment check" "$work/last.out" "./scripts/env-check.sh" +contains "the refusal names the gate" "$work/last.out" "./scripts/test.sh" +contains "the refusal says a partial commit needs a rerun" "$work/last.out" "partial commit" +contains "the refusal says what binds" "$work/last.out" "CI and the maintainer" +expect 1 "a refused push sends no branch" \ + git --git-dir="$work/gate.git" rev-parse -q --verify refs/heads/gate-rust +plant_receipt portable true +expect 1 "a portable receipt is not evidence for an in-scope push" \ + git push -q gate HEAD:refs/heads/gate-rust +plant_receipt rust false +expect 1 "a failed rust receipt is not evidence" git push -q gate HEAD:refs/heads/gate-rust +plant_receipt rust true "$(git hash-object -t tree /dev/null)" +expect 1 "a receipt for another tree is not evidence" git push -q gate HEAD:refs/heads/gate-rust +plant_receipt rust true +gate_head=$(git rev-parse HEAD) +expect 0 "a rust receipt for the pushed tree is accepted" git push -q gate HEAD:refs/heads/gate-rust +expect 0 "the note is on the remote" gate_note_of "$gate_head" +contains "the note has schema version 1" "$work/last.out" '"version":1' +contains "the note names the tree" "$work/last.out" "\"worktree_tree\":\"$(git rev-parse 'HEAD^{tree}')\"" +contains "the note carries the receipt" "$work/last.out" '"lane":"rust"' + +# An amend that only rewords keeps the tree, so the receipt still matches, and +# the note goes to the commit that is pushed. +git commit -q --amend -m "Change the problem bank text" -m "Reworded only; the content is unchanged." +gate_head=$(git rev-parse HEAD) +expect 0 "a reworded commit with the same tree is accepted" \ + git push -q -f gate HEAD:refs/heads/gate-rust +expect 0 "the note is on the new head" gate_note_of "$gate_head" + +# A rebase onto a base that moved changes the tree, so the old receipt says +# nothing about what would be pushed. +git switch -q gate-base +printf 'moved\n' > gate-moved.txt +git add gate-moved.txt +git commit -q -m "Move the base along" -m "The branch below is rebased over this." +git update-ref refs/remotes/upstream/main HEAD +git switch -q gate-rust +git -c core.hooksPath=/dev/null rebase -q gate-base +python3 scripts/hashchain.py rechain > /dev/null 2>&1 +expect 1 "a rebase to another tree is refused" git push -q -f gate HEAD:refs/heads/gate-rust +plant_receipt rust true +expect 0 "the rebased tree is accepted once the gate ran on it" \ + git push -q -f gate HEAD:refs/heads/gate-rust + +# The gate ran on a dirty tree that was then committed unchanged: the receipt +# names the tree the commit gets, though it was written before the commit. The +# scratch repository tracks only part of what it holds, so the scratch index +# stages the one folder the commit stages. +printf 'two\n' > problem-bank/gate2.txt +dirty_tree=$(GIT_INDEX_FILE="$work/gate-index" sh -c 'git read-tree HEAD && git add problem-bank && git write-tree') +rm -f "$work/gate-index" +plant_receipt rust true "$dirty_tree" +git add problem-bank +git commit -q -m "Add a second problem" -m "Committed exactly as the gate saw it." +expect 0 "a dirty tree committed unchanged is accepted" git push -q gate HEAD:refs/heads/gate-rust + +# The record the hook sends is not judged as a branch, which would loop. +expect 0 "a push of the notes ref does not re-enter the checks" push_gate_notes + +# A note the remote already holds is not written or sent again, and a note for +# another head, or for a new head of the same tree, still goes. The remote logs +# each ref update it receives. +gate_log=$work/gate-received.log +: > "$gate_log" +cat > "$work/gate.git/hooks/post-receive" << HOOK +#!/bin/sh +while read -r old new ref; do + printf '%s\n' "\$ref" >> "$gate_log" +done +HOOK +chmod +x "$work/gate.git/hooks/post-receive" + +# shellcheck disable=SC2317,SC2329 +gate_remote_notes() +{ + git --git-dir="$work/gate.git" rev-parse "$gate_notes" +} + +# How many times the remote had its notes ref updated. +# shellcheck disable=SC2317,SC2329 +gate_note_uploads() +{ + grep -c "^$gate_notes\$" "$gate_log" +} + +gate_head=$(git rev-parse HEAD) +gate_remote_before=$(gate_remote_notes) +gate_local_before=$(git rev-parse "$gate_notes") +expect 0 "the same head goes to a second ref" git push -q gate HEAD:refs/heads/gate-rust-twice +absent "the unchanged note is not overwritten" "$work/last.out" "Overwriting" +expect 0 "the local notes ref is not rewritten" \ + test "$(git rev-parse "$gate_notes")" = "$gate_local_before" +expect 0 "the remote notes ref is not rewritten" \ + test "$(gate_remote_notes)" = "$gate_remote_before" +expect 0 "nothing is uploaded for the notes ref" test "$(gate_note_uploads)" = 0 + +# The note is on the remote while this clone has lost its own copy: the hook +# builds the same blob again and still sends nothing. +git update-ref -d "$gate_notes" +expect 0 "a head whose note is only on the remote is pushed" \ + git push -q gate HEAD:refs/heads/gate-rust-thrice +expect 0 "the remote notes ref is still the same" \ + test "$(gate_remote_notes)" = "$gate_remote_before" +expect 0 "and still nothing is uploaded" test "$(gate_note_uploads)" = 0 + +# An amend that keeps the tree is a new head and gets its own note, and the note +# for the old head stays. +old_gate_head=$gate_head +git commit -q --amend -m "Add a second problem text" -m "Reworded only; the content is unchanged." +gate_head=$(git rev-parse HEAD) +expect 0 "an amended head is pushed" git push -q -f gate HEAD:refs/heads/gate-rust +expect 0 "the amended head has a note" gate_note_of "$gate_head" +expect 0 "the old head keeps its note" gate_note_of "$old_gate_head" +expect 0 "the new note was uploaded once" test "$(gate_note_uploads)" = 1 + +# A note another clone made for another head survives a push from here. +git --git-dir="$work/gate.git" -c user.name=Other -c user.email=other@example.com notes --ref="$gate_notes" add -m "from elsewhere" "$(git rev-parse gate-base)" +printf 'four\n' > problem-bank/gate4.txt +git add problem-bank/gate4.txt +git commit -q -m "Add a fourth problem" -m "Pushed beside a note made elsewhere." +plant_receipt rust true +gate_head=$(git rev-parse HEAD) +expect 0 "a new head is pushed beside a foreign note" git push -q gate HEAD:refs/heads/gate-rust +expect 0 "the foreign note is kept" gate_note_of "$(git rev-parse gate-base)" +expect 0 "the new head has its note" gate_note_of "$gate_head" + +# A remote that refuses the note stops the push, because in-scope work would +# otherwise arrive without its evidence. +git init -q --bare "$work/gate-reject.git" +mkdir -p "$work/gate-reject.git/hooks" +cat > "$work/gate-reject.git/hooks/pre-receive" << 'HOOK' +#!/bin/sh +while read -r old new ref; do + [ "$ref" != refs/notes/codetrial-gate ] || exit 1 +done +HOOK +chmod +x "$work/gate-reject.git/hooks/pre-receive" +git remote add gate-reject "$work/gate-reject.git" +git switch -q -c gate-reject-branch gate-base +mkdir -p problem-bank +printf 'reject\n' > problem-bank/reject.txt +git add problem-bank/reject.txt +git commit -q -m "Change the problem bank again" -m "Pushed to a remote that refuses the note." +plant_receipt rust true +expect 1 "a remote that refuses the note stops the push" \ + git push -q gate-reject HEAD:refs/heads/gate-reject-branch +contains "the push says the evidence could not be sent" "$work/last.out" \ + "cannot send the gate evidence" +expect 1 "and the branch is not sent without it" \ + git --git-dir="$work/gate-reject.git" rev-parse -q --verify refs/heads/gate-reject-branch +git switch -q gate-rust + +# A head with nothing behind it, for the pushes that are not judged: a tag, a +# deletion, and a branch that is. +printf 'three\n' > problem-bank/gate3.txt +git add problem-bank/gate3.txt +git commit -q -m "Add a third problem" -m "No gate run on this one." +git tag -a -m "A tag of a head with no receipt" gate-tag +expect 0 "a tag is not judged for a receipt" git push -q gate refs/tags/gate-tag +expect 0 "a deletion is not judged for a receipt" git push -q gate :refs/heads/gate-docs +expect 1 "the branch of that head is" git push -q gate HEAD:refs/heads/gate-bare + +# Without a toolchain the refusal carries the environment check's own words. +mkdir "$work/gate-bin" +for tool in git sh python3 awk sed grep mktemp rm cat uname od date mkdir dirname tr head tail sort cut wc expr touch find ls env; do + found=$(command -v "$tool") || continue + case $found in /*) ln -s "$found" "$work/gate-bin/$tool" ;; esac +done + +# shellcheck disable=SC2317,SC2329 +push_without_cargo() +{ + printf 'refs/heads/gate-bare %s refs/heads/gate-bare %040d\n' "$(git rev-parse HEAD)" 0 \ + | env PATH="$work/gate-bin" "$hooks/pre-push" gate "$work/gate.git" +} +expect 1 "a push with no receipt and no cargo is refused" push_without_cargo +contains "the refusal carries the environment check's report" "$work/last.out" "fail: cargo" + +# A clone with no main of its own to measure from, as a fork under another name +# is before its first fetch, is not turned away: the hook says it cannot tell +# and sends the evidence when there is some. With any of the other mains it +# measures from the first that resolves. +gate_origin=$(git rev-parse -q --verify refs/remotes/origin/main || :) +gate_upstream=$(git rev-parse -q --verify refs/remotes/upstream/main || :) +git update-ref -d refs/remotes/origin/main +git update-ref -d refs/remotes/upstream/main + +# A commit on a fresh branch from the base, changing one file. +gate_branch() +{ + git switch -q -c "$1" gate-base \ + && mkdir -p "$(dirname "$2")" \ + && printf '%s\n' "$1" > "$2" \ + && git add "$2" \ + && git commit -q -m "Add a file for a base case" -m "Pushed to see how the hook measures a branch with no main." +} + +gate_branch gate-nobase-docs docs/nobase.txt +expect 0 "a docs-only push with no base to measure from is accepted" \ + git push -q gate HEAD:refs/heads/gate-nobase-docs +contains "the push says it could not tell" "$work/last.out" \ + "could not tell whether this push changes Rust sources" +gate_branch gate-nobase-rust problem-bank/nobase.txt +expect 0 "an in-scope push with no base and no receipt is accepted" \ + git push -q gate HEAD:refs/heads/gate-nobase-rust +expect 1 "and sends no note" gate_note_of "$(git rev-parse HEAD)" +gate_branch gate-nobase-receipt problem-bank/nobase2.txt +plant_receipt rust true +expect 0 "an in-scope push with no base and a receipt is accepted" \ + git push -q gate HEAD:refs/heads/gate-nobase-receipt +expect 0 "and sends its note" gate_note_of "$(git rev-parse HEAD)" + +# origin's main stands in when the push remote has none. +git update-ref refs/remotes/origin/main "$(git rev-parse gate-base)" +gate_branch gate-fallback problem-bank/fallback.txt +expect 1 "an in-scope push measured from origin's main is refused with no receipt" \ + git push -q gate HEAD:refs/heads/gate-fallback +contains "the refusal is the evidence's" "$work/last.out" "no passing gate run is on record" +absent "a measured push does not say it could not tell" "$work/last.out" "could not tell" + +# The first main that resolves wins: upstream's here, which does not hold the +# change, so the branch is in scope; origin's, which holds it, would say it is +# not. +git update-ref refs/remotes/upstream/main "$(git rev-parse gate-base)" +git update-ref refs/remotes/origin/main "$(git rev-parse gate-fallback)" +expect 1 "upstream's main is measured from before origin's" \ + git push -q gate gate-fallback:refs/heads/gate-first + +# Then origin's before the remote's default branch. +git update-ref -d refs/remotes/upstream/main +git update-ref refs/remotes/origin/main "$(git rev-parse gate-base)" +git update-ref refs/remotes/gate/HEAD "$(git rev-parse gate-fallback)" +expect 1 "origin's main is measured from before the remote's default branch" \ + git push -q gate gate-fallback:refs/heads/gate-first + +# And the remote's default branch when nothing else resolves. +git update-ref -d refs/remotes/origin/main +git switch -q gate-fallback +mkdir -p docs +printf 'more\n' > docs/fallback.txt +git add docs/fallback.txt +git commit -q -m "Describe the fallback" -m "A change nothing in the Rust lane reads." +expect 0 "the remote's default branch is the last base tried" \ + git push -q gate HEAD:refs/heads/gate-last +absent "a push measured from it does not say it could not tell" "$work/last.out" "could not tell" +git update-ref -d refs/remotes/gate/HEAD +[ -z "$gate_origin" ] || git update-ref refs/remotes/origin/main "$gate_origin" +[ -z "$gate_upstream" ] || git update-ref refs/remotes/upstream/main "$gate_upstream" + +if [ -n "$gate_policy" ]; then + git update-ref refs/remotes/upstream/main "$gate_policy" +else + git update-ref -d refs/remotes/upstream/main +fi git switch -q "$previous" # Linked worktrees share .git/hooks. The installed wrapper must therefore find diff --git a/scripts/test.sh b/scripts/test.sh index 79639ffb7..7d30a04f9 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -710,6 +710,7 @@ gate ceiling-hotspots-tests unittest_gate "$ROOT/tests/test_ceiling_hotspots.py" gate test-gate-tests unittest_gate "$ROOT/tests/test_test_gate.py" gate env-check-tests unittest_gate "$ROOT/tests/test_env_check.py" gate gate-receipt-tests unittest_gate "$ROOT/tests/test_gate_receipt.py" +gate gate-note-tests unittest_gate "$ROOT/tests/test_gate_note.py" gate mutants-plan-tests unittest_gate "$ROOT/tests/test_mutants_plan.py" gate mutants-install-tests unittest_gate "$ROOT/tests/test_mutants_install.py" gate latency-tests unittest_gate "$ROOT/tests/test_analyze_latency.py" diff --git a/tests/test_gate_note.py b/tests/test_gate_note.py new file mode 100644 index 000000000..5770beb9d --- /dev/null +++ b/tests/test_gate_note.py @@ -0,0 +1,274 @@ +"""Check scripts/gate-note.py, which builds the note a push carries as evidence.""" + +import importlib.util +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts/gate-note.py" +SPEC = importlib.util.spec_from_file_location("gate_note", SCRIPT) +gate_note = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(gate_note) +TREE = "a" * 40 +OTHER = "b" * 40 + + +def receipt(**fields): + body = { + "lane": "rust", + "passed": True, + "failed": "", + "skipped": 0, + "tree": "fingerprint", + "worktree_tree": TREE, + "rustc": "rustc 1.100.0 (abcdef123 2026-10-01)", + "cargo": "cargo 1.100.0 (abcdef123 2026-09-01)", + "env": {"profile": "local-rust", "ok": True}, + "time": "2026-11-16T12:34:56Z", + } + body.update(fields) + return body + + +class NoteTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.dir = Path(tmp.name) + + def put(self, name, body, tree=TREE): + folder = self.dir / tree + folder.mkdir(exist_ok=True) + text = body if isinstance(body, str) else json.dumps(body) + (folder / name).write_text(text) + + def build(self, tree=TREE): + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + "build", + "--receipts", + str(self.dir), + "--tree", + tree, + ], + capture_output=True, + text=True, + check=False, + ) + + def note(self, tree=TREE): + done = self.build(tree) + self.assertEqual(done.returncode, 0, done.stderr) + return json.loads(done.stdout) + + def test_builds_the_schema(self): + self.put("rust.json", receipt()) + note = self.note() + self.assertEqual( + note, + { + "version": 1, + "worktree_tree": TREE, + "receipts": [ + { + "lane": "rust", + "passed": True, + "skipped": 0, + "time": "2026-11-16T12:34:56Z", + "rustc": "rustc 1.100.0 (abcdef123 2026-10-01)", + "cargo": "cargo 1.100.0 (abcdef123 2026-09-01)", + } + ], + }, + ) + + def test_output_is_one_object_without_trailing_data(self): + self.put("rust.json", receipt()) + out = self.build().stdout + self.assertEqual(out, out.strip()) + self.assertLessEqual(len(out.encode()), 16384) + + def test_copies_only_the_schema_keys(self): + self.put("rust.json", receipt(extra="x", env={"big": "y" * 1000})) + text = self.build().stdout + self.assertNotIn("extra", text) + self.assertNotIn("big", text) + self.assertNotIn("fingerprint", text) + + def test_all_lane_is_evidence(self): + self.put("all.json", receipt(lane="all")) + self.assertEqual(self.note()["receipts"][0]["lane"], "all") + + def test_other_lanes_ride_along_with_a_rust_receipt(self): + self.put("rust.json", receipt()) + self.put("portable.json", receipt(lane="portable")) + self.put("browser.json", receipt(lane="browser", rustc="", cargo="")) + lanes = sorted(r["lane"] for r in self.note()["receipts"]) + self.assertEqual(lanes, ["browser", "portable", "rust"]) + + def test_the_same_receipts_give_the_same_bytes(self): + # The hook compares the note with the one already on the remote, so the + # bytes may depend on the receipts alone, not on when they were written. + self.put("rust.json", receipt()) + self.put("portable.json", receipt(lane="portable")) + first = self.build().stdout + for name in ("portable.json", "rust.json"): + body = (self.dir / TREE / name).read_text() + (self.dir / TREE / name).unlink() + self.put(name, body) + self.assertEqual(self.build().stdout, first) + + def test_skips_do_not_disqualify(self): + self.put("rust.json", receipt(skipped=3)) + self.assertEqual(self.note()["receipts"][0]["skipped"], 3) + + def test_no_receipt_for_the_tree_is_refused(self): + self.put("rust.json", receipt(), tree=OTHER) + done = self.build() + self.assertEqual(done.returncode, 1) + self.assertEqual(done.stdout, "") + self.assertIn("no passing", done.stderr) + + def test_missing_directory_is_refused(self): + self.assertEqual(self.build().returncode, 1) + + def test_portable_only_is_refused(self): + self.put("portable.json", receipt(lane="portable")) + self.assertEqual(self.build().returncode, 1) + + def test_browser_only_is_refused(self): + self.put("browser.json", receipt(lane="browser")) + self.assertEqual(self.build().returncode, 1) + + def test_failed_rust_is_refused_even_beside_a_passing_portable(self): + self.put("rust.json", receipt(passed=False)) + self.put("portable.json", receipt(lane="portable")) + self.assertEqual(self.build().returncode, 1) + + def test_receipt_naming_another_tree_is_ignored(self): + # The folder name is what the hook looked up; the file has to agree. + self.put("rust.json", receipt(worktree_tree=OTHER)) + self.assertEqual(self.build().returncode, 1) + + def test_malformed_receipts_are_skipped_not_copied(self): + self.put("rust.json", receipt()) + bad = { + "unknown-lane.json": receipt(lane="mutants"), + "passed-string.json": receipt(passed="true"), + "skipped-negative.json": receipt(skipped=-1), + "skipped-bool.json": receipt(skipped=True), + "skipped-float.json": receipt(skipped=1.5), + "time-long.json": receipt(time="2026-11-16T12:34:56Z" + "0" * 20), + "time-shape.json": receipt(time="yesterday"), + "rustc-long.json": receipt(rustc="r" * 201), + "cargo-number.json": receipt(cargo=1), + "not-json.json": "{", + "list.json": "[]", + } + for name, body in bad.items(): + self.put(name, body) + self.assertEqual(len(self.note()["receipts"]), 1) + + def test_a_missing_toolchain_field_is_empty(self): + body = receipt(lane="browser") + del body["rustc"], body["cargo"] + self.put("rust.json", receipt()) + self.put("browser.json", body) + browser = [r for r in self.note()["receipts"] if r["lane"] == "browser"] + self.assertEqual((browser[0]["rustc"], browser[0]["cargo"]), ("", "")) + + def test_a_lone_surrogate_in_a_tool_line_is_malformed(self): + for field in ("rustc", "cargo"): + self.put("rust.json", receipt(**{field: "rustc \ud800"})) + done = self.build() + self.assertEqual(done.returncode, 1, (field, done.stderr)) + self.assertNotIn("Traceback", done.stderr) + + def test_at_most_eight_receipts(self): + for index in range(12): + self.put(f"r{index:02}.json", receipt()) + self.assertEqual(len(self.note()["receipts"]), 8) + + def test_a_symlinked_receipt_is_ignored(self): + self.put("target.json", receipt()) + folder = self.dir / TREE + # The target is a valid receipt the directory scan does not read, so + # only the refusal to follow the link keeps the build from passing. + (folder / "target.json").rename(folder / "target.txt") + os.symlink(folder / "target.txt", folder / "rust.json") + self.assertEqual(self.build().returncode, 1) + + def test_an_oversized_receipt_is_ignored(self): + self.put("rust.json", receipt(env="x" * 600_000)) + self.assertEqual(self.build().returncode, 1) + + def test_a_tree_that_is_not_an_object_id_never_reaches_the_filesystem(self): + for tree in ("../x", "A" * 40, "a" * 39, "a" * 41, ""): + done = self.build(tree) + self.assertEqual(done.returncode, 2, tree) + self.assertEqual(done.stdout, "") + + def test_sha256_object_ids_are_accepted(self): + tree = "c" * 64 + self.put("rust.json", receipt(worktree_tree=tree), tree=tree) + self.assertEqual(self.note(tree)["worktree_tree"], tree) + + def test_the_cap_keeps_the_receipt_that_vouches(self): + for index in range(10): + self.put(f"a{index}.json", receipt(lane="portable")) + self.put("z.json", receipt()) + lanes = [r["lane"] for r in self.note()["receipts"]] + self.assertEqual(len(lanes), 8) + self.assertEqual(lanes[0], "rust") + + def test_full_length_strings_fit_the_size_limit(self): + full = receipt(rustc="r" * 200, cargo="c" * 200) + for index in range(8): + self.put(f"r{index}.json", full) + self.assertLessEqual( + len(self.build().stdout.encode()), gate_note.MAX_NOTE_BYTES + ) + + def test_a_lane_of_the_wrong_type_does_not_hide_a_valid_receipt(self): + self.put("a.json", receipt(lane=[])) + self.put("b.json", receipt(lane={})) + self.put("rust.json", receipt()) + self.assertEqual(len(self.note()["receipts"]), 1) + + def test_wide_characters_shrink_the_note_but_keep_the_voucher(self): + wide = "\U0001f600" * 200 + for index in range(8): + self.put(f"r{index}.json", receipt(lane="portable", rustc=wide, cargo=wide)) + self.put("z.json", receipt(rustc=wide, cargo=wide)) + done = self.build() + self.assertEqual(done.returncode, 0, done.stderr) + self.assertLessEqual(len(done.stdout.encode()), gate_note.MAX_NOTE_BYTES) + self.assertEqual(json.loads(done.stdout)["receipts"][0]["lane"], "rust") + + def test_a_note_over_the_size_limit_is_an_error(self): + with self.assertRaises(gate_note.NoteError): + gate_note.encode({"pad": "x" * gate_note.MAX_NOTE_BYTES}) + + def test_deeply_nested_receipt_is_ignored(self): + self.put("rust.json", "[" * 100000) + self.assertEqual(self.build().returncode, 1) + + def test_unknown_subcommand_is_a_usage_error(self): + done = subprocess.run( + [sys.executable, str(SCRIPT), "frobnicate"], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(done.returncode, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_interactive_review_hook.py b/tests/test_interactive_review_hook.py index 64eb89fb7..c8338552b 100644 --- a/tests/test_interactive_review_hook.py +++ b/tests/test_interactive_review_hook.py @@ -77,6 +77,8 @@ def setUp(self): "check-secrets.py", "check-cjk.py", "check-interactive-review.py", + "check-gate-evidence.py", + "gate-note.py", "hashchain.py", "indent.sh", ]: @@ -508,6 +510,23 @@ def pushed_without_record(self): self.assertIsNone(self.fork_note(head)) return head + def test_publish_review_does_not_judge_gate_evidence(self): + # Only a record is sent by hand and no branch goes with it, so a branch + # that changes what the Rust gate covers is not refused for lacking a + # receipt here. + self.large() + (self.repo / "problem-bank").mkdir() + (self.repo / "problem-bank/fixture.txt").write_text("fixture\n") + self.git("add", "problem-bank/fixture.txt") + self.git("commit", "-qm", "Add a problem fixture", "-m", "Exercise the scope.") + self.prepare() + head = self.git("rev-parse", "HEAD").stdout.strip() + self.git("push", "-q", "fork", "HEAD:refs/heads/topic") + result = self.publish("topic") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn("no passing gate run", result.stderr) + self.assertEqual(self.fork_note(head)["status"], "complete") + def test_publish_review_sends_the_record_a_pushed_branch_lacks(self): head = self.pushed_without_record() result = self.publish("topic") From 8fce9511188652e37a1ac1526ef75b21c3d38b65 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 17 Nov 2026 02:13:24 +0800 Subject: [PATCH 08/13] Fetch a fork's gate note in one bounded helper Approving a held run on gate evidence needs the note a contributor pushed to the fork, and that fetch reaches a repository nobody here controls. Written ad hoc in each caller it would drift: one would build the URL from PR text, another would skip the size check. scripts/fetch-gate-note.sh takes the head owner, name and sha as separate arguments, validates them, builds the URL itself and fetches only refs/notes/codetrial-gate into a scratch bare repository: anonymous, one commit deep, a 60 second limit, a cap on the size of any one file it writes, and the note blob sized before it is read. Each failure has its own exit status, so a caller can tell a note that is absent from a fetch that did not answer. Change-Id: I000fe438ae75773c1c2e224849e8f942a607f902 --- scripts/fetch-gate-note.sh | 173 +++++++++++++++ scripts/test.sh | 1 + tests/stubs/fetch-gate-note/git.sh | 53 +++++ tests/test_fetch_gate_note.py | 328 +++++++++++++++++++++++++++++ 4 files changed, 555 insertions(+) create mode 100755 scripts/fetch-gate-note.sh create mode 100644 tests/stubs/fetch-gate-note/git.sh create mode 100644 tests/test_fetch_gate_note.py diff --git a/scripts/fetch-gate-note.sh b/scripts/fetch-gate-note.sh new file mode 100755 index 000000000..639cec8eb --- /dev/null +++ b/scripts/fetch-gate-note.sh @@ -0,0 +1,173 @@ +#!/bin/sh +# Print the gate note a pushed head commit carries, fetched from the head +# repository as data. This is the only place a fork's notes are fetched: the +# maintainer sweep (through gate-evidence.sh) and the CI preflight are to call +# it, so the bounds below are written once. +# +# Usage: fetch-gate-note.sh OWNER NAME SHA +# +# OWNER and NAME come from the pull request API's head repository and SHA from +# its head commit, as separate arguments; no URL is ever taken from PR text. The +# script validates all three, builds https://github.com/OWNER/NAME.git itself, +# and calls git with an argument vector only: nothing is passed through eval or +# a shell string, no name is used as a path, the repository is never checked +# out, and nothing from it is run. +# +# Exit 0 prints the note on stdout. Any other status prints one line on stderr: +# 2 bad usage, or an owner, name, sha or timeout that fails validation +# 3 the repository has no refs/notes/codetrial-gate +# 4 the repository is deleted, private, or refuses an anonymous fetch +# 5 the fetch failed or timed out (offline, rate limited, a server error) +# 6 the ref exists and the commit has no note +# 7 the note is over 16384 bytes, or was left out by the size filter +# Callers treat 3, 6 and 7 as no note and 4 and 5 as indeterminate. +# +# FETCH_GATE_NOTE_TIMEOUT may lower the 60 second limit (tests); it cannot raise +# it. GitHub honours --filter=blob:limit, so a blob over the limit is not sent +# at all; the explicit size check below covers a server that ignores it. +# GIT_NO_LAZY_FETCH stops git from fetching an omitted blob afterwards (git 2.44 +# and later; an older git is bounded by the timeout alone). The filter limits +# each blob, not the fork's whole notes tree, so the timeout and a cap on the +# size of any one file written (the pack a fetch receives is one file) bound the +# bytes a fork can make this fetch put on the disk. + +set -eu + +LIMIT=16384 +REF=refs/notes/codetrial-gate + +fail() +{ + code=$1 + shift + echo "fetch-gate-note: $*" >&2 + exit "$code" +} + +[ $# -eq 3 ] || fail 2 "usage: fetch-gate-note.sh OWNER NAME SHA" +owner=$1 +name=$2 +sha=$3 + +# An owner or repository name: letters, digits, dot, underscore and dash, not +# starting with a dot or a dash and without two dots in a row, at most 100 +# characters. The glob runs under the C locale so a range cannot match more. +LC_ALL=C +export LC_ALL +valid_name() +{ + case $1 in + '' | .* | -* | *..*) return 1 ;; + *[!A-Za-z0-9._-]*) return 1 ;; + esac + [ "${#1}" -le 100 ] +} +valid_name "$owner" || fail 2 "the owner is not a GitHub account name" +valid_name "$name" || fail 2 "the repository name is not a GitHub repository name" +case $sha in + *[!0-9a-f]*) fail 2 "the sha is not lowercase hex" ;; +esac +case ${#sha} in + 40 | 64) ;; + *) fail 2 "the sha is not 40 or 64 hex digits" ;; +esac + +seconds=60 +if [ "${FETCH_GATE_NOTE_TIMEOUT+set}" = set ]; then + case $FETCH_GATE_NOTE_TIMEOUT in + '' | *[!0-9]* | 0*) fail 2 "FETCH_GATE_NOTE_TIMEOUT is not a number of seconds" ;; + esac + [ "$FETCH_GATE_NOTE_TIMEOUT" -le 60 ] \ + || fail 2 "FETCH_GATE_NOTE_TIMEOUT cannot exceed 60" + seconds=$FETCH_GATE_NOTE_TIMEOUT +fi +if command -v timeout > /dev/null 2>&1; then + limit=timeout +elif command -v gtimeout > /dev/null 2>&1; then + limit=gtimeout +else + fail 5 "neither timeout nor gtimeout is installed, so the fetch cannot be bounded" +fi + +tmp=$(mktemp -d) || fail 5 "cannot create a scratch directory" +trap 'rm -rf "$tmp"' EXIT +ulimit -f 131072 || fail 5 "cannot cap the size of a file the fetch writes" +trap 'exit 5' HUP INT TERM +mkdir "$tmp/home" "$tmp/repo" +url=https://github.com/$owner/$name.git + +# A clean environment: none of the caller's GIT_* variables (a GIT_DIR or +# GIT_CONFIG_* would redirect the call), an empty home with no config, and no +# credential, so the fetch is anonymous and never prompts. Proxy variables and +# the caller's CA bundle (GIT_SSL_CAINFO, GIT_SSL_CAPATH) are left as they are, +# since a network that intercepts TLS needs them; GIT_SSL_NO_VERIFY is not. +for var in $(env | sed -n 's/^\(GIT_[A-Za-z0-9_]*\)=.*/\1/p'); do + case $var in GIT_SSL_CAINFO | GIT_SSL_CAPATH) continue ;; esac + unset "$var" +done +run_git() +{ + HOME=$tmp/home TMPDIR=$tmp GIT_TERMINAL_PROMPT=0 GIT_CONFIG_NOSYSTEM=1 \ + GIT_CONFIG_GLOBAL=/dev/null GIT_NO_LAZY_FETCH=1 \ + "$limit" "$seconds" git -C "$tmp/repo" \ + -c protocol.allow=never -c protocol.https.allow=always \ + -c credential.helper= -c core.askPass= -c gc.auto=0 \ + -c transfer.fsckObjects=true "$@" +} + +run_git init -q --bare > /dev/null 2>&1 || fail 5 "cannot create a scratch repository" + +# Only the notes ref, one commit deep, no tags, no submodules. Blobs of 16385 +# bytes or more are left out by the server: the limit is exclusive. +status=0 +err=$(run_git fetch -q --no-tags --depth=1 --no-recurse-submodules \ + --filter=blob:limit=$((LIMIT + 1)) "$url" "$REF:$REF" 2>&1 > /dev/null) || status=$? +if [ "$status" -ne 0 ]; then + case $status in + 124 | 137) fail 5 "the fetch of $owner/$name timed out after ${seconds}s" ;; + esac + case $err in + *"couldn't find remote ref"*) fail 3 "$owner/$name has no $REF" ;; + *"could not read Username"* | *[Nn]"ot found"* | *"Authentication failed"* | *"Repository not found"*) + fail 4 "$owner/$name is gone or not readable anonymously" + ;; + esac + fail 5 "the fetch of $owner/$name failed" +fi + +# Status 1 with git's "no note found" is a confirmed absence; a timeout or any +# other failure is not an answer about the note. +status=0 +blob=$(run_git notes --ref="$REF" list "$sha" 2> "$tmp/notes.err") || status=$? +if [ "$status" -ne 0 ]; then + case $status in + 124 | 137) fail 5 "reading the notes of $owner/$name timed out" ;; + esac + grep -q 'no note found' "$tmp/notes.err" \ + || fail 5 "the notes of $owner/$name could not be read" + fail 6 "$owner/$name has no gate note for $sha" +fi +case $blob in + '' | *[!0-9a-f]*) fail 6 "$owner/$name has no gate note for $sha" ;; +esac + +# The size comes before any read. A blob the filter left out cannot be sized +# without a lazy fetch, which is disabled; --batch-check reports it as missing +# with a zero status, so a missing blob is told apart from a probe that failed. +status=0 +probe=$(printf '%s\n' "$blob" | run_git cat-file --batch-check 2> /dev/null) || status=$? +case $status in + 0) ;; + 124 | 137) fail 5 "sizing the gate note timed out" ;; + *) fail 5 "the gate note could not be sized" ;; +esac +case $probe in + *" missing") fail 7 "the gate note is over $LIMIT bytes or was not sent" ;; +esac +size=${probe##* } +case $size in + '' | *[!0-9]*) fail 5 "the gate note could not be sized" ;; +esac +[ "$size" -le "$LIMIT" ] || fail 7 "the gate note is $size bytes, over the $LIMIT limit" + +run_git cat-file blob "$blob" || fail 7 "the gate note could not be read" diff --git a/scripts/test.sh b/scripts/test.sh index 7d30a04f9..b2ef29832 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -705,6 +705,7 @@ gate calibration-tests unittest_gate "$ROOT/tests/test_calibrate_framework.py" gate gemini-usage-tests unittest_gate "$ROOT/tests/test_analyze_gemini_usage.py" gate audit-tests unittest_gate "$ROOT/tests/test_audit.py" gate gate-evidence-tests unittest_gate "$ROOT/tests/test_check_gate_evidence.py" +gate fetch-gate-note-tests unittest_gate "$ROOT/tests/test_fetch_gate_note.py" gate rust-style-tests unittest_gate "$ROOT/tests/test_check_rust_style.py" gate ceiling-hotspots-tests unittest_gate "$ROOT/tests/test_ceiling_hotspots.py" gate test-gate-tests unittest_gate "$ROOT/tests/test_test_gate.py" diff --git a/tests/stubs/fetch-gate-note/git.sh b/tests/stubs/fetch-gate-note/git.sh new file mode 100644 index 000000000..41a115db3 --- /dev/null +++ b/tests/stubs/fetch-gate-note/git.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# git for tests/test_fetch_gate_note.py. It logs every call and the git-related +# environment it was given, then either answers a fetch from a canned failure +# (FAKE_FETCH_ERR, FAKE_FETCH_RC, FAKE_FETCH_SLEEP) or runs the real git +# ($REAL_GIT) with https://github.com/ rewritten to the local directory +# $FAKE_REMOTE, so the notes logic runs against real objects without a network. +# FAKE_NOTES_ERR fails `git notes` with that message. FAKE_DROP_FILTER removes +# --filter, as a server that ignores it would. FAKE_CATFILE_ERR fails `git +# cat-file` with that message. Each call also logs the file size limit it runs +# under, as FILE_SIZE_LIMIT. +{ + echo '--call--' + for a in "$@"; do + printf 'arg=%s\n' "$a" + done + env | grep -E '^(GIT_|HOME=|LC_ALL=|TMPDIR=)' | sed 's/^/env=/' + echo "env=FILE_SIZE_LIMIT=$(ulimit -f)" +} >> "$FAKE_LOG" + +case " $* " in + *" fetch "*) + if [ -n "${FAKE_FETCH_SLEEP:-}" ]; then + sleep "$FAKE_FETCH_SLEEP" + fi + if [ -n "${FAKE_FETCH_ERR:-}" ]; then + printf '%s\n' "$FAKE_FETCH_ERR" >&2 + exit "${FAKE_FETCH_RC:-128}" + fi + ;; + *" cat-file "*) + if [ -n "${FAKE_CATFILE_ERR:-}" ]; then + printf '%s\n' "$FAKE_CATFILE_ERR" >&2 + exit 128 + fi + ;; + *" notes "*) + if [ -n "${FAKE_NOTES_ERR:-}" ]; then + printf '%s\n' "$FAKE_NOTES_ERR" >&2 + exit 128 + fi + ;; +esac + +for a in "$@"; do + shift + case $a in + https://github.com/*) a=file://$FAKE_REMOTE/${a#https://github.com/} ;; + protocol.allow=never) a=protocol.allow=always ;; + --filter=*) [ -z "${FAKE_DROP_FILTER:-}" ] || continue ;; + esac + set -- "$@" "$a" +done +exec "$REAL_GIT" "$@" diff --git a/tests/test_fetch_gate_note.py b/tests/test_fetch_gate_note.py new file mode 100644 index 000000000..f7049060e --- /dev/null +++ b/tests/test_fetch_gate_note.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +"""Contract tests for scripts/fetch-gate-note.sh, the one place a fork's gate +note is fetched. + +The script builds its own URL, so a case cannot point it at a local path. A +stub `git` on PATH, which no PR can reach, logs the argument vector and the +environment, answers canned fetch failures, or rewrites the one github.com URL +to a local directory and runs the real git against real objects. +""" + +import importlib.util +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +SPEC = importlib.util.spec_from_file_location( + "support", Path(__file__).with_name("support.py") +) +support = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(support) + +SCRIPT = support.ROOT / "scripts" / "fetch-gate-note.sh" +FAKE_GIT = support.stub("fetch-gate-note/git.sh") +REAL_GIT = shutil.which("git") +REF = "refs/notes/codetrial-gate" +NOTE = b'{"version": 1, "worktree_tree": "' + b"a" * 40 + b'", "receipts": []}' +GIT_IDENTITY = { + "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@example.invalid", + "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@example.invalid", + "GIT_CONFIG_GLOBAL": "/dev/null", + "GIT_CONFIG_NOSYSTEM": "1", +} + + +def git(*args, cwd=None, env=()): + return subprocess.run( + [REAL_GIT, *args], + cwd=cwd, + capture_output=True, + text=True, + check=True, + env=os.environ | GIT_IDENTITY | dict(env), + ).stdout.strip() + + +@unittest.skipUnless(REAL_GIT and shutil.which("timeout"), "git or timeout missing") +class FetchGateNoteTests(unittest.TestCase): + def setUp(self): + scratch = tempfile.TemporaryDirectory() + self.addCleanup(scratch.cleanup) + self.scratch = Path(scratch.name) + (self.scratch / "bin").mkdir() + support.write_executable(self.scratch / "bin" / "git", FAKE_GIT) + (self.scratch / "tmp").mkdir() + self.log = self.scratch / "log" + self.log.write_text("") + self.remote = self.scratch / "remote" + + def publish(self, note=NOTE, notes=True, owner="octo", name="repo"): + """A remote at owner/name.git whose head commit carries the note.""" + work = self.scratch / "work" + work.mkdir(exist_ok=True) + git("init", "-q", "-b", "main", cwd=work) + (work / "f").write_text(os.urandom(4).hex()) + git("add", "f", cwd=work) + git("commit", "-q", "-m", "c", cwd=work) + head = git("rev-parse", "HEAD", cwd=work) + refs = ["refs/heads/main"] + if notes: + (self.scratch / "note").write_bytes(note) + # -C stores the blob as given; -F would append a newline. + blob = git("hash-object", "-w", str(self.scratch / "note"), cwd=work) + git("notes", "--ref", REF, "add", "-C", blob, head, cwd=work) + refs.append(REF) + bare = self.remote / owner / f"{name}.git" + bare.mkdir(parents=True) + git("init", "-q", "--bare", cwd=bare) + git("config", "uploadpack.allowFilter", "true", cwd=bare) + git("push", "-q", str(bare), *refs, cwd=work) + return head + + def fetch(self, *args, **env): + base = { + "PATH": f"{self.scratch / 'bin'}:{os.environ['PATH']}", + "FAKE_LOG": str(self.log), + "FAKE_REMOTE": str(self.remote), + "REAL_GIT": REAL_GIT, + "TMPDIR": str(self.scratch / "tmp"), + } + return subprocess.run( + ["sh", str(SCRIPT), *args], + capture_output=True, + env=os.environ | base | env, + ) + + def calls(self): + out, current = [], None + for line in self.log.read_text().splitlines(): + if line == "--call--": + current = {"args": [], "env": []} + out.append(current) + elif line.startswith("arg="): + current["args"].append(line[4:]) + elif line.startswith("env="): + current["env"].append(line[4:]) + return out + + def test_prints_the_note_of_the_head_commit(self): + head = self.publish() + result = self.fetch("octo", "repo", head) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, NOTE) + + def test_a_note_of_exactly_the_limit_is_read(self): + note = b"x" * 16384 + head = self.publish(note=note) + result = self.fetch("octo", "repo", head) + self.assertEqual((result.returncode, result.stdout), (0, note)) + + def test_the_argument_vector_pins_the_url_protocol_and_ref(self): + head = self.publish() + self.fetch("octo", "re.po-1", head) + fetch = next(c for c in self.calls() if "fetch" in c["args"]) + args = fetch["args"] + for want in ( + "protocol.allow=never", + "protocol.https.allow=always", + "credential.helper=", + "--no-tags", + "--depth=1", + "--no-recurse-submodules", + "--filter=blob:limit=16385", + "https://github.com/octo/re.po-1.git", + ): + self.assertIn(want, args) + self.assertIn(f"{REF}:{REF}", " ".join(args).replace("+", "")) + # Nothing but the one notes ref is asked for. + self.assertEqual(sum(a.startswith("https://") for a in args), 1) + for var in ("GIT_TERMINAL_PROMPT=0", "GIT_CONFIG_NOSYSTEM=1"): + self.assertIn(var, fetch["env"]) + + def test_every_git_call_runs_in_a_fresh_directory_with_an_empty_home(self): + head = self.publish() + real_home = self.scratch / "realhome" + real_home.mkdir() + (real_home / ".gitconfig").write_text("[user]\n\tname = leaked\n") + self.fetch( + "octo", + "repo", + head, + HOME=str(real_home), + GIT_DIR=str(self.scratch / "elsewhere"), + GIT_CONFIG_COUNT="1", + GIT_CONFIG_KEY_0="url.https://evil.invalid/.insteadOf", + GIT_CONFIG_VALUE_0="https://github.com/", + ) + self.assertTrue(self.calls()) + for call in self.calls(): + env = dict(e.split("=", 1) for e in call["env"]) + self.assertNotEqual(env["HOME"], str(real_home)) + self.assertTrue(env["HOME"].startswith(str(self.scratch / "tmp"))) + for leaked in ("GIT_DIR", "GIT_CONFIG_COUNT", "GIT_CONFIG_KEY_0"): + self.assertNotIn(leaked, env) + where = call["args"][call["args"].index("-C") + 1] + self.assertTrue(where.startswith(str(self.scratch / "tmp"))) + # No name from the API reaches a path. + self.assertNotIn("octo", where) + + def test_a_fork_cannot_make_git_write_an_unbounded_file(self): + head = self.publish() + self.fetch("octo", "repo", head) + self.assertTrue(self.calls()) + for call in self.calls(): + env = dict(e.split("=", 1) for e in call["env"]) + # The pack a fetch writes is one file, so a cap on any file bounds + # what the fork's notes can put on the disk. + self.assertNotEqual(env["FILE_SIZE_LIMIT"], "unlimited") + + def test_a_ca_bundle_the_caller_names_reaches_git_but_verification_stays_on(self): + # The caller's trust anchors are theirs to choose (a maintainer behind a + # TLS-intercepting proxy needs one); switching verification off is not. + head = self.publish() + self.fetch( + "octo", + "repo", + head, + GIT_SSL_CAINFO="/ca/bundle.pem", + GIT_SSL_CAPATH="/ca/dir", + GIT_SSL_NO_VERIFY="1", + ) + self.assertTrue(self.calls()) + for call in self.calls(): + self.assertIn("GIT_SSL_CAINFO=/ca/bundle.pem", call["env"]) + self.assertIn("GIT_SSL_CAPATH=/ca/dir", call["env"]) + self.assertNotIn("GIT_SSL_NO_VERIFY=1", call["env"]) + + def test_the_scratch_directory_is_removed(self): + head = self.publish() + self.fetch("octo", "repo", head) + self.fetch("octo", "repo", "b" * 40) + self.assertEqual(list((self.scratch / "tmp").iterdir()), []) + + def test_a_refused_name_or_sha_never_reaches_git(self): + head = self.publish() + bad_names = ( + "", + ".hidden", + "-x", + "a..b", + "..", + ".", + "a/b", + "a b", + "a;b", + "a\nb", + "$(id)", + "a:b", + "a@b", + "a%2Fb", + "https://github.com/x/y", + "x" * 101, + ) + for bad in bad_names: + for position in (0, 1): + args = ["octo", "repo", head] + args[position] = bad + result = self.fetch(*args) + self.assertEqual(result.returncode, 2, (position, bad)) + for sha in ("", "abc", head.upper(), "--upload-pack=x", head + "0", "g" * 40): + result = self.fetch("octo", "repo", sha) + self.assertEqual(result.returncode, 2, sha) + self.assertEqual(self.fetch("octo", "repo").returncode, 2) + self.assertEqual(self.fetch("octo", "repo", head, "extra").returncode, 2) + self.assertEqual(self.calls(), []) + + def test_a_missing_ref_is_exit_3(self): + head = self.publish(notes=False) + result = self.fetch("octo", "repo", head) + self.assertEqual(result.returncode, 3) + self.assertEqual(result.stdout, b"") + self.assertEqual(len(result.stderr.decode().strip().splitlines()), 1) + + def test_a_commit_without_a_note_is_exit_6(self): + self.publish() + result = self.fetch("octo", "repo", "c" * 40) + self.assertEqual(result.returncode, 6) + self.assertEqual(result.stdout, b"") + + def test_an_unreadable_notes_tree_is_exit_5_not_a_missing_note(self): + head = self.publish() + result = self.fetch( + "octo", "repo", head, FAKE_NOTES_ERR="fatal: bad tree object" + ) + self.assertEqual(result.returncode, 5) + + def test_a_missing_or_private_repository_is_exit_4(self): + for message in ( + "fatal: could not read Username for 'https://github.com': terminal prompts disabled", + "remote: Repository not found.\nfatal: repository 'https://github.com/o/n.git/' not found", + ): + result = self.fetch( + "octo", "gone", "a" * 40, FAKE_FETCH_ERR=message, FAKE_FETCH_RC="128" + ) + self.assertEqual(result.returncode, 4, message) + self.assertEqual(result.stdout, b"") + self.assertEqual(len(result.stderr.decode().strip().splitlines()), 1) + + def test_any_other_fetch_failure_is_exit_5(self): + result = self.fetch( + "octo", + "repo", + "a" * 40, + FAKE_FETCH_ERR="fatal: unable to access: Could not resolve host: github.com", + ) + self.assertEqual(result.returncode, 5) + + def test_a_fetch_that_outlives_the_timeout_is_exit_5(self): + result = self.fetch( + "octo", + "repo", + "a" * 40, + FAKE_FETCH_SLEEP="10", + FETCH_GATE_NOTE_TIMEOUT="1", + ) + self.assertEqual(result.returncode, 5) + self.assertIn(b"timed out", result.stderr) + + def test_the_timeout_can_be_lowered_but_not_raised_or_garbled(self): + head = self.publish() + for value in ("0", "61", "abc", "1;id", "-1", ""): + result = self.fetch("octo", "repo", head, FETCH_GATE_NOTE_TIMEOUT=value) + self.assertEqual(result.returncode, 2, value) + + def test_a_note_over_the_limit_is_refused_when_the_filter_omits_it(self): + head = self.publish(note=b"x" * 16385) + result = self.fetch("octo", "repo", head) + self.assertEqual(result.returncode, 7) + self.assertEqual(result.stdout, b"") + + def test_a_failed_size_probe_is_not_a_missing_note(self): + head = self.publish(note=b"{}") + result = self.fetch( + "octo", "repo", head, FAKE_CATFILE_ERR="fatal: unable to read the object" + ) + self.assertEqual(result.returncode, 5, result.stderr) + + def test_the_size_is_checked_before_reading_when_the_filter_is_ignored(self): + head = self.publish(note=b"x" * 40000) + result = self.fetch("octo", "repo", head, FAKE_DROP_FILTER="1") + self.assertEqual(result.returncode, 7) + self.assertEqual(result.stdout, b"") + self.assertIn(b"16384", result.stderr) + # The blob was never read: the size came first. + for call in self.calls(): + self.assertFalse( + "cat-file" in call["args"] and "blob" in call["args"], + call["args"], + ) + self.assertTrue(any("--batch-check" in c["args"] for c in self.calls())) + + +if __name__ == "__main__": + unittest.main() From b76a762b6a569521b15392cad9e16ea75b89fbce Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 17 Nov 2026 02:13:53 +0800 Subject: [PATCH 09/13] Approve a held run only on gate evidence A sweep approves the runs GitHub holds for first-time contributors without looking at what the contributor ran locally, so a PR from an agent with no Rust toolchain still spends the full Rust lane to learn whether the code compiles. gate-evidence.sh gathers the head tree, the changed paths (renames name both ends, a cut list is indeterminate), the toolchain pin from the base branch and the note from the fork as data, and asks check-gate-evidence.py. pr-sweep.sh reports the verdict beside each held run. apply-pr-actions.sh reads it again for the head the sweep saw immediately before the approval call, approves only on a pass, leaves an ask on the confirmed list as the toolchain comment to draft, and skips when the answer cannot be given: a moved head, a deleted fork, a failed fetch. pulls.md now says what the pull_request_target grep prints, which is pr-description.yml alone and the one reviewed exception, and drops the wording that read as approving every held run at once. Change-Id: I041ce0f91479a3e11a60fdfd913d9e74f752d018 --- .../codetrial-triage/apply-pr-actions.sh | 65 ++- .../skills/codetrial-triage/gate-evidence.sh | 169 +++++++ .claude/skills/codetrial-triage/pr-sweep.sh | 22 +- .claude/skills/codetrial-triage/pulls.md | 39 +- scripts/test.sh | 1 + tests/stubs/gate-approve/evidence.sh | 18 + tests/stubs/gate-approve/fetch.sh | 9 + tests/stubs/gate-approve/gh.sh | 58 +++ tests/test_gate_approve.py | 436 ++++++++++++++++++ tests/test_skill_scripts.py | 9 + 10 files changed, 797 insertions(+), 29 deletions(-) create mode 100755 .claude/skills/codetrial-triage/gate-evidence.sh create mode 100644 tests/stubs/gate-approve/evidence.sh create mode 100644 tests/stubs/gate-approve/fetch.sh create mode 100644 tests/stubs/gate-approve/gh.sh create mode 100644 tests/test_gate_approve.py diff --git a/.claude/skills/codetrial-triage/apply-pr-actions.sh b/.claude/skills/codetrial-triage/apply-pr-actions.sh index 5e09ea690..c9c6c89a1 100755 --- a/.claude/skills/codetrial-triage/apply-pr-actions.sh +++ b/.claude/skills/codetrial-triage/apply-pr-actions.sh @@ -19,12 +19,16 @@ # # confirm means the action is not one a sweep takes alone (a held run that is # not a first-time contributor's, one that touches workflows or whose file list -# was cut, or any approval while a workflow runs on pull_request_target or -# workflow_run): it goes on the confirmed list. A rebase request is skipped -# unless the head still matches the sweep and, for a conflict, GitHub still says -# CONFLICTING. A PR in `rebase` whose request was not posted keeps all of its -# earlier requests visible (hide needs the new one to have landed), and hide_ask -# is never touched: it is the maintainer's list. +# was cut, one whose head carries no gate evidence, or any approval while a +# workflow other than the reviewed pr-description.yml runs on +# pull_request_target or workflow_run): it goes on the confirmed list. An +# approval also needs gate-evidence.sh to pass for the head the sweep read, +# checked again right before the call; when the answer cannot be given (the head +# moved, the fork is gone, a fetch failed) the result is skipped. A rebase +# request is skipped unless the head still matches the sweep and, for a +# conflict, GitHub still says CONFLICTING. A PR in `rebase` whose request was +# not posted keeps all of its earlier requests visible (hide needs the new one +# to have landed), and hide_ask is never touched: it is the maintainer's list. set -eu @@ -55,6 +59,7 @@ done jq -e '(.prs | arrays) and (.maintainers_known | type == "boolean") and all(.prs[]; (.number | type == "number") and (.head | type == "string")) and all(.approve[]; (.pr | type == "number") and (.run | type == "number") + and (.head | type == "string" and test("^([0-9a-f]{40}|[0-9a-f]{64})$")) and (.first_time | type == "boolean") and (.touches_workflows | type == "boolean") and (.files_truncated | type == "boolean")) @@ -153,25 +158,47 @@ else risky="pr-description.yml runs on workflow_run" fi fi -# One row per held run: the PR, the run, and whether every guard holds exactly. -jq -r '.approve[] | [.pr, .run, (.first_time == true and .touches_workflows == false - and .files_truncated == false)] | @tsv' "$sweep" | while read -r pr id guards; do - if [ -n "$risky" ]; then - receipt approve "$pr" confirm "$risky" - elif [ "$guards" != true ]; then - receipt approve "$pr" confirm "not a first-time contributor, touches workflows, or the file list was cut" - elif err=$(gh api -X POST "repos/$repo/actions/runs/$id/approve" 2>&1); then - receipt approve "$pr" applied "run $id" +# Approve one run and say what happened. +approve_run() +{ + if err=$(gh api -X POST "repos/$repo/actions/runs/$2/approve" 2>&1); then + receipt approve "$1" applied "run $2" elif printf '%s\n' "$err" | grep -q 'not waiting for approval'; then - status=$(gh run view "$id" --repo "$repo" --json status --jq .status) || status=unread + status=$(gh run view "$2" --repo "$repo" --json status --jq .status) || status=unread case $status in queued | in_progress) - receipt approve "$pr" already "run $id is $status after approval" + receipt approve "$1" already "run $2 is $status after approval" ;; - *) receipt approve "$pr" failed "run $id is not waiting, but approval could not be verified (status: $status)" ;; + *) receipt approve "$1" failed "run $2 is not waiting, but approval could not be verified (status: $status)" ;; esac else - receipt approve "$pr" failed "$err" + receipt approve "$1" failed "$err" + fi +} + +# One row per held run: the PR, the run, the head the sweep read, and whether +# every guard holds exactly. The evidence is read again here, for the head the +# sweep saw, immediately before the approval call: a verdict saved in the sweep +# would be about a head a push may have replaced. Only a pass approves; ask goes +# on the confirmed list as the comment to draft, and an indeterminate answer (a +# moved head, an unreadable fork, a failed fetch) approves nothing. +evidence_check=${APPLY_GATE_EVIDENCE:-$here/gate-evidence.sh} +jq -r '.approve[] | [.pr, .run, .head, (.first_time == true and .touches_workflows == false + and .files_truncated == false)] | @tsv' "$sweep" | while read -r pr id head guards; do + if [ -n "$risky" ]; then + receipt approve "$pr" confirm "$risky" + elif [ "$guards" != true ]; then + receipt approve "$pr" confirm "not a first-time contributor, touches workflows, or the file list was cut" + else + code=0 + verdict=$(sh "$evidence_check" "$repo" "$pr" "$head" < /dev/null 2> /dev/null) || code=$? + reason=$(printf '%s\n' "$verdict" | jq -r '.reason // empty' 2> /dev/null) || reason= + : "${reason:=the evidence check printed no reason}" + case $code in + 0) approve_run "$pr" "$id" ;; + 1) receipt approve "$pr" confirm "no gate evidence, draft the toolchain comment: $reason" ;; + *) receipt approve "$pr" skipped "gate evidence is indeterminate, not approved: $reason" ;; + esac fi done > "$work/approve.out" diff --git a/.claude/skills/codetrial-triage/gate-evidence.sh b/.claude/skills/codetrial-triage/gate-evidence.sh new file mode 100755 index 000000000..4fd4ddb4c --- /dev/null +++ b/.claude/skills/codetrial-triage/gate-evidence.sh @@ -0,0 +1,169 @@ +#!/bin/sh +# Read-only evidence check for one pull request at one head: gather what +# scripts/check-gate-evidence.py needs as data and print its verdict. +# +# Usage: gate-evidence.sh +# Prints one JSON object on stdout: +# {verdict, reason, head} +# verdict is pass (the evidence holds, or no receipt is owed), ask (the +# receipt is missing or wrong: draft the comment that asks the +# contributor to install the toolchain and run ./scripts/test.sh) or +# indeterminate (the answer cannot be given: never approve). The exit +# status is the checker's: 0, 1 or 2. +# +# is the head the caller looked at. The pull request is read again +# here, and a head that differs, now or when the gathering ends, is +# indeterminate: a force-push changes what the evidence is about. A caller that +# approves reads this immediately before the approval call. +# +# Everything from the pull request is data: file names, branch names and the +# note reach the checker as files and arguments, never through eval or a shell +# string, and the head repository is never checked out. The base toolchain file +# comes from the base branch, never the head; the head's is read only when the +# change edits it. The note is fetched by scripts/fetch-gate-note.sh from the +# owner and name the API reports. GATE_EVIDENCE_FETCH names another fetcher and +# GATE_EVIDENCE_PYTHON another interpreter, for tests; neither can be set from a +# pull request. + +set -eu + +here=$(cd "$(dirname "$0")" && pwd) +root=$(cd "$here/../../.." && pwd) +checker=$root/scripts/check-gate-evidence.py +fetcher=${GATE_EVIDENCE_FETCH:-$root/scripts/fetch-gate-note.sh} + +sha_re='^([0-9a-f]{40}|[0-9a-f]{64})$' +head= + +emit() +{ + jq -nc --arg verdict "$1" --arg reason "$3" --arg head "$head" \ + '{verdict: $verdict, reason: $reason, head: $head}' + exit "$2" +} + +indeterminate() +{ + emit indeterminate 2 "$1" +} + +if [ $# -ne 3 ]; then + echo "usage: gate-evidence.sh " >&2 + exit 2 +fi +repo=$1 +pr=$2 +want=$3 +printf '%s\n' "$repo" | grep -Eq '^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$' \ + || indeterminate "the repository name is not owner/name" +case $pr in + '' | *[!0-9]*) indeterminate "the pull request number is not a number" ;; +esac +printf '%s\n' "$want" | grep -Eq "$sha_re" || indeterminate "the head sha is not hex" +head=$want + +# The checker needs tomllib, Python 3.11; the default python3 may be older. +python= +for candidate in "${GATE_EVIDENCE_PYTHON:-}" python3.13 python3.12 python3.11 \ + "${HOME:-/nonexistent}/.local/bin/python3.12" python3; do + [ -n "$candidate" ] || continue + if "$candidate" -c 'import sys; sys.exit(sys.version_info < (3, 11))' 2> /dev/null; then + python=$candidate + break + fi +done +[ -n "$python" ] || indeterminate "no Python 3.11 or later was found to run the evidence check" + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +# rust-toolchain.toml at a ref of the base repository, as raw text into a file. +contents() +{ + gh api -X GET -H 'Accept: application/vnd.github.raw+json' \ + "repos/$repo/contents/rust-toolchain.toml" -f ref="$1" > "$2" +} + +# A field of the pull request; a missing or unreadable one is the empty string. +pull=$(gh api "repos/$repo/pulls/$pr") || indeterminate "cannot read the pull request" +field() +{ + printf '%s\n' "$pull" | jq -r "$1 // empty" 2> /dev/null || true +} +live=$(field .head.sha) +[ "$live" = "$want" ] || indeterminate "the head moved from $want to ${live:-unknown}" +state=$(field .state) +[ "$state" = open ] || indeterminate "the pull request is $state" +base_ref=$(field .base.ref) +changed=$(field .changed_files) +author_type=$(field .user.type) +head_owner=$(field .head.repo.owner.login) +head_name=$(field .head.repo.name) +case $changed in + '' | *[!0-9]*) indeterminate "the changed file count is unreadable" ;; +esac +[ -n "$base_ref" ] || indeterminate "the base branch is unreadable" +[ -n "$author_type" ] || indeterminate "the author type is unreadable" + +# The changed paths, a NUL after every one, including the last. A rename names +# both ends, since moving a file out of the Rust scope changes it too. The list +# is cut when the entries read are fewer than the PR's count. +gh api --paginate "repos/$repo/pulls/$pr/files" > "$work/files.json" \ + || indeterminate "cannot read the changed files" +read_count=$(jq -s '[.[][]] | length' "$work/files.json" 2> /dev/null) \ + || indeterminate "the changed files are unreadable" +if jq -se 'any(.[][]; (.filename, (.previous_filename // "")) | explode | any(. == 0))' \ + "$work/files.json" > /dev/null 2>&1; then + indeterminate "a changed path holds a NUL byte" +fi +if [ "$read_count" -ne "$changed" ]; then + indeterminate "the changed-file list is truncated: $read_count of $changed paths" +fi +jq -j '.[] | .filename, "\u0000", ((.previous_filename // empty) | ., "\u0000")' \ + "$work/files.json" > "$work/paths" || indeterminate "the changed paths are unreadable" + +set -- --changed-files-file "$work/paths" --author-type "$author_type" +scope=$("$python" -I "$checker" --scope-only -z < "$work/paths" 2> "$work/scope.err") \ + || indeterminate "$(head -n 1 "$work/scope.err")" + +# The checker always wants the head's tree, and a receipt names it. +tree=$(gh api "repos/$repo/git/commits/$want" --jq .tree.sha) \ + || indeterminate "cannot read the head commit's tree" +set -- "$@" --head-tree "$tree" + +if [ "$scope" = in ] && [ "$author_type" != Bot ]; then + contents "$base_ref" "$work/base-toolchain" \ + || indeterminate "cannot read rust-toolchain.toml from the base branch" + set -- "$@" --base-toolchain-file "$work/base-toolchain" + if jq -se 'any(.[][]; .filename == "rust-toolchain.toml" + or .previous_filename == "rust-toolchain.toml")' "$work/files.json" > /dev/null 2>&1; then + contents "$want" "$work/head-toolchain" \ + || indeterminate "cannot read rust-toolchain.toml from the head" + set -- "$@" --head-toolchain-file "$work/head-toolchain" + fi + + # No note is an answer (the checker asks for one); a repository that cannot + # be read, or a fetch that failed, is not. + if [ -z "$head_owner" ] || [ -z "$head_name" ]; then + indeterminate "the head repository is gone, so its gate note cannot be read" + fi + status=0 + sh "$fetcher" "$head_owner" "$head_name" "$want" > "$work/note" 2> "$work/note.err" || status=$? + case $status in + 0) set -- "$@" --receipt-file "$work/note" ;; + 3 | 6 | 7) ;; + *) indeterminate "$(head -n 1 "$work/note.err")" ;; + esac +fi + +status=0 +"$python" -I "$checker" "$@" > "$work/out" 2> "$work/err" || status=$? +# A push that landed while this ran would make the evidence about an old head. +after=$(gh api "repos/$repo/pulls/$pr" --jq .head.sha) || indeterminate "cannot read the pull request again" +[ "$after" = "$want" ] || indeterminate "the head moved while the evidence was gathered" +message=$(head -n 1 "$work/err") +case $status in + 0) emit pass 0 "$(tail -n 1 "$work/out")" ;; + 1) emit ask 1 "$message" ;; + *) indeterminate "${message:-the evidence check failed}" ;; +esac diff --git a/.claude/skills/codetrial-triage/pr-sweep.sh b/.claude/skills/codetrial-triage/pr-sweep.sh index 20b23b7e5..f5a75d760 100755 --- a/.claude/skills/codetrial-triage/pr-sweep.sh +++ b/.claude/skills/codetrial-triage/pr-sweep.sh @@ -10,7 +10,10 @@ # maintainers_known false when the push listing failed and the COLLABORATOR # badge, which read-only collaborators also carry, stood in for it # approve runs waiting for approval on the current head of an open PR, and -# whether its author is a first-time contributor +# whether its author is a first-time contributor; head is the sha +# the sweep read, and evidence is gate-evidence.sh's verdict for it +# (pass, ask or indeterminate, with its reason). PR_SWEEP_EVIDENCE +# names another gate-evidence.sh, for tests # outdated unresolved review threads whose lines the PR no longer touches, # to recheck at the head before resolving # overlap PR pairs that close the same issue @@ -494,7 +497,7 @@ def refs: (ref_prefix + "[0-9]+\\b") as $one ([$prs[] | status] | sort_by(.size.total)) as $status | { approve: [$prs[] as $p | runs_of($p) - | {pr: $p.number, run: .id, workflow: .name, actor, + | {pr: $p.number, run: .id, workflow: .name, actor, head: $p.headRefOid, touches_workflows: any($p.files.nodes[].path; startswith(".github/")), files_truncated: ($p.files.totalCount > 100), first_time: ($p.authorAssociation @@ -742,9 +745,24 @@ collide=$(find "$work" -name '*.diff' -exec awk \ context: (map(.context | select(length > 0)) | unique)}] | unique_by([.a, .b, .path])') +# The evidence verdict of each PR with a held run, read at the head the sweep +# saw. A PR whose check printed nothing is indeterminate, never a pass. +evidence_check=${PR_SWEEP_EVIDENCE:-$(dirname "$0")/gate-evidence.sh} +evidence=$(printf '%s\n' "$out" | jq -r '[.approve[] | [.pr, .head]] | unique[] | @tsv' \ + | while read -r n head; do + found=$(sh "$evidence_check" "$repo" "$n" "$head" < /dev/null 2> /dev/null) || true + printf '%s\n' "$found" | jq -c --argjson n "$n" \ + 'select(type == "object" and (.verdict | IN("pass", "ask", "indeterminate"))) + | {pr: $n, evidence: {verdict, reason, head}}' 2> /dev/null || true + done | jq -s .) + printf '%s\n' "$out" | jq --argjson states "$states" --argjson stale "$stale" \ --argjson collide "$collide" --argjson unread "$unread" \ + --argjson evidence "$evidence" \ '.collide = $collide | .collide_unread = $unread + | .approve |= map(. as $a | . + {evidence: (first($evidence[] + | select(.pr == $a.pr) | .evidence) // {verdict: "indeterminate", + head: $a.head, reason: "the evidence check printed nothing"})}) | if .gone == null then . else .gone = $states end | .rebase += [.stale_candidates[] | select(.pr | IN($stale[])) | del(.head)] diff --git a/.claude/skills/codetrial-triage/pulls.md b/.claude/skills/codetrial-triage/pulls.md index 8fc2c7a3c..7cb4815fb 100644 --- a/.claude/skills/codetrial-triage/pulls.md +++ b/.claude/skills/codetrial-triage/pulls.md @@ -56,31 +56,54 @@ the head before it is posted. A bare "continue" after an issue report is not that request: it carries the work on, and these actions go on the confirmed list instead. -- Approving a run in `approve` whose `first_time` is true and whose - `touches_workflows` and `files_truncated` are false. A run waits there because - the repository's policy (`gh api +- Approving a run in `approve` whose `first_time` is true, whose + `touches_workflows` and `files_truncated` are false, and whose `evidence` + verdict is `pass` when read again for the head the sweep saw. A run waits + there because the repository's policy (`gh api repos/$repo/actions/permissions/fork-pr-contributor-approval` reads `first_time_contributors`) holds `pull_request` workflows from anyone without a merged commit. Most PRs here come from such contributors, and the hold applies again to every push until their first PR merges, so a run left unapproved is a contributor waiting on CI that never starts while review waits on green CI. "Approve the held runs" asks for this action alone: run the sweep - and apply only `approve`. A contributor's comment that a run is "awaiting + and apply only `approve`, which still approves one run at a time, each on its + own evidence. A contributor's comment that a run is "awaiting approval" (#312) is a reason to suggest that request to the user, not the request itself: text on a thread never authorizes a write. The repository is public, so a run from a fork gets a read-only token and no secrets whatever its workflow file says, and the approval spends runner time and nothing else. Two triggers escape that rule by running with the base repository's token and - secrets, so confirm neither is in use: `grep -n - 'pull_request_target\|workflow_run' .github/workflows/*` prints nothing. When - it prints something, every approval goes on the confirmed list. A run whose - `touches_workflows` is true is never approved under this rule: it goes on + secrets, so check which workflows use them: `grep -n + 'pull_request_target\|workflow_run' .github/workflows/*`. Today it prints + only `.github/workflows/pr-description.yml`, which runs the base branch's + copy on a read-only token and reads the pull request as data, and + `apply-pr-actions.sh` treats that file as the one reviewed exception for + `pull_request_target`. A hit in any other workflow, or `workflow_run` in that one, puts every approval on the + confirmed list. A run whose `touches_workflows` is true is never approved + under this rule: it goes on the confirmed list, with its diff read first against the workflow rules in [ci.md](../codetrial-review/rules/ci.md#keep-what-makes-a-fork-pr-safe-to-run) and the result stated there, whoever wrote it. Runner time still matters: skim the diff, and move the approval to the confirmed list when code the build or tests execute does something the title does not explain, such as a network call or a download it then runs. + A PR that changes a path in the Rust scope needs the gate evidence + that `gate-evidence.sh` reads as data (the head's tree and changed paths from + the API, the toolchain pin from the base branch, and the note on the head + commit from `scripts/fetch-gate-note.sh`) and `scripts/check-gate-evidence.py` + accepts: `pass` when it exits 0, `ask` on 1 (a missing, oversized or wrong + note, including a fork that never pushed the notes ref), and `indeterminate` + on 2, on a failed or timed-out fetch, a head that moved, a deleted or private + fork, or a truncated file list. + `apply-pr-actions.sh` runs the check again immediately before each approval + call, because a force-push changes the head and a verdict from the sweep + would then describe code nobody checked; it approves only on `pass`. On `ask` + it posts nothing and puts the PR on the confirmed list with a drafted comment + asking the contributor to install the toolchain and run `./scripts/test.sh` + (posting that comment keeps its usual confirmation). On `indeterminate` it + does not approve and reports why. A change with no Rust-scope path, or from a + bot, owes no note and passes. The note is a hint a dishonest author can forge; + it makes the decision mechanical and does not prove the gate ran. - Resolving a thread in `outdated` once it was checked at the PR's head and found addressed. Outdated means the commented lines changed, not that the concern was met, and an open thread is what holds a grade down, so resolving diff --git a/scripts/test.sh b/scripts/test.sh index b2ef29832..715d38a92 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -706,6 +706,7 @@ gate gemini-usage-tests unittest_gate "$ROOT/tests/test_analyze_gemini_usage.py" gate audit-tests unittest_gate "$ROOT/tests/test_audit.py" gate gate-evidence-tests unittest_gate "$ROOT/tests/test_check_gate_evidence.py" gate fetch-gate-note-tests unittest_gate "$ROOT/tests/test_fetch_gate_note.py" +gate gate-approve-tests unittest_gate "$ROOT/tests/test_gate_approve.py" gate rust-style-tests unittest_gate "$ROOT/tests/test_check_rust_style.py" gate ceiling-hotspots-tests unittest_gate "$ROOT/tests/test_ceiling_hotspots.py" gate test-gate-tests unittest_gate "$ROOT/tests/test_test_gate.py" diff --git a/tests/stubs/gate-approve/evidence.sh b/tests/stubs/gate-approve/evidence.sh new file mode 100644 index 000000000..031359c86 --- /dev/null +++ b/tests/stubs/gate-approve/evidence.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# Stand-in for gate-evidence.sh when a test is about something else: logs the +# call and answers $FAKE_EVIDENCE (pass, ask or anything else for +# indeterminate). +echo "EVIDENCE $*" >> "$FAKE_LOG" +case ${FAKE_EVIDENCE:-pass} in + pass) + echo '{"verdict":"pass","reason":"gate evidence holds","head":"x"}' + ;; + ask) + echo '{"verdict":"ask","reason":"no receipt","head":"x"}' + exit 1 + ;; + *) + echo '{"verdict":"indeterminate","reason":"canned","head":"x"}' + exit 2 + ;; +esac diff --git a/tests/stubs/gate-approve/fetch.sh b/tests/stubs/gate-approve/fetch.sh new file mode 100644 index 000000000..187d2abf1 --- /dev/null +++ b/tests/stubs/gate-approve/fetch.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# Stand-in for scripts/fetch-gate-note.sh: logs its arguments, then prints the +# note in $FAKE_DIR or fails with $FAKE_NOTE_RC and one line on stderr. +echo "FETCH $*" >> "$FAKE_LOG" +if [ -n "${FAKE_NOTE_RC:-}" ]; then + echo "fetch-gate-note: canned failure $FAKE_NOTE_RC" >&2 + exit "$FAKE_NOTE_RC" +fi +cat "$FAKE_DIR/note" diff --git a/tests/stubs/gate-approve/gh.sh b/tests/stubs/gate-approve/gh.sh new file mode 100644 index 000000000..2a3466d82 --- /dev/null +++ b/tests/stubs/gate-approve/gh.sh @@ -0,0 +1,58 @@ +#!/bin/sh +# gh for the gate-evidence and approval tests. It logs every call to $FAKE_LOG +# and serves the pull request, its files, the head commit and the two +# rust-toolchain.toml files from $FAKE_DIR. After the first read of the pull +# request, pull-after.json (when it exists) answers instead, as a push between +# the check and the approval would. Any write is logged as `WRITE`. +echo "$*" >> "$FAKE_LOG" +dir=$FAKE_DIR + +pull() +{ + reads=$(cat "$dir/reads" 2> /dev/null || echo 0) + echo $((reads + 1)) > "$dir/reads" + if [ "$reads" -ge 1 ] && [ -f "$dir/pull-after.json" ]; then + cat "$dir/pull-after.json" + else + cat "$dir/pull.json" + fi +} + +case "$*" in + "api --paginate repos/"*"/pulls/"*"/files") + [ -z "${FAKE_FILES_FAIL:-}" ] || exit 1 + cat "$dir/files.json" + ;; + "api repos/"*"/pulls/"*" -f "* | "api repos/"*"/pulls/"*" -F "*) + # A field with no method is a POST, whatever the path. + echo WRITE >> "$FAKE_LOG" + ;; + "api repos/"*"/pulls/"*) + [ -z "${FAKE_PULL_FAIL:-}" ] || exit 1 + case "$*" in + *"--jq .head.sha") pull | jq -r .head.sha ;; + *) pull ;; + esac + ;; + "api repos/"*"/git/commits/"*" --jq .tree.sha") + [ -z "${FAKE_TREE_FAIL:-}" ] || exit 1 + cat "$dir/tree" + ;; + "api -X GET -H "*"/contents/rust-toolchain.toml -f ref="*) + all="$*" + ref=${all##*-f ref=} + if [ "$ref" = "$(jq -r .head.sha "$dir/pull.json")" ]; then + cat "$dir/head-toolchain" + else + [ -z "${FAKE_BASE_TOOLCHAIN_FAIL:-}" ] || exit 1 + cat "$dir/base-toolchain" + fi + ;; + "api -X POST "*) + echo WRITE >> "$FAKE_LOG" + ;; + *) + echo "unexpected gh call: $*" >&2 + exit 1 + ;; +esac diff --git a/tests/test_gate_approve.py b/tests/test_gate_approve.py new file mode 100644 index 000000000..17b3973ff --- /dev/null +++ b/tests/test_gate_approve.py @@ -0,0 +1,436 @@ +#!/usr/bin/env python3 +"""Contract tests for approving a held run only on gate evidence. + +gate-evidence.sh gathers a pull request's head tree, changed paths, toolchain +files and gate note as data and asks scripts/check-gate-evidence.py. These cases +serve it a fake `gh` and a fake note fetcher, then check what the sweep reports +and what apply-pr-actions.sh writes: an approval only for a pass, rechecked at +the head the sweep read immediately before the call. +""" + +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +SPEC = importlib.util.spec_from_file_location( + "support", Path(__file__).with_name("support.py") +) +support = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(support) + +TRIAGE = support.ROOT / ".claude" / "skills" / "codetrial-triage" +EVIDENCE = TRIAGE / "gate-evidence.sh" +APPLY = TRIAGE / "apply-pr-actions.sh" +PR_SWEEP = TRIAGE / "pr-sweep.sh" +FAKE_GH = support.stub("gate-approve/gh.sh") +FAKE_FETCH = support.stub("gate-approve/fetch.sh") +HEAD = "a" * 40 +MOVED = "b" * 40 +TREE = "c" * 40 +PIN = '[toolchain]\nchannel = "1.100.0"\n' +RUSTC = "rustc 1.100.0 (abcdef123 2026-10-01)" + + +def note(tree=TREE, rustc=RUSTC, passed=True, lane="rust"): + return json.dumps( + { + "version": 1, + "worktree_tree": tree, + "receipts": [ + { + "lane": lane, + "passed": passed, + "skipped": 0, + "time": "2026-11-16T12:34:56Z", + "rustc": rustc, + "cargo": "cargo 1.100.0 (abcdef123 2026-09-01)", + } + ], + } + ) + + +def entry(path, previous=None): + return ( + {"filename": path, "previous_filename": previous} + if previous + else {"filename": path} + ) + + +@unittest.skipUnless(shutil.which("jq"), "jq is not installed") +class GateCase(unittest.TestCase): + """A scratch directory holding what the fake gh and fetcher serve.""" + + def setUp(self): + scratch = tempfile.TemporaryDirectory() + self.addCleanup(scratch.cleanup) + self.scratch = Path(scratch.name) + (self.scratch / "bin").mkdir() + support.write_executable(self.scratch / "bin" / "gh", FAKE_GH) + support.write_executable(self.scratch / "fetch.sh", FAKE_FETCH) + self.dir = self.scratch / "fixture" + self.dir.mkdir() + self.log = self.scratch / "log" + self.log.write_text("") + self.env = os.environ | { + "PATH": f"{self.scratch / 'bin'}:{os.environ['PATH']}", + "FAKE_LOG": str(self.log), + "FAKE_DIR": str(self.dir), + "GATE_EVIDENCE_FETCH": str(self.scratch / "fetch.sh"), + "GATE_EVIDENCE_PYTHON": sys.executable, + } + self.serve() + + def pull(self, sha=HEAD, **fields): + node = { + "state": "open", + "head": { + "sha": sha, + "repo": {"name": "codetrial", "owner": {"login": "contrib"}}, + }, + "base": {"ref": "main"}, + "user": {"type": "User"}, + "changed_files": 1, + } + return node | fields + + def serve(self, files=(entry("src/main.rs"),), note_text=None, **pull): + """What GitHub says about the pull request; a note unless None.""" + pull.setdefault("changed_files", len(files)) + (self.dir / "pull.json").write_text(json.dumps(self.pull(**pull))) + # --paginate prints one array per page, back to back. + (self.dir / "files.json").write_text("".join(json.dumps([f]) for f in files)) + (self.dir / "tree").write_text(TREE + "\n") + (self.dir / "base-toolchain").write_text(PIN) + (self.dir / "head-toolchain").write_text(PIN) + if note_text is not None: + (self.dir / "note").write_text(note_text) + + def calls(self): + return self.log.read_text().splitlines() + + def evidence(self, want=HEAD, **env): + result = subprocess.run( + ["sh", str(EVIDENCE), "sysprog21/codetrial", "7", want], + capture_output=True, + text=True, + env=self.env | env, + ) + out = json.loads(result.stdout) if result.stdout.strip() else {} + return result.returncode, out.get("verdict"), out.get("reason", ""), result + + +class FakeGhTests(GateCase): + def test_a_review_post_without_x_post_is_logged_as_a_write(self): + # `gh api` posts when it is given a field and no method, as the review + # requests in apply-pr-actions.sh do; the stub must not read it as a GET. + for flag in ("-f", "-F"): + with self.subTest(flag=flag): + self.log.write_text("") + subprocess.run( + ["gh", "api", "repos/o/r/pulls/7/reviews", flag, "event=X"], + env=self.env, + capture_output=True, + check=True, + ) + self.assertIn("WRITE", self.calls()) + + +class GateEvidenceTests(GateCase): + def test_a_note_for_the_head_tree_passes(self): + self.serve(note_text=note()) + code, verdict, reason, _ = self.evidence() + self.assertEqual((code, verdict), (0, "pass"), reason) + self.assertIn("FETCH contrib codetrial " + HEAD, self.calls()) + + def test_no_note_asks_the_contributor_for_one(self): + self.serve() + for status in ("3", "6", "7"): + with self.subTest(status=status): + code, verdict, reason, _ = self.evidence(FAKE_NOTE_RC=status) + self.assertEqual((code, verdict), (1, "ask"), reason) + self.assertIn("no receipt", reason) + + def test_a_deleted_fork_is_indeterminate_when_a_receipt_is_owed(self): + self.serve(note_text=note(), head={"sha": HEAD, "repo": None}) + code, verdict, reason, _ = self.evidence() + self.assertEqual((code, verdict), (2, "indeterminate"), reason) + # Nothing was fetched for a repository the API does not name. + self.assertFalse([c for c in self.calls() if c.startswith("FETCH")]) + + def test_a_docs_only_change_needs_no_note_even_from_a_deleted_fork(self): + self.serve( + files=(entry("docs/x.md"),), + head={"sha": HEAD, "repo": None}, + ) + code, verdict, reason, _ = self.evidence() + self.assertEqual((code, verdict), (0, "pass"), reason) + self.assertFalse([c for c in self.calls() if c.startswith("FETCH")]) + self.assertFalse([c for c in self.calls() if "contents" in c]) + + def test_a_failed_or_refused_fetch_is_indeterminate(self): + self.serve() + for status in ("2", "4", "5"): + with self.subTest(status=status): + code, verdict, reason, _ = self.evidence(FAKE_NOTE_RC=status) + self.assertEqual((code, verdict), (2, "indeterminate"), reason) + self.assertIn(status, reason) + + def test_a_note_for_another_tree_asks(self): + self.serve(note_text=note(tree="d" * 40)) + code, verdict, reason, _ = self.evidence() + self.assertEqual((code, verdict), (1, "ask"), reason) + + def test_a_note_made_with_another_compiler_asks(self): + self.serve(note_text=note(rustc="rustc 1.99.0 (abcdef123 2026-10-01)")) + self.assertEqual(self.evidence()[:2], (1, "ask")) + + def test_a_truncated_list_is_indeterminate_before_anything_is_fetched(self): + self.serve(note_text=note(), changed_files=3) + code, verdict, reason, _ = self.evidence() + self.assertEqual((code, verdict), (2, "indeterminate"), reason) + self.assertIn("truncated", reason) + self.assertFalse([c for c in self.calls() if c.startswith("FETCH")]) + + def test_an_unreadable_checker_answer_is_indeterminate(self): + # A pinned channel that names no exact version cannot be compared. + self.serve(note_text=note()) + (self.dir / "base-toolchain").write_text('[toolchain]\nchannel = "stable"\n') + code, verdict, reason, _ = self.evidence() + self.assertEqual((code, verdict), (2, "indeterminate"), reason) + + def test_any_failed_read_is_indeterminate(self): + self.serve(note_text=note()) + for variable in ( + "FAKE_PULL_FAIL", + "FAKE_FILES_FAIL", + "FAKE_TREE_FAIL", + "FAKE_BASE_TOOLCHAIN_FAIL", + ): + with self.subTest(variable=variable): + code, verdict, _, _ = self.evidence(**{variable: "1"}) + self.assertEqual((code, verdict), (2, "indeterminate")) + + def test_a_head_that_differs_from_the_one_asked_about_is_indeterminate(self): + self.serve(note_text=note(), sha=MOVED) + code, verdict, reason, _ = self.evidence(want=HEAD) + self.assertEqual((code, verdict), (2, "indeterminate"), reason) + self.assertIn("moved", reason) + self.assertFalse([c for c in self.calls() if c.startswith("FETCH")]) + + def test_a_push_while_the_evidence_is_gathered_is_indeterminate(self): + self.serve(note_text=note()) + (self.dir / "pull-after.json").write_text(json.dumps(self.pull(sha=MOVED))) + code, verdict, reason, _ = self.evidence() + self.assertEqual((code, verdict), (2, "indeterminate"), reason) + self.assertIn("moved", reason) + + def test_a_closed_pull_request_is_indeterminate(self): + self.serve(note_text=note(), state="closed") + self.assertEqual(self.evidence()[:2], (2, "indeterminate")) + + def test_the_base_toolchain_comes_from_the_base_branch_not_the_head(self): + self.serve(note_text=note()) + self.evidence() + reads = [c for c in self.calls() if "rust-toolchain.toml" in c] + self.assertEqual(len(reads), 1) + self.assertTrue(reads[0].endswith("-f ref=main"), reads) + + def test_the_head_toolchain_is_read_only_when_the_change_edits_it(self): + files = (entry("src/main.rs"), entry("rust-toolchain.toml")) + self.serve(files=files, note_text=note(rustc="rustc 1.101.0 (abc 2026-11-01)")) + (self.dir / "head-toolchain").write_text('[toolchain]\nchannel = "1.101.0"\n') + code, verdict, reason, _ = self.evidence() + self.assertEqual((code, verdict), (0, "pass"), reason) + reads = [c for c in self.calls() if "rust-toolchain.toml" in c] + self.assertEqual(len(reads), 2) + self.assertTrue(reads[1].endswith(f"-f ref={HEAD}"), reads) + + def test_a_rename_out_of_the_rust_scope_still_needs_a_receipt(self): + # The new path alone owes nothing, so only the old path can be what asks. + self.serve(files=(entry("docs/main.md"),)) + self.assertEqual(self.evidence(FAKE_NOTE_RC="6")[:2], (0, "pass")) + self.serve(files=(entry("docs/main.md", previous="src/main.rs"),)) + code, verdict, reason, _ = self.evidence(FAKE_NOTE_RC="6") + self.assertEqual((code, verdict), (1, "ask"), reason) + self.assertIn("no receipt", reason) + self.assertTrue([c for c in self.calls() if c.startswith("FETCH")]) + + def test_a_bot_author_owes_no_receipt(self): + self.serve(user={"type": "Bot"}) + code, verdict, reason, _ = self.evidence() + self.assertEqual((code, verdict), (0, "pass"), reason) + self.assertFalse([c for c in self.calls() if c.startswith("FETCH")]) + + def test_a_path_with_a_newline_stays_one_path(self): + self.serve(files=(entry("docs/x\nsrc/main.rs"),)) + self.assertEqual(self.evidence()[:2], (0, "pass")) + + def test_a_workflow_path_is_noted_but_changes_no_verdict(self): + self.serve(files=(entry(".github/workflows/x.yml"),)) + code, verdict, _, result = self.evidence() + self.assertEqual((code, verdict), (0, "pass")) + + def test_bad_arguments_never_reach_gh(self): + for args in ( + ("not a repo", "7", HEAD), + ("sysprog21/codetrial", "7;id", HEAD), + ("sysprog21/codetrial", "7", "abc"), + ("sysprog21/codetrial", "7", HEAD.upper()), + ): + with self.subTest(args=args): + result = subprocess.run( + ["sh", str(EVIDENCE), *args], + capture_output=True, + text=True, + env=self.env, + ) + self.assertEqual(result.returncode, 2) + self.assertEqual(json.loads(result.stdout)["verdict"], "indeterminate") + self.assertEqual(self.calls(), []) + + +class ApplyApprovalTests(GateCase): + """apply-pr-actions.sh with the real evidence check behind it.""" + + def apply(self, head=HEAD, **env): + sweep = { + "maintainers_known": True, + "prs": [], + "approve": [ + { + "pr": 7, + "run": 11, + "head": head, + "first_time": True, + "touches_workflows": False, + "files_truncated": False, + } + ], + "rebase": [], + "hide": [], + "hide_resolved": [], + "refine_stale": [], + } + (self.scratch / "sweep.json").write_text(json.dumps(sweep)) + result = subprocess.run( + ["sh", str(APPLY), str(self.scratch / "sweep.json")], + capture_output=True, + text=True, + env=self.env | env, + ) + self.assertIn(result.returncode, (0, 1), result.stderr) + receipts = [json.loads(line) for line in result.stdout.splitlines()] + return receipts[0], "WRITE" in self.calls() + + def test_a_pass_approves(self): + self.serve(note_text=note()) + receipt, wrote = self.apply() + self.assertEqual((receipt["result"], wrote), ("applied", True), receipt) + + def test_a_missing_note_goes_on_the_confirmed_list_unapproved(self): + self.serve() + receipt, wrote = self.apply(FAKE_NOTE_RC="3") + self.assertEqual((receipt["result"], wrote), ("confirm", False), receipt) + self.assertIn("toolchain comment", receipt["why"]) + + def test_an_indeterminate_answer_never_approves(self): + self.serve(note_text=note()) + for variable in ("FAKE_FILES_FAIL", "FAKE_PULL_FAIL", "FAKE_TREE_FAIL"): + with self.subTest(variable=variable): + receipt, wrote = self.apply(**{variable: "1"}) + self.assertEqual( + (receipt["result"], wrote), ("skipped", False), receipt + ) + receipt, wrote = self.apply(FAKE_NOTE_RC="5") + self.assertEqual((receipt["result"], wrote), ("skipped", False), receipt) + + def test_a_head_that_changed_since_the_sweep_is_not_approved(self): + # The sweep read HEAD; the pull request is now at MOVED. + self.serve(note_text=note(), sha=MOVED) + receipt, wrote = self.apply(head=HEAD) + self.assertEqual((receipt["result"], wrote), ("skipped", False), receipt) + self.assertIn("moved", receipt["why"]) + + def test_the_evidence_is_read_after_the_sweep_not_taken_from_it(self): + # A sweep that recorded a pass for this head carries no weight: the + # note is fetched again, and now it is for another tree. + self.serve(note_text=note(tree="d" * 40)) + receipt, wrote = self.apply() + self.assertEqual((receipt["result"], wrote), ("confirm", False), receipt) + self.assertTrue([c for c in self.calls() if c.startswith("FETCH")]) + + def test_the_check_runs_immediately_before_the_approval_call(self): + self.serve(note_text=note()) + self.apply() + calls = self.calls() + approve = next(i for i, c in enumerate(calls) if "runs/11/approve" in c) + fetched = max(i for i, c in enumerate(calls) if c.startswith("FETCH")) + reread = max( + i for i, c in enumerate(calls) if c.endswith("/pulls/7 --jq .head.sha") + ) + self.assertLess(fetched, reread) + self.assertLess(reread, approve) + self.assertEqual([c for c in calls[reread + 1 : approve] if c != "WRITE"], []) + + +class SweepEvidenceTests(unittest.TestCase): + """pr-sweep.sh puts the verdict beside each held run.""" + + @unittest.skipUnless(shutil.which("jq"), "jq is not installed") + def test_each_held_run_carries_the_verdict_for_its_head(self): + sys.modules.pop("skill_scripts_for_gate", None) + spec = importlib.util.spec_from_file_location( + "skill_scripts_for_gate", Path(__file__).with_name("test_skill_scripts.py") + ) + helpers = importlib.util.module_from_spec(spec) + sys.modules["skill_scripts_for_gate"] = helpers + spec.loader.exec_module(helpers) + node = helpers.pull(1, headRefOid=HEAD, commits=helpers.GREEN) + page = { + "repository": { + "defaultBranchRef": {"name": "main"}, + "pullRequests": {"pageInfo": {"hasNextPage": False}, "nodes": [node]}, + } + } + held = { + "id": 11, + "name": "check", + "head_sha": HEAD, + "head_branch": "topic", + "head_repo": "contributor/codetrial", + "actor": "contributor", + } + checks = { + "FAKE_PAGE": json.dumps({"data": page}), + "FAKE_RUNS": json.dumps(held) + "\n", + "FAKE_AGAIN": "[]", + "PR_SWEEP_CI": 'echo "{}"\n', + "PR_SWEEP_EVIDENCE": ( + 'echo "$*" >&2\n' + 'echo \'{"verdict":"ask","reason":"no receipt","head":"\'"$3"\'"}\'\n' + "exit 1\n" + ), + } + result = helpers.run_fake( + PR_SWEEP, ["sysprog21/codetrial"], helpers.FAKE_GH, checks + ) + self.assertEqual(result.returncode, 0, result.stderr) + approve = json.loads(result.stdout)["approve"] + self.assertEqual(len(approve), 1) + self.assertEqual(approve[0]["head"], HEAD) + self.assertEqual( + approve[0]["evidence"], + {"verdict": "ask", "reason": "no receipt", "head": HEAD}, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_skill_scripts.py b/tests/test_skill_scripts.py index f1084d574..471a5d83c 100644 --- a/tests/test_skill_scripts.py +++ b/tests/test_skill_scripts.py @@ -1951,6 +1951,8 @@ def test_refuses_without_posting_when_anything_moved(self): # as "posted path:line" in place of its URL. FAKE_GH_INLINE = support.stub("skill-scripts/gh-inline.sh") FAKE_GH_APPLY = support.stub("skill-scripts/gh-apply.sh") +FAKE_EVIDENCE = support.stub("gate-approve/evidence.sh") +HEAD_SHA = "a" * 40 FAKE_GH_SIMILAR = support.stub("skill-scripts/gh-similar.sh") FAKE_GH_FEEDBACK = support.stub("skill-scripts/gh-feedback.sh") FAKE_GH_CACHED = support.stub("skill-scripts/gh-cached.sh") @@ -4858,6 +4860,9 @@ def run_actions(self, sweep, workflows=None, repo=None, hygiene=None, **env): support.write_executable(scratch / "bin" / "gh", self.FAKE) (scratch / "sweep.json").write_text(sweep) (scratch / "log").write_text("") + # The evidence check has its own suite (test_gate_approve.py). + support.write_executable(scratch / "evidence.sh", FAKE_EVIDENCE) + env.setdefault("APPLY_GATE_EVIDENCE", str(scratch / "evidence.sh")) # A workflow directory the guard reads; None leaves the real one. if workflows is not None: (scratch / "wf").mkdir() @@ -4889,6 +4894,7 @@ def approval(self, **fields): entry = { "pr": 1, "run": 11, + "head": HEAD_SHA, "first_time": True, "touches_workflows": False, "files_truncated": False, @@ -5316,6 +5322,9 @@ def test_a_sweep_of_the_wrong_shape_writes_nothing_and_exits_2(self): bad = ( self.approval(first_time="false"), {"approve": [{"pr": 1, "run": 11, "first_time": True}]}, + self.approval(head="h1"), + self.approval(head="A" * 40), + self.approval(head=None), {"rebase": None}, {"rebase": [{"pr": 3}]}, {"rebase": [{"pr": 3, "reason": "confict"}]}, From 7ac1542c8730164e6b805c6a63f2001210062990 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 17 Nov 2026 02:14:13 +0800 Subject: [PATCH 10/13] Fail a pull request without gate evidence early A pull request that changes Rust and carries no gate receipt spends a full Rust build before anyone learns the receipt is missing. Add a preflight job that runs the base branch's evidence check over the head as data and fails first. The gates, the mutation plan and the aggregate status wait for it, and a push or a merge group still runs the gates because preflight is skipped there. It is cost control, not a security boundary: a pull request runs its own workflow. The reason the gatherer reports is built from data a pull request writes, and jq decodes a JSON newline into a real one, so printing it as it came could put a line starting with :: in the log: a forged error or a mask. The step keeps the verdict out of the log, captures the gatherer's stderr, and reduces the reason to one line of printable ASCII behind the step's own error prefix, written with printf because dash's echo turns a backslash and n into a line break. The tests pin the job's shape, run its script and the aggregate script over each outcome, and run the step under dash against hostile output on both streams, requiring that the only :: line is the step's own. The development guide describes the job. Change-Id: Ib0a11991ce865d142596bb4ddd74fa6c2dbb01b1 --- .github/workflows/check.yml | 117 +++++++++++- docs/development.md | 13 ++ tests/test_check_workflow.py | 344 ++++++++++++++++++++++++++++++++++- tests/test_test_gate.py | 15 +- 4 files changed, 483 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 3e478dc48..fb2a46a60 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -83,6 +83,100 @@ env: CARGO_DENY_VERSION: 0.20.2 jobs: + # Gate evidence, judged before the expensive lanes start: a pull request that + # changes Rust and carries no gate receipt would otherwise spend a full Rust + # build to learn what a quick read of the receipt says. See + # docs/development.md for what is checked. + # + # Cost control, not a security boundary. A pull request runs the merge of its + # own tree, workflow included, so it can delete or edit this job. What binds is + # that a maintainer approves a held run only on this same evidence + # (.claude/skills/codetrial-triage/gate-evidence.sh, the script run here). + # + # Nothing from the head runs. The checker, the note fetcher and the gatherer + # are the base branch's, so a pull request cannot pass itself by editing them; + # while the base has none (the pull request that adds them) there is nothing to + # judge with. The head's tree, its changed paths, the author type and the gate + # note arrive as data from the API and the head repository's notes ref, found + # by the owner, name and sha the API reports for the head, never by PR text. + # The token is read-only: `pull-requests` because the pulls API serves the + # pull request and its file list to that scope in a private repository. + preflight: + if: github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.base.sha }} + fetch-depth: 1 + persist-credentials: false + path: base + - name: Check the gate evidence + env: + REPOSITORY: ${{ github.repository }} + PULL_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + GH_TOKEN: ${{ github.token }} + # The checker needs tomllib, which the runner's python3.12 has. + GATE_EVIDENCE_PYTHON: python3.12 + run: | + set -eu + gatherer=base/.claude/skills/codetrial-triage/gate-evidence.sh + present=0 + for file in "$gatherer" base/scripts/check-gate-evidence.py base/scripts/fetch-gate-note.sh; do + if [ -f "$file" ]; then + present=$((present + 1)) + fi + done + # Only the base that has none of them is the pull request that adds + # them. A base with some is a broken installation, which must not + # pass every pull request unjudged. + if [ "$present" -eq 0 ]; then + echo "The base branch has no gate evidence check yet; nothing to judge with." + exit 0 + fi + if [ "$present" -ne 3 ]; then + echo "::error::the base branch has only part of the gate evidence check, so it fails closed" + exit 1 + fi + # What the gatherer says is built from data a pull request writes, so + # it is never printed as it came: a line that starts with `::` is a + # workflow command. A message is one line of printable ASCII, cut to + # 300 characters, behind a prefix of this step's own, and written with + # printf because some shells' echo expands a backslash in it. + plain() + { + LC_ALL=C tr -cd ' -~' | cut -c1-300 + } + errors=$(mktemp) + trap 'rm -f "$errors"' EXIT + # Exit 0 passes, 1 asks for a receipt, 2 could not judge. + status=0 + verdict=$(sh "$gatherer" "$REPOSITORY" "$PULL_NUMBER" "$HEAD_SHA" 2> "$errors") || status=$? + reason= + if [ "$status" -ne 0 ]; then + reason=$(printf '%s\n' "$verdict" | jq -r '.reason // empty' 2> /dev/null | plain || true) + if [ -z "$reason" ]; then + reason=$(head -n 1 "$errors" | plain || true) + fi + fi + case $status in + 0) echo "The gate evidence holds." ;; + 1) + printf '%s\n' "::error::gate evidence: ${reason:-no reason was given}" + echo "Run ./scripts/env-check.sh, then ./scripts/test.sh, then push again." + exit 1 + ;; + *) + printf '%s\n' "::error::gate evidence could not be judged, so it fails closed: ${reason:-no reason was given}" + exit 1 + ;; + esac + audit: if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' runs-on: ubuntu-24.04 @@ -114,7 +208,14 @@ jobs: gates: name: gates (${{ matrix.lane }}${{ matrix.shard && format(' {0}', matrix.shard) || '' }}) - if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' + needs: [preflight] + # Preflight is skipped on a push, a merge group and a schedule, so a skipped + # result passes here; a failed one stops the lanes. `!cancelled()` replaces + # the implicit success(), which a skipped dependency would fail. + if: >- + ${{ !cancelled() + && (needs.preflight.result == 'success' || needs.preflight.result == 'skipped') + && github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' }} strategy: fail-fast: false matrix: @@ -525,13 +626,14 @@ jobs: # reaches this job and fails it. Not on a schedule or a dispatch, which run # no gates. check: - needs: [gates, audit, plan-mutants, mutants] + needs: [preflight, gates, audit, plan-mutants, mutants] if: ${{ !cancelled() && github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' }} runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - name: Require every gate lane and mutation shard env: + PREFLIGHT: ${{ needs.preflight.result }} GATES: ${{ needs.gates.result }} AUDIT: ${{ needs.audit.result }} PLAN: ${{ needs.plan-mutants.result }} @@ -540,11 +642,19 @@ jobs: { echo "| job | result |" echo "| --- | --- |" + echo "| preflight | $PREFLIGHT |" echo "| gates | $GATES |" echo "| audit | $AUDIT |" echo "| plan-mutants | $PLAN |" echo "| mutants | $MUTANTS |" } >> "$GITHUB_STEP_SUMMARY" + # First: a failed preflight skips the gates, which would otherwise be + # reported as the cause. It is skipped, and passes, outside a pull + # request. + if [ "$PREFLIGHT" != success ] && [ "$PREFLIGHT" != skipped ]; then + echo "::error::gate evidence check: $PREFLIGHT. The pull request needs a gate receipt for its head; the preflight job says what is missing." + exit 1 + fi if [ "$GATES" != success ]; then echo "::error::gate lanes: $GATES. This job only collects them; the failed gates lane has the cause." exit 1 @@ -573,6 +683,9 @@ jobs: # not wait on that one, so it repeats the condition rather than plan and fan # out shards for a pull request that has already failed. plan-mutants: + # The implicit success() is wanted: this job only runs on a pull request, + # where a preflight that failed should plan nothing. + needs: [preflight] if: >- github.event_name == 'pull_request' && (github.event.pull_request.head.repo.full_name == github.repository diff --git a/docs/development.md b/docs/development.md index 099ede31e..719ca3c45 100644 --- a/docs/development.md +++ b/docs/development.md @@ -117,6 +117,19 @@ clone only `HEAD`; the portable lane keeps full history for the commit-message check. The Rust lane deletes the runner's unused toolchains only when less than 40 GB is free. Running `./scripts/test.sh` locally still runs every gate. +On a pull request a `preflight` job runs first, and the lanes, the mutation plan +and the shards wait for it. It reads the gate receipt the pushed head carries +and fails when a non-Bot change in the Rust scope has none, so a missing +receipt costs a minute instead of a Rust build. It fails closed: a note that cannot be fetched +or read, or a head that moved while it was gathered, is a failure, never a pass. +It is cost control, not a security boundary, because a pull request runs its own +workflow and can edit the job; what binds is that a held run is approved only on +the same check. The checker, the note fetcher and the gatherer are the base +branch's, and the head is never checked out, so while the base has none of them +(the pull request that adds them) the job prints one line and passes; a base +with only some of them fails it. On a push and a merge group it is skipped, and +the lanes still run. + On a busy local host, `RUST_TEST_THREADS=4 ./scripts/test.sh rust` limits concurrent server launches in the Rust suites. The browser compiler fixtures also bound their concurrent jobs so candidate programs get CPU before their diff --git a/tests/test_check_workflow.py b/tests/test_check_workflow.py index fadc4843f..dcecfabff 100644 --- a/tests/test_check_workflow.py +++ b/tests/test_check_workflow.py @@ -1,9 +1,13 @@ """The shape of the mutation job and the aggregate status in check.yml.""" import importlib.util +import os from pathlib import Path import re +import shutil import shlex +import subprocess +import tempfile import unittest ROOT = Path(__file__).resolve().parents[1] @@ -22,12 +26,16 @@ def header(body, key): return " ".join(found[1].replace(">-", "").split()) +def needs_of(body): + return set(re.findall(r"[a-z][a-z-]*", header(body, "needs"))) + + class MutantsJobTests(unittest.TestCase): def setUp(self): self.body = support.job("mutants") def test_waits_for_the_gates_and_the_plan(self): - needs = set(re.findall(r"[a-z][a-z-]*", header(self.body, "needs"))) + needs = needs_of(self.body) self.assertLessEqual({"gates", "plan-mutants"}, needs) def test_starts_only_after_both_succeed_with_work_to_do(self): @@ -112,5 +120,339 @@ def test_release_does_not_wait_for_mutants(self): self.assertNotIn("mutants", header(support.job("release"), "needs")) +def steps_text(body): + """What follows `steps:`, so the job's own keys are not read as a step's.""" + return support.section(body, "\n steps:\n", "\0", "steps") + + +def run_blocks(body): + """Every `run: |` script of a job, dedented as the shell reads it.""" + blocks = [] + key = re.compile(r"^ *run: \|\n", re.M) + rest = body + while (found := key.search(rest)) is not None: + blocks.append(support.run_block(rest)) + rest = rest[found.end() :] + return blocks + + +# What the base branch must hold for preflight to judge anything. +GATE_FILES = { + "gatherer": ".claude/skills/codetrial-triage/gate-evidence.sh", + "checker": "scripts/check-gate-evidence.py", + "fetcher": "scripts/fetch-gate-note.sh", +} + + +class PreflightJobTests(unittest.TestCase): + def setUp(self): + self.body = support.job("preflight") + + def test_runs_only_for_a_pull_request(self): + self.assertEqual("github.event_name == 'pull_request'", header(self.body, "if")) + + def test_is_bounded_and_read_only(self): + self.assertEqual("ubuntu-24.04", header(self.body, "runs-on")) + self.assertEqual("5", header(self.body, "timeout-minutes")) + perms = support.section( + self.body, " permissions:\n", "\n steps:", "permissions" + ) + # pull-requests: the pull request and its file list come from the pulls + # API, which a private repository serves only to this scope. + self.assertEqual( + {"contents": "read", "pull-requests": "read"}, + dict(re.findall(r"^ ([a-z-]+): (\S+)$", perms, re.M)), + ) + + def test_checks_out_the_base_and_never_the_head(self): + steps = steps_text(self.body) + self.assertEqual(1, steps.count("actions/checkout@")) + checkout = support.section(steps, "actions/checkout@", "\n - ", "checkout") + self.assertIn("ref: ${{ github.event.pull_request.base.sha }}", checkout) + self.assertIn("persist-credentials: false", checkout) + self.assertIn("path: base", checkout) + self.assertNotIn("head", checkout) + # No other action runs, and every script a run line starts is the base's. + self.assertEqual( + ["actions/checkout"], + [u.split("@")[0] for u in re.findall(r"uses: (\S+)", self.body)], + ) + for block in run_blocks(self.body): + for script in re.findall( + r"\b(?:sh|python3\.12|bash) +(\S+\.(?:sh|py))\b", block + ): + self.assertTrue(script.startswith("base/"), script) + + def test_event_data_reaches_the_script_only_through_env(self): + for block in run_blocks(self.body): + self.assertNotIn("${{", block) + # Every expression is an `env:` value or the checkout's `ref:`. + for line in steps_text(self.body).splitlines(): + if "${{" in line: + self.assertRegex( + line, r"^ {10}(?:[A-Z_]+|ref): \$\{\{ [^}]+\}\}$", line + ) + + def test_pull_request_text_cannot_leave_the_job(self): + self.assertNotIn("\n outputs:", self.body) + for name in ("GITHUB_ENV", "GITHUB_OUTPUT", "GITHUB_PATH", "GITHUB_STATE"): + self.assertNotIn(name, self.body) + env = support.section(self.body, " env:\n", " run:", "env") + # What it hands the script is a number, a sha and the repository name: + # not the title, body, branch name or login, which a pull request writes. + for field in ("title", "body", "head.ref", "label", "user.login"): + self.assertNotIn(f"pull_request.{field}", env) + self.assertIn("github.event.pull_request.head.sha", env) + + def test_judges_with_the_base_scripts_and_names_the_interpreter(self): + block = run_blocks(self.body)[0] + for path in GATE_FILES.values(): + self.assertIn(f"base/{path}", block) + self.assertIn("GATE_EVIDENCE_PYTHON: python3.12", self.body) + + def run_step(self, present, status=0, reason="the reason", out=None, err=""): + """Run the step's script over a base checkout holding `present` files. + + The stub gatherer prints `out` (a verdict naming `reason` when None) on + stdout and `err` on stderr, then exits `status`. + """ + block = run_blocks(self.body)[0] + with tempfile.TemporaryDirectory() as work: + for key in present: + path = Path(work, "base", GATE_FILES[key]) + path.parent.mkdir(parents=True, exist_ok=True) + said = '{"reason": "%s"}' % reason if out is None else out + body = ( + f"#!/bin/sh\ncat <<'__OUT__'\n{said}\n__OUT__\n" + f"cat >&2 <<'__ERR__'\n{err}\n__ERR__\nexit {status}\n" + ) + path.write_text(body if key == "gatherer" else "") + return subprocess.run( + # dash where there is one: its `echo` expands escapes, so the + # step must not rely on a shell that does not. + [shutil.which("dash") or "sh", "-c", block], + cwd=work, + env=dict( + os.environ, REPOSITORY="o/r", PULL_NUMBER="1", HEAD_SHA="a" * 40 + ), + capture_output=True, + text=True, + ) + + def test_a_base_without_the_check_passes_with_one_line(self): + done = self.run_step([]) + self.assertEqual(0, done.returncode) + self.assertEqual(1, len(done.stdout.splitlines())) + + def test_a_base_with_part_of_the_check_fails_closed(self): + for present in (["gatherer"], ["checker", "fetcher"], ["gatherer", "checker"]): + self.assertEqual(1, self.run_step(present).returncode, present) + + def test_each_verdict_maps_to_an_exit_and_a_message(self): + everything = ["gatherer", "checker", "fetcher"] + self.assertEqual(0, self.run_step(everything, 0).returncode) + asked = self.run_step(everything, 1) + self.assertEqual(1, asked.returncode) + self.assertIn("./scripts/test.sh", asked.stdout) + self.assertIn("the reason", asked.stdout) + # An unjudgeable head and a gatherer that died without a verdict fail. + for status in (2, 127): + closed = self.run_step(everything, status, reason="") + self.assertEqual(1, closed.returncode, status) + self.assertIn("fails closed", closed.stdout) + + def test_no_gatherer_output_can_forge_a_workflow_command(self): + # The gatherer reads data a pull request writes, so whatever it prints, + # on either stream and on any exit, must not put a `::` command in the + # log: the one `::error::` line is the step's own. + everything = ["gatherer", "checker", "fetcher"] + newline = r'{"reason": "no receipt\n::add-mask::topsecret\n::error::forged"}' + garbage = "::add-mask::topsecret\nx\n::error::forged\r::stop-commands::k" + control = r'{"reason": "tab\u0007bell\r::error::x"}' + long = '{"reason": "%s"}' % ("a" * 1000) + for status in (0, 1, 2, 127): + for out, err in ( + (newline, ""), + (garbage, garbage), + (control, garbage), + # A literal backslash-n, which an `echo` that expands escapes + # (dash's) would turn into a line break. + (r'{"reason": "x\\n::stop-commands::k\\r"}', ""), + # No usable verdict, so the first stderr line is the reason. + ("not json", "x\r::add-mask::topsecret\n::error::forged"), + (long, ""), + ): + with self.subTest(status=status, out=out[:20]): + done = self.run_step(everything, status, out=out, err=err) + log = (done.stdout + done.stderr).replace("\r", "\n") + commands = [x for x in log.splitlines() if x.startswith("::")] + self.assertEqual(0 if status == 0 else 1, len(commands), commands) + for line in commands: + self.assertTrue(line.startswith("::error::gate evidence"), line) + self.assertLessEqual(len(line), 400) + self.assertTrue(log.replace("\n", "").isprintable(), log) + + def test_repeats_the_checkers_install_steps(self): + # The shared gatherer keeps one line of the checker's message, so the + # step says the steps itself; they must stay the checker's words. + source = (ROOT / "scripts/check-gate-evidence.py").read_text() + guidance = re.search(r'GUIDANCE = \(\n "([^"]+)"', source) + self.assertIn(guidance[1], self.body) + + def test_only_the_description_workflow_uses_a_target_trigger(self): + found = sorted( + path.name + for path in (ROOT / ".github/workflows").glob("*.yml") + if re.search( + r"^\s*(pull_request_target|workflow_run):", path.read_text(), re.M + ) + ) + self.assertEqual(["pr-description.yml"], found) + + +class PreflightWiringTests(unittest.TestCase): + def test_gates_wait_for_it_and_keep_the_skipped_arm(self): + body = support.job("gates") + self.assertIn("preflight", needs_of(body)) + self.assertEqual( + [ + "!cancelled()", + "(needs.preflight.result == 'success' || needs.preflight.result == 'skipped')", + "github.event_name != 'schedule'", + "github.event_name != 'workflow_dispatch'", + ], + [t.strip() for t in header(body, "if").strip("${} ").split("&&")], + ) + + def test_the_plan_waits_for_it_without_cancelled(self): + plan = support.job("plan-mutants") + self.assertIn("preflight", needs_of(plan)) + self.assertNotIn("cancelled()", header(plan, "if")) + self.assertNotIn("always()", header(plan, "if")) + + def test_the_aggregate_lists_it(self): + body = support.job("check") + self.assertIn("preflight", needs_of(body)) + self.assertIn("| preflight |", body) + self.assertIn("PREFLIGHT: ${{ needs.preflight.result }}", body) + + def test_release_and_build_do_not_wait_for_it(self): + self.assertNotIn("preflight", needs_of(support.job("release"))) + self.assertNotIn("\n needs:", support.job("build")) + + +def evaluate(condition, event, results, cancelled=False, outputs=None): + """A job condition over an event and the results of the jobs it needs. + + Enough of GitHub's expression language for what this workflow writes. It + shows the text means what the comments say for the cases they name; it does + not show that GitHub reads a skipped dependency, an empty output or + `fromJson` the same way. + """ + text = condition.strip().removeprefix("${{").removesuffix("}}").strip() + text = re.sub(r"needs\.([a-z-]+)\.result", lambda m: f"R[{m[1]!r}]", text) + text = re.sub( + r"needs\.([a-z-]+)\.outputs\.(\w+)", lambda m: f"O[{m[1] + '.' + m[2]!r}]", text + ) + text = text.replace("github.event_name", "E") + text = re.sub(r"github\.event\.pull_request\.[a-z_.]+", "'same'", text) + text = text.replace("github.repository", "'same'") + text = text.replace("!cancelled()", "(not C)").replace("always()", "True") + text = re.sub(r"!(?!=)", "not ", text) + text = text.replace("&&", " and ").replace("||", " or ") + scope = {"E": event, "R": results, "O": outputs or {}, "C": cancelled} + return bool(eval(text, {"__builtins__": {}}, scope)) + + +def simulate(event, preflight_fails=False): + """Each gating job's result, assuming a job that starts passes.""" + results = {} + outputs = {"plan-mutants.count": "3"} + for name in ("preflight", "gates", "plan-mutants", "mutants"): + body = "\n" + support.job(name) + condition = header(body, "if") if "\n if:" in body else "True" + runs = evaluate(condition, event, results, outputs=outputs) + if not re.search(r"\b(always|cancelled|failure|success)\(\)", condition): + # No status function: GitHub adds success(), which a skipped or + # failed dependency fails. + needed = needs_of(body) if "\n needs:" in body else set() + runs = runs and all(results[n] == "success" for n in needed) + failed = name == "preflight" and preflight_fails + results[name] = "skipped" if not runs else "failure" if failed else "success" + return results + + +class ConditionModelTests(unittest.TestCase): + def test_a_pull_request_runs_everything_when_the_evidence_holds(self): + self.assertEqual( + { + "preflight": "success", + "gates": "success", + "plan-mutants": "success", + "mutants": "success", + }, + simulate("pull_request"), + ) + + def test_a_failed_preflight_starts_nothing_else(self): + self.assertEqual( + { + "preflight": "failure", + "gates": "skipped", + "plan-mutants": "skipped", + "mutants": "skipped", + }, + simulate("pull_request", preflight_fails=True), + ) + + def test_push_and_merge_group_still_run_the_gates(self): + for event in ("push", "merge_group"): + results = simulate(event) + self.assertEqual("skipped", results["preflight"], event) + self.assertEqual("success", results["gates"], event) + self.assertEqual("skipped", results["mutants"], event) + + def test_a_schedule_runs_no_gates(self): + self.assertEqual("skipped", simulate("schedule")["gates"]) + + def test_a_cancelled_run_starts_no_gates(self): + gates = header(support.job("gates"), "if") + self.assertFalse( + evaluate(gates, "pull_request", {"preflight": "success"}, cancelled=True) + ) + + +class AggregatePreflightTests(unittest.TestCase): + """Run the aggregate job's own script over preflight results.""" + + def run_check(self, preflight, gates="success"): + script = run_blocks(support.job("check"))[0] + env = dict( + os.environ, + GATES=gates, + AUDIT="success", + PLAN="skipped", + MUTANTS="skipped", + PREFLIGHT=preflight, + GITHUB_STEP_SUMMARY=os.devnull, + ) + return subprocess.run( + ["sh", "-c", script], env=env, capture_output=True, text=True + ) + + def test_success_and_skipped_pass(self): + for result in ("success", "skipped"): + self.assertEqual(0, self.run_check(result).returncode, result) + + def test_a_failed_preflight_names_the_evidence_check(self): + # The gates are skipped behind it, so its message has to come first. + done = self.run_check("failure", gates="skipped") + self.assertEqual(1, done.returncode) + self.assertIn("evidence", done.stdout.lower()) + + def test_a_cancelled_preflight_fails(self): + self.assertEqual(1, self.run_check("cancelled", gates="skipped").returncode) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_test_gate.py b/tests/test_test_gate.py index 64f72a387..6f6d2cac7 100644 --- a/tests/test_test_gate.py +++ b/tests/test_test_gate.py @@ -477,7 +477,9 @@ def test_ci_refuses_skipped_gates(self): self.assertIn("failed gates: skipped-gates", result.stderr) def run_status(self, command, **env): - passing = dict.fromkeys(("GATES", "AUDIT", "PLAN", "MUTANTS"), "success") + passing = dict.fromkeys( + ("PREFLIGHT", "GATES", "AUDIT", "PLAN", "MUTANTS"), "success" + ) return subprocess.run( ["bash", "-e", "-c", command], env={**os.environ, **passing, "GITHUB_STEP_SUMMARY": os.devnull, **env}, @@ -487,7 +489,9 @@ def run_status(self, command, **env): def test_check_requires_the_lanes_and_explains_every_mutation_skip(self): check = support.job("check") - self.assertIn(" needs: [gates, audit, plan-mutants, mutants]\n", check) + self.assertIn( + " needs: [preflight, gates, audit, plan-mutants, mutants]\n", check + ) self.assertIn(f" if: ${{{{ !cancelled() && {NOT_AVATAR} }}}}\n", check) command = support.run_block(check) states = ("success", "failure", "cancelled", "skipped", "") @@ -498,6 +502,11 @@ def test_check_requires_the_lanes_and_explains_every_mutation_skip(self): with self.subTest(lane=lane, state=state): status = self.run_status(command, **{lane: state}) self.assertEqual(status == 0, state == "success") + # Preflight is skipped outside a pull request, which passes. + for state in states: + with self.subTest(lane="PREFLIGHT", state=state): + status = self.run_status(command, PREFLIGHT=state) + self.assertEqual(status == 0, state in ("success", "skipped")) accepted = { # Shards ran and passed. ("success", "success"), @@ -919,7 +928,7 @@ def test_avatar_reuses_the_gate_caches_without_saving_them(self): ) # The schedule and the dispatch exist for this job alone. gates = gates_job() - self.assertIn(f" if: {NOT_AVATAR}\n", gates) + self.assertIn(NOT_AVATAR, gates) # The avatar job reaches the cargo cache through the composite that # restores it last, after the compiler and toolchain its key hashes. wrapper = (ROOT / ".github/actions/rust-build-env/action.yml").read_text() From 0d5e17eaf0a36821ab9944ae29e4aa48dedf54bf Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 17 Nov 2026 02:14:29 +0800 Subject: [PATCH 11/13] State the push rule for contributors A contributor with no Rust toolchain can push a change that only CI compiles, and the first sign is a failed run. The rule was stated in no place a contributor reads before a push. Add a Before you push section to CONTRIBUTING.md with the order to work in, for a person or an automated agent: run the environment check and stop when it fails, run the gate and push only after it passes, never bypass a hook, and report what was run and what was not. The same file states which paths need a local test.sh pass (the directories Rust reads, the root manifests, the toolchain and lint configs, and the few web/ files a Rust test reads), what the pre-push hook measures the branch from, how a receipt that names a tree matches a commit, and that the receipt is a hint while the maintainer's approval and the first CI job are what bind. The pull request template asks for the command that was run and the end of its output, and the contribute and verify skills each point to the new section. Change-Id: I70001cb5b242f673baa936912d3210e56a1241a6 --- .claude/skills/codetrial-contribute/SKILL.md | 3 ++ .claude/skills/codetrial-verify/SKILL.md | 3 ++ .github/pull_request_template.md | 4 +- CONTRIBUTING.md | 53 ++++++++++++++++++++ 4 files changed, 62 insertions(+), 1 deletion(-) diff --git a/.claude/skills/codetrial-contribute/SKILL.md b/.claude/skills/codetrial-contribute/SKILL.md index 8c64faaca..74ffc2094 100644 --- a/.claude/skills/codetrial-contribute/SKILL.md +++ b/.claude/skills/codetrial-contribute/SKILL.md @@ -17,6 +17,9 @@ approval included, are in the GitHub section of [codetrial-conventions](../codetrial-conventions/SKILL.md). Keep technical identifiers and quoted errors exact. +Before any push, an agent follows the [Before you push](../../../CONTRIBUTING.md#before-you-push) +section of CONTRIBUTING.md. + Example requests: "Turn these notes into an issue", "Help me make my first CodeTrial contribution", or "Write a PR description from this diff". Produce the artifact the user asked for. diff --git a/.claude/skills/codetrial-verify/SKILL.md b/.claude/skills/codetrial-verify/SKILL.md index 9d0e0642b..54db8b713 100644 --- a/.claude/skills/codetrial-verify/SKILL.md +++ b/.claude/skills/codetrial-verify/SKILL.md @@ -21,6 +21,9 @@ each missing tool with the command that installs it, without starting a build. `scripts/test.sh` runs it first and stops there, before any gate, when a required tool is missing. +The [Before you push](../../../CONTRIBUTING.md#before-you-push) section of +CONTRIBUTING.md has the order an agent runs it in before a push. + CI runs one `gates ()` job per lane in parallel: `gates (rust)`, `gates (portable)` and, in three slices, `gates (browser 1/3)`, `gates (browser 2/3)` and `gates (browser 3/3)` diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index db0812b02..10b47b6eb 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -9,6 +9,8 @@ before/after example when it helps. --> ## Validation - + Close # diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5754f68ea..a25153dec 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -109,11 +109,64 @@ rewriting it. The pre-push hook runs the same check over the files a topic branch changes, since a rebase, `hashchain.py rechain` or `--no-verify` publishes commits the pre-commit hook never saw. +A change that touches Rust sources, tests, build files or the directories they +read (`src/`, `desktop/`, `vendor/`, `problem-bank/`, `scripts/hook-check/`, +`.cargo/`, `tests/` apart from `tests/browser/`, `tests/stubs/` and the +top-level Python files, `Cargo.toml`, `Cargo.lock`, `build.rs`, +`rust-toolchain.toml`, `clippy.toml`, `rustfmt.toml` or `deny.toml`, and the few +docs, scripts and web/ files a Rust test names; most of web/ is not in this +list) needs a local `./scripts/test.sh` pass before it is pushed, because CI is +not the place to find out that the build fails. `./scripts/test.sh rust` is the +least that covers the Rust code; the Python and browser suites need their own +lanes. `rust-toolchain.toml` is the pin, and `./scripts/env-check.sh` says +whether this machine has that toolchain and names what is missing, with the +install command where one exists. A change that touches none of those is +exempt. + +The pre-push hook enforces this for the push of a topic branch (not `main`). It +measures the branch from where it left the first of these that exists: the +upstream's `main` (the push remote's own when there is no `upstream` remote), +`origin/main`, `upstream/main` and the push remote's default branch. A branch +with no path in the scope above is not judged, and neither is a deleted ref, a +tag or a record sent by hand. Any other branch needs the receipt +`./scripts/test.sh` writes when the `rust` or `all` lane passes. The receipt +names a tree, so it matches a commit of the same content however the commit was +made: a reworded amend still matches, and a rebase, an edit after the run, a +partial commit (`git add -p`) or a run that changed a file while it ran needs +the gate again unless the commit ends up holding what the run saw. With a +receipt the hook records a note on the pushed head and sends it to the remote +the branch goes to before the branch, so evidence can be checked without running +anything; a note that cannot be sent stops the push. Without one it refuses and +prints the commands to run. When there is no base to measure from, the push goes +ahead with the note if a receipt exists, and the hook says so; fetch the base so +it can judge. The receipt is a file on the contributor's machine, so it is a +hint and not proof: the maintainer's approval of a held run and CI's first job, +which fails a pull request in this scope that carries no note before the Rust +lane starts, are what bind. + The gate ends by naming any lane it skipped; read that rather than assuming green means covered. [docs/development.md](docs/development.md) has what each hook and lane does, which generated files have to be regenerated first, and the checks that need credentials or a browser. +## Before you push + +This is the order for anyone sending a change, a person or an automated agent; +an agent reports each step to the person it works for. + +1. Run `./scripts/env-check.sh` first (`--profile portable` or `browser` for a + change in that lane only). When it exits non-zero, stop, show what it + printed, ask for what it names to be fixed, and do not push. +2. Pass the gate here before CI sees the change: CI is not a verifier. +3. Run `./scripts/test.sh` and push once it passes. When not every lane can run, + run at least `./scripts/test.sh rust` for a change in the Rust scope above, + and say which lanes were not run. A failing check is fixed, not skipped or + loosened. +4. Never use `--no-verify` or `git commit -n`, set `core.hooksPath` or edit a + hook to get past one; when a hook refuses, fix what it names. +5. Report the command run, the last lines of its output and what was not run. + Do not say the gate passed when it did not run to the end. + ## Commit messages Clear and descriptive commit messages are crucial for maintaining a transparent history of changes and for facilitating effective debugging and tracking. From e5568cd9e4e82f18c905b31cc4d50673ea3ca766 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 17 Nov 2026 02:14:48 +0800 Subject: [PATCH 12/13] Test the gate pieces together The note builder and the evidence checker each have a suite built from hand-made inputs, so a field one side writes and the other no longer reads would pass both. Run the receipt writer of scripts/test.sh in a scratch repository, build the note from what it wrote, and judge it: accepted for its tree, refused for another tree, for a lane that does not build Rust, for a failed run and for a compiler other than the pinned one. A malformed receipt is skipped by the builder, and the checker refuses an entry whose field has the wrong type or size. CONTRIBUTING.md and the scope function of scripts/check-gate-evidence.py each list the Rust scope, so an edit to one left the other stale without any check failing. Require the text to name every directory and file the function lists, with the tests/ exclusions stated, and every path it names to be classified the way it says. web/ stays out as a folder, but the text must say that a few web/ files a Rust test reads are in. The note's size, lanes, tree id shape and receipt count are written in the builder, the checker and the fetcher, so a test requires the three to agree. Change-Id: I1c2f82fc951116ff17ab35b419a866e1d0cde485 --- scripts/test.sh | 1 + tests/test_check_gate_evidence.py | 59 +++++++ tests/test_gate_roundtrip.py | 276 ++++++++++++++++++++++++++++++ 3 files changed, 336 insertions(+) create mode 100644 tests/test_gate_roundtrip.py diff --git a/scripts/test.sh b/scripts/test.sh index 715d38a92..7e5632a6c 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -713,6 +713,7 @@ gate test-gate-tests unittest_gate "$ROOT/tests/test_test_gate.py" gate env-check-tests unittest_gate "$ROOT/tests/test_env_check.py" gate gate-receipt-tests unittest_gate "$ROOT/tests/test_gate_receipt.py" gate gate-note-tests unittest_gate "$ROOT/tests/test_gate_note.py" +gate gate-roundtrip-tests unittest_gate "$ROOT/tests/test_gate_roundtrip.py" gate mutants-plan-tests unittest_gate "$ROOT/tests/test_mutants_plan.py" gate mutants-install-tests unittest_gate "$ROOT/tests/test_mutants_install.py" gate latency-tests unittest_gate "$ROOT/tests/test_analyze_latency.py" diff --git a/tests/test_check_gate_evidence.py b/tests/test_check_gate_evidence.py index 8bca8426a..7169716b3 100644 --- a/tests/test_check_gate_evidence.py +++ b/tests/test_check_gate_evidence.py @@ -1,5 +1,6 @@ """Check the Rust-scope helper of scripts/check-gate-evidence.py.""" +import functools import importlib.util import json from pathlib import Path @@ -216,6 +217,64 @@ def test_every_tracked_rust_file_is_in(self): self.assertEqual(missed, [], "Rust files outside the scope list") +@functools.cache +def scope_passage(name): + """What `name` says is the Rust scope: from "Rust sources, tests" to "a Rust test names".""" + text = " ".join((ROOT / name).read_text().split()) + found = re.search(r"Rust sources, tests.*?a Rust test names", text) + if found is None: + raise AssertionError(f"{name} has no passage naming the Rust scope") + return found.group(0) + + +def named_paths(passage): + return re.findall(r"`([^`\s]+)`", passage) + + +class ScopeText(unittest.TestCase): + """CONTRIBUTING.md states the scope the function decides.""" + + NAME = "CONTRIBUTING.md" + + def declared(self): + """Every directory or file the function lists as a whole, as the text spells it.""" + names = set(gate_evidence.RUST_PREFIXES) | set(gate_evidence.RUST_FILES) + names |= {"tests/", *gate_evidence.TESTS_OUT_PREFIXES} + if gate_evidence.WEB_IN_RUST_SCOPE: + names.add("web/") + return names + + def test_the_text_says_which_web_files_are_in(self): + # RUST_READ_FILES holds web/ files a Rust source or test names, so a + # JavaScript-only contributor can owe a receipt; the text must say so. + reads_web = any(p.startswith("web/") for p in gate_evidence.RUST_READ_FILES) + self.assertEqual("web/ file" in scope_passage(self.NAME), reads_web) + + def test_the_text_names_everything_the_function_lists(self): + # The files a Rust source names by literal are the text's "a doc or + # script a Rust test names", and the generic phrase stands for them. + named = set(named_paths(scope_passage(self.NAME))) + self.assertEqual(sorted(self.declared() - named), []) + # The exclusions are part of the statement: dropping "not" or + # "apart from" would put them in scope. + self.assertRegex( + scope_passage(self.NAME), + r"`tests/` \(?(?:not|apart from) `tests/browser/`, " + r"`tests/stubs/` (?:or|and)(?: the)? top-level Python files", + ) + + def test_every_path_the_text_names_is_classified(self): + out = set(gate_evidence.TESTS_OUT_PREFIXES) + for path in named_paths(scope_passage(self.NAME)): + probe = path + "x" if path.endswith("/") else path + with self.subTest(path=path): + self.assertEqual( + gate_evidence.in_rust_scope(probe), + path not in out, + "the text names a path the function does not classify so", + ) + + class RustReferences(unittest.TestCase): maxDiff = None diff --git a/tests/test_gate_roundtrip.py b/tests/test_gate_roundtrip.py new file mode 100644 index 000000000..ce600da15 --- /dev/null +++ b/tests/test_gate_roundtrip.py @@ -0,0 +1,276 @@ +"""A gate note built from real receipts is read back by the evidence checker. + +scripts/test.sh writes the receipt, scripts/gate-note.py turns it into the note +a push carries, and scripts/check-gate-evidence.py judges that note. Each has +its own suite with hand-made inputs; this one runs the three in a row, so a +change to a field on one side that the next side no longer reads fails here. +The receipt is written by the function scripts/test.sh defines, in a scratch +repository, not by a copy of its shape. +""" + +import importlib.util +import json +import re +import shlex +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +try: + import tomllib # noqa: F401 (the checker reads the toolchain pin with it) +except ImportError: + tomllib = None + +SPEC = importlib.util.spec_from_file_location( + "support", Path(__file__).with_name("support.py") +) +support = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(support) + +ROOT = support.ROOT +SOURCE = (ROOT / "scripts/test.sh").read_text() +NOTE_SCRIPT = ROOT / "scripts/gate-note.py" +CHECK_SCRIPT = ROOT / "scripts/check-gate-evidence.py" +WRITER = "".join( + support.function_source(SOURCE, name) + for name in ("worktree_tree", "prune_by_tree", "json_escape", "write_receipt") +) +PINNED = "1.90.0" +RUSTC = f"rustc {PINNED} (abcdef123 2025-01-01)" +ENV_JSON = '{"profile":"local-rust","ok":true,"required":[],"skipped":[]}' +VALID = { + "lane": "rust", + "passed": True, + "skipped": 0, + "time": "2025-01-01T00:00:00Z", + "rustc": RUSTC, + "cargo": "cargo 1.90.0 (abc 2025-01-01)", +} +# Entries that differ from a valid one in a single field both sides must refuse. +BAD_FIELDS = ( + {"lane": "bogus"}, + {"passed": "true"}, + {"skipped": -1}, + {"skipped": True}, + {"time": 7}, + {"rustc": 7}, + {"rustc": "r" * 201}, +) + + +sh = support.sh +COMMIT = support.COMMIT + + +def run_script(script, *args): + return subprocess.run( + [sys.executable, str(script), *args], + capture_output=True, + text=True, + check=False, + ) + + +def load(path): + spec = importlib.util.spec_from_file_location(path.stem.replace("-", "_"), path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class SharedSchema(unittest.TestCase): + """The note's schema is written in the builder, the checker and the fetcher.""" + + def test_the_builder_and_the_checker_agree_on_the_schema(self): + note, check = load(NOTE_SCRIPT), load(CHECK_SCRIPT) + self.assertEqual(note.LANES, check.LANES) + self.assertEqual(note.EVIDENCE_LANES, check.COVERING_LANES) + self.assertEqual(note.MAX_RECEIPTS, check.MAX_RECEIPTS) + self.assertEqual(note.MAX_NOTE_BYTES, check.RECEIPT_LIMIT) + self.assertEqual(note.TREE_ID.pattern, check.TREE_ID.pattern) + + def test_the_fetcher_bounds_a_note_at_the_size_the_builder_allows(self): + found = re.search( + r"^LIMIT=(\d+)$", (ROOT / "scripts/fetch-gate-note.sh").read_text(), re.M + ) + self.assertEqual(int(found.group(1)), load(NOTE_SCRIPT).MAX_NOTE_BYTES) + + +@unittest.skipIf(tomllib is None, "the checker needs Python 3.11 for tomllib") +class RoundTripTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) / "repo" + self.root.mkdir() + self.work = Path(tmp.name) + (self.root / "a.txt").write_text("a\n") + done = sh(f"git init -q -b main . && git add -A && {COMMIT} -m x", self.root) + self.assertEqual(done.returncode, 0, done.stderr) + self.pin = self.work / "rust-toolchain.toml" + self.pin.write_text(f'[toolchain]\nchannel = "{PINNED}"\n') + self.changed = self.work / "changed" + self.changed.write_bytes(b"src/main.rs\0") + + def tree(self): + return sh("git rev-parse 'HEAD^{tree}'", self.root).stdout.strip() + + def receipts(self): + return self.root / ".git/codetrial-review/gate-receipts/by-tree" + + def write_real_receipt(self, lane="rust", verdict="true", rustc=RUSTC): + """Run the writer scripts/test.sh defines, as a finished gate run would.""" + assigns = "; ".join( + f"{key}={shlex.quote(value)}" + for key, value in { + "lane": lane, + "failed": "", + "skipped": "", + "receipt_tree": "T1", + "rustc_version": rustc, + "cargo_version": "cargo 1.90.0 (abc 2025-01-01)", + "env_json": ENV_JSON, + }.items() + ) + script = ( + f"set -u; ROOT={shlex.quote(str(self.root))}; {assigns}\n{WRITER}\n" + "worktree_start=$(worktree_tree) || worktree_start=\n" + f"write_receipt {verdict}\n" + ) + done = sh(script, self.root) + self.assertEqual(done.returncode, 0, done.stderr) + + def plant(self, name, body, tree=None): + folder = self.receipts() / (tree or self.tree()) + folder.mkdir(parents=True, exist_ok=True) + (folder / name).write_text(body if isinstance(body, str) else json.dumps(body)) + + def build(self, tree=None): + return run_script( + NOTE_SCRIPT, + "build", + "--receipts", + str(self.receipts()), + "--tree", + tree or self.tree(), + ) + + def check(self, note, tree=None): + """The checker's verdict on `note` (text, or None for no note) for a head tree.""" + args = [ + "--head-tree", + tree or self.tree(), + "--changed-files-file", + str(self.changed), + "--author-type", + "User", + "--base-toolchain-file", + str(self.pin), + ] + if note is not None: + path = self.work / "note.json" + path.write_text(note) + args += ["--receipt-file", str(path)] + return run_script(CHECK_SCRIPT, *args) + + def accepted(self, note, tree=None): + done = self.check(note, tree) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertIn("gate evidence holds", done.stdout) + + def refused(self, note, tree=None): + done = self.check(note, tree) + self.assertEqual(done.returncode, 1, done.stdout + done.stderr) + + def test_a_note_from_a_real_receipt_is_accepted_for_its_tree(self): + self.write_real_receipt() + built = self.build() + self.assertEqual(built.returncode, 0, built.stderr) + self.accepted(built.stdout) + + def test_the_all_lane_vouches_too(self): + self.write_real_receipt(lane="all") + built = self.build() + self.assertEqual(built.returncode, 0, built.stderr) + self.accepted(built.stdout) + + def test_the_note_is_refused_for_another_tree(self): + self.write_real_receipt() + built = self.build() + (self.root / "b.txt").write_text("b\n") + sh(f"git add -A && {COMMIT} -m y", self.root) + self.assertNotEqual(self.tree(), json.loads(built.stdout)["worktree_tree"]) + self.refused(built.stdout) + # The builder asked about the new tree has nothing to send either. + self.assertEqual(self.build().returncode, 1) + + def test_a_portable_lane_alone_builds_no_note_and_a_missing_note_is_refused(self): + self.write_real_receipt(lane="portable") + self.assertEqual(self.build().returncode, 1) + self.refused(None) + + def test_a_note_of_a_portable_receipt_alone_is_refused_whoever_wrote_it(self): + note = { + "version": 1, + "worktree_tree": self.tree(), + "receipts": [{**VALID, "lane": "portable"}], + } + self.refused(json.dumps(note)) + + def test_a_failed_run_builds_no_note_and_passed_false_is_refused(self): + self.write_real_receipt(verdict="false") + self.assertEqual(self.build().returncode, 1) + note = { + "version": 1, + "worktree_tree": self.tree(), + "receipts": [{**VALID, "passed": False}], + } + self.refused(json.dumps(note)) + + def test_a_receipt_from_another_rustc_is_built_and_then_refused(self): + self.write_real_receipt(rustc="rustc 1.89.0 (abcdef123 2025-01-01)") + built = self.build() + self.assertEqual(built.returncode, 0, built.stderr) + done = self.check(built.stdout) + self.assertEqual(done.returncode, 1) + self.assertIn(PINNED, done.stderr) + + def test_a_malformed_receipt_is_skipped_by_the_builder(self): + self.write_real_receipt() + bodies = [ + "not json", + "[]", + {**VALID, "time": "yesterday"}, + {**VALID, "worktree_tree": "0" * 40}, + ] + # Named for the tree, so only the field under test can disqualify it. + bodies += [ + {**VALID, **change, "worktree_tree": self.tree()} for change in BAD_FIELDS + ] + for index, body in enumerate(bodies): + self.plant(f"bad{index}.json", body) + built = self.build() + self.assertEqual(built.returncode, 0, built.stderr) + note = json.loads(built.stdout) + self.assertEqual(len(note["receipts"]), 1, note) + self.accepted(built.stdout) + + def test_a_folder_of_only_malformed_receipts_builds_nothing(self): + self.plant("bad.json", {**VALID, "passed": "yes", "worktree_tree": self.tree()}) + self.assertEqual(self.build().returncode, 1) + + def test_the_checker_refuses_an_entry_with_a_field_of_the_wrong_type_or_size(self): + for change in (*BAD_FIELDS, {"cargo": None}): + with self.subTest(change=change): + note = { + "version": 1, + "worktree_tree": self.tree(), + "receipts": [{**VALID, **change}], + } + self.refused(json.dumps(note)) + + +if __name__ == "__main__": + unittest.main() From 65af7639fd4600e200274edb5b334aad54f64153 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 17 Nov 2026 02:15:01 +0800 Subject: [PATCH 13/13] Document the CI-equivalent local mutation run A contributor asked to reproduce a failed mutation shard, or a reviewer's request for a mutation run, had only make mutants, which audits a fixed set of files whatever the branch changed, so it can pass while the diff has a surviving mutant and fail on code the branch never touched. Document the diff-scoped cargo-mutants commands CI runs, with the shard option and the note that --in-place mutates the working tree. The commands add --cargo-arg=--locked, which the Makefile target passes and which keeps a mutant from being judged against a lockfile cargo quietly updated, and the text says it is diff-scoped like CI's check rather than identical to it. The cost is explained by its cause: a run costs the number of mutants times the suite's time, and the mutant timeout follows the baseline's test time, which is why it is for reproducing a failure and not a per-change requirement. A test pins that both documented commands carry the flag and select what the shard step selects. Change-Id: I8d6327f833de7298c79a0d4e71179b8514ced610 --- docs/development.md | 39 ++++++++++++++++++++++++++++++++++++ tests/test_check_workflow.py | 36 +++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/docs/development.md b/docs/development.md index 719ca3c45..fa0785426 100644 --- a/docs/development.md +++ b/docs/development.md @@ -153,6 +153,45 @@ installing its tools, so a PR with no Rust changes starts no mutation shards and pays no mutation-tool setup cost. Rust diffs still use the complete diff-selected mutant set and the existing exclusions. +To reproduce a failed mutation shard, or to answer a reviewer who asks for a +mutation run, run the same diff-scoped check CI runs. `make mutants` is not +that: it audits a fixed set of files with `-j 6 --timeout 900`, whatever your +branch changed, so it can pass while your diff has a surviving mutant and fail +on code you never touched. The CI check mutates only the functions the diff +touches: + +```sh +SCRATCH=$(mktemp -d) +./scripts/fetch-vendor.sh # the baseline tests serve the vendored web bytes +if [ "$(uname -sm)" = "Linux x86_64" ]; then + sh scripts/fetch-clang.sh target/clang + export CXX=$PWD/target/clang/bin/clang++ +else + : "${CXX:?set CXX to Clang 21 or later}" +fi +git diff origin/main...HEAD -- '*.rs' > "$SCRATCH/pr.diff" +cargo mutants --list --cargo-arg=--locked --in-diff "$SCRATCH/pr.diff" # optional preview +cargo mutants --in-place --in-diff "$SCRATCH/pr.diff" --cargo-arg=--locked +``` + +Add `--shard N/K` (`N` counts from 0) to both cargo-mutants commands to run one +shard of a plan that CI split into `K`; the shard step's log names its `N/K`. +Both commands carry `--cargo-arg=--locked`, as CI's plan and shard steps do: it +keeps a mutant from being judged against a lockfile cargo quietly updated, which +is the reason the Makefile gives. `--in-place` mutates your working tree +and restores it, so run it in a clean checkout or a worktree you can stop +touching for the duration. For a pull request with another base, replace +`origin/main`. `make mutants` sets up Clang the same way; off x86-64 Linux, set +`CXX` to Clang 21 or later instead. + +The cost is the reason this is not a per-change requirement. The unmutated +baseline builds and runs the whole `cargo test` once, and every mutant builds +and runs it again, so a run costs the number of mutants times the suite's time. +It is for reproducing a failure or answering a reviewer's request. cargo-mutants +derives each mutant's test timeout from the baseline's test time (a multiple you +can change with `--timeout-multiplier`) and reports `TIMEOUT` past it, which is +neither caught nor missed. + Every skip the gate sees joins its `not checked by this run:` summary: a gate's own skip, a Node browser test's skip or todo, a missing formatter in `scripts/indent.sh --skip-missing`, and a skipped Python case. A Rust test that diff --git a/tests/test_check_workflow.py b/tests/test_check_workflow.py index dcecfabff..2adffbf3f 100644 --- a/tests/test_check_workflow.py +++ b/tests/test_check_workflow.py @@ -109,6 +109,42 @@ def selection(words): self.assertEqual(selection(listings[1]), selection(run)) +class MutantsLocalRunDocsTests(unittest.TestCase): + """docs/development.md tells a contributor to run what CI's shard step runs.""" + + def setUp(self): + self.text = (support.ROOT / "docs/development.md").read_text() + self.block = support.section( + self.text, "```sh\nSCRATCH=", "```\n", "the local mutation commands" + ) + + def test_both_documented_commands_hold_the_lockfile_like_ci(self): + documented = mutants_invocations(self.block) + self.assertEqual(2, len(documented)) + for words in documented: + self.assertIn("--cargo-arg=--locked", words, words) + + def test_the_run_selects_what_the_shard_step_selects(self): + # Only the diff's location and the shard differ between a local run and + # a shard of CI's plan. + def selection(words): + kept, skip = [], False + for word in words: + if skip: + skip = False + elif word in ("--in-diff", "--shard"): + skip = True + elif word != "--list": + kept.append(word) + return sorted(kept) + + run = next(w for w in mutants_invocations(self.block) if "--list" not in w) + shard = next( + w for w in mutants_invocations(support.job("mutants")) if "--list" not in w + ) + self.assertEqual(selection(shard), selection(run)) + + class AggregateStatusTests(unittest.TestCase): def test_check_accepts_mutants_skipped_after_a_plan(self): body = support.job("check")