diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 2cf880325..c5dcd0464 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -629,7 +629,8 @@ jobs: set -o pipefail count=0 if [ -s "$RUNNER_TEMP/pr.diff" ]; then - cargo mutants --list --in-diff "$RUNNER_TEMP/pr.diff" >"$RUNNER_TEMP/list.txt" + cargo mutants --list --cargo-arg=--locked \ + --in-diff "$RUNNER_TEMP/pr.diff" >"$RUNNER_TEMP/list.txt" count=$(wc -l <"$RUNNER_TEMP/list.txt" | tr -d ' ') fi width=$(( (count + 14) / 15 )) @@ -673,10 +674,21 @@ jobs: # asks is what a change failed to test, and that is worth answering while the # change can still be edited. mutants: - needs: plan-mutants - # No mutants means no runners. A docs-only or JS-only pull request should + needs: [plan-mutants, gates] + # A shard rebuilds and runs the whole suite in an unmutated tree, so it + # fails the same way the Rust lane does, and pays several minutes of setup + # per shard to learn it. Waiting for the gates avoids that; the trade-off + # is that shards start after the slowest gates lane rather than beside it. + # + # `plan-mutants.result` is explicit because the plan is skipped on a push + # or a merge group, where its count is empty and `'' != '0'` would pass. + # No mutants means no runners: a docs-only or JS-only pull request should # not start a job to discover it has nothing to do. - if: needs.plan-mutants.outputs.count != '0' + if: >- + ${{ !cancelled() + && needs.plan-mutants.result == 'success' + && needs.gates.result == 'success' + && needs.plan-mutants.outputs.count != '0' }} runs-on: ubuntu-24.04 timeout-minutes: 45 strategy: @@ -739,7 +751,7 @@ jobs: run: | git diff "origin/$BASE_REF...HEAD" -- '*.rs' >"$RUNNER_TEMP/pr.diff" echo "::group::Mutants in shard $SHARD" - cargo mutants --list --in-diff "$RUNNER_TEMP/pr.diff" \ + cargo mutants --list --cargo-arg=--locked --in-diff "$RUNNER_TEMP/pr.diff" \ --shard "$SHARD" \ | tee "$RUNNER_TEMP/shard.txt" echo "::endgroup::" @@ -758,7 +770,7 @@ jobs: # overrun would be swallowed by the construct meant to report it. status=0 timeout --signal=INT --kill-after=60s 35m \ - cargo mutants --in-place --in-diff "$RUNNER_TEMP/pr.diff" \ + cargo mutants --in-place --cargo-arg=--locked --in-diff "$RUNNER_TEMP/pr.diff" \ --shard "$SHARD" \ || status=$? if [ "$status" -eq 124 ]; then diff --git a/docs/development.md b/docs/development.md index 39ef67865..c58e219de 100644 --- a/docs/development.md +++ b/docs/development.md @@ -3,7 +3,7 @@ The gate, the formatters, the hooks, the generated files, and where a slow run's time actually goes. `make` targets and the one-line summary of each are in [README.md](../README.md#development); everything here is the detail behind -them. +them. This sentence is a scratch edit for CI verification. Selection admission and refresh behavior are specified in [interview selection](interview-selection.md). The Strict browser regression tests use diff --git a/scripts/test.sh b/scripts/test.sh index d663e04ab..62660fb12 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -747,6 +747,7 @@ gate git-hooks "$ROOT/scripts/test-git-hooks.sh" gate commit-reflow-tests unittest_gate "$ROOT/tests/test_commit_reflow.py" gate commit-draft-tests unittest_gate "$ROOT/tests/test_commit_draft.py" gate pr-text-tests unittest_gate "$ROOT/tests/test_check_pr_text.py" +gate check-workflow-tests unittest_gate "$ROOT/tests/test_check_workflow.py" gate shell-syntax shell_syntax gate browser-check-env env BROWSER_CHECK_VALIDATE_ENV_ONLY=1 sh "$ROOT/scripts/browser-check.sh" diff --git a/tests/test_check_workflow.py b/tests/test_check_workflow.py new file mode 100644 index 000000000..fadc4843f --- /dev/null +++ b/tests/test_check_workflow.py @@ -0,0 +1,116 @@ +"""The shape of the mutation job and the aggregate status in check.yml.""" + +import importlib.util +from pathlib import Path +import re +import shlex +import unittest + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location( + "check_workflow_support", ROOT / "tests/support.py" +) +support = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(support) + + +def header(body, key): + """The value of a job-level key, with a folded scalar joined to one line.""" + found = re.search(rf"^ {key}: (.*(?:\n .*)*)", body, re.M) + if found is None: + raise AssertionError(f"job no longer sets {key}") + return " ".join(found[1].replace(">-", "").split()) + + +class MutantsJobTests(unittest.TestCase): + def setUp(self): + self.body = support.job("mutants") + + def test_waits_for_the_gates_and_the_plan(self): + needs = set(re.findall(r"[a-z][a-z-]*", header(self.body, "needs"))) + self.assertLessEqual({"gates", "plan-mutants"}, needs) + + def test_starts_only_after_both_succeed_with_work_to_do(self): + condition = header(self.body, "if").strip("${} ") + # Exactly these terms, all required. `!cancelled()` replaces the + # implicit success() that would otherwise skip the job. plan-mutants + # is skipped on push and merge_group, where its count output is empty + # and would pass the != '0' guard on its own. + self.assertEqual( + [ + "!cancelled()", + "needs.plan-mutants.result == 'success'", + "needs.gates.result == 'success'", + "needs.plan-mutants.outputs.count != '0'", + ], + [term.strip() for term in condition.split("&&")], + ) + + +def mutants_invocations(body): + """The argument words of every `cargo mutants` command. + + Continuation lines are joined, comments dropped, and the words end at the + first redirection, pipe or `||`, so a flag only counts when the shell would + pass it. + """ + found = [] + for command in re.findall( + r"^[^#\n]*?cargo mutants\b((?:[^\n]*\\\n)*[^\n]*)", body, re.M + ): + words = [] + for word in shlex.split(command.replace("\\\n", " "), comments=True): + if word[0] in "|>": + break + words.append(word) + found.append(words) + return found + + +class MutantsLockfileTests(unittest.TestCase): + def setUp(self): + self.plan = mutants_invocations(support.job("plan-mutants")) + self.shard = mutants_invocations(support.job("mutants")) + + def test_every_selecting_or_running_command_holds_the_lockfile(self): + # cargo-mutants forwards the flag to every cargo it runs; without it + # a mutant can be judged against a lockfile cargo quietly updated. + commands = self.plan + self.shard + self.assertEqual(3, len(commands)) + for words in commands: + self.assertIn("--cargo-arg=--locked", words, words) + + def test_plan_and_shard_select_the_same_mutants(self): + # The shard count comes from the plan's list, so what is listed has to + # match on both sides: the same flags with the same values. + def selection(words): + kept, skip = [], False + for word in words: + if skip: + skip = False + elif word == "--shard": + skip = True + elif word not in ("--list", "--in-place"): + kept.append(word) + return sorted(kept) + + listings = [w for w in self.plan + self.shard if "--list" in w] + self.assertEqual(2, len(listings)) + self.assertEqual(selection(listings[0]), selection(listings[1])) + run = next(w for w in self.shard if "--list" not in w) + self.assertEqual(selection(listings[1]), selection(run)) + + +class AggregateStatusTests(unittest.TestCase): + def test_check_accepts_mutants_skipped_after_a_plan(self): + body = support.job("check") + self.assertRegex(body, r"success/success \| skipped/success[^)]*\) ;;") + # A failed gates lane fails the job before the mutants table is read. + self.assertLess(body.index('"$GATES" != success'), body.index("case ")) + + def test_release_does_not_wait_for_mutants(self): + self.assertNotIn("mutants", header(support.job("release"), "needs")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mutants_plan.py b/tests/test_mutants_plan.py index ff9b26e47..019fb2438 100644 --- a/tests/test_mutants_plan.py +++ b/tests/test_mutants_plan.py @@ -42,7 +42,7 @@ def setUp(self): set -eu echo "$*" >>"$RUNNER_TEMP/cargo.log" [ "$FAKE_FAIL" = 0 ] || exit 7 -[ "$*" = "mutants --list --in-diff $RUNNER_TEMP/pr.diff" ] || exit 9 +[ "$*" = "mutants --list --cargo-arg=--locked --in-diff $RUNNER_TEMP/pr.diff" ] || exit 9 i=0 while [ "$i" -lt "$FAKE_COUNT" ]; do echo "mutant $i"