From 5a63119b48d7d98fb0ae315cd1a1471aaeb97034 Mon Sep 17 00:00:00 2001 From: Akshar Patel <123344143+AksharP5@users.noreply.github.com> Date: Thu, 22 Oct 2026 10:26:30 -0400 Subject: [PATCH 1/2] fix(verify): reject stale performance reports on reruns --- .github/workflows/perf-budget.yml | 2 + scripts/verify/perf-ab.py | 4 +- scripts/verify/test_perf_ab.py | 91 +++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 scripts/verify/test_perf_ab.py diff --git a/.github/workflows/perf-budget.yml b/.github/workflows/perf-budget.yml index 999f0b46333..f1629edb3b0 100644 --- a/.github/workflows/perf-budget.yml +++ b/.github/workflows/perf-budget.yml @@ -29,6 +29,8 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false + - name: Test performance report isolation + run: python3 -B scripts/verify/test_perf_ab.py - name: Scope id: scope env: diff --git a/scripts/verify/perf-ab.py b/scripts/verify/perf-ab.py index 2d48a702ca0..0e4c8de5bb2 100644 --- a/scripts/verify/perf-ab.py +++ b/scripts/verify/perf-ab.py @@ -50,7 +50,7 @@ def run_once(side, target, case, tier, fixtures, report): CAP_VERIFY_CASE=case, CAP_VERIFY_TIER=tier, ) - report.mkdir(parents=True, exist_ok=True) + report.mkdir(parents=True) started = time.monotonic() with (report / "test.log").open("w") as log: process = subprocess.Popen( @@ -87,7 +87,7 @@ def main(): parser.add_argument("--base", required=True) parser.add_argument("--head", required=True) parser.add_argument("--fixtures", required=True) - parser.add_argument("--out", required=True) + parser.add_argument("--out", required=True, help="use a fresh path for each comparison; existing reports are preserved") parser.add_argument("--pairs", type=int, default=3) parser.add_argument("--summary") args = parser.parse_args() diff --git a/scripts/verify/test_perf_ab.py b/scripts/verify/test_perf_ab.py new file mode 100644 index 00000000000..8b7f9125611 --- /dev/null +++ b/scripts/verify/test_perf_ab.py @@ -0,0 +1,91 @@ +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +SCRIPT = Path(__file__).with_name("perf-ab.py").resolve() +SPEC = importlib.util.spec_from_file_location("perf_ab", SCRIPT) +PERF = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(PERF) + + +@unittest.skipUnless(hasattr(os, "wait4"), "The performance runner requires POSIX wait4") +class PerformanceReportTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name).resolve() + (self.root / "crates" / "verify").mkdir(parents=True) + + def side(self, records, exit_code=0): + binary = self.root / "fixture.py" + binary.with_suffix(".json").write_text(json.dumps({"records": records, "exit": exit_code})) + binary.write_text(f"#!{sys.executable}\n" + r"""import json +import os +from pathlib import Path +import sys + +Path(__file__).with_suffix(".started").touch() +fixture = json.loads(Path(__file__).with_suffix(".json").read_text()) +if fixture["records"]: + report = Path(os.environ["CAP_VERIFY_REPORT_DIR"]) + path = report / f"cap-verify-{os.getpid()}.jsonl" + path.write_text("".join(json.dumps(row) + "\n" for row in fixture["records"])) +print("fixture completed") +sys.exit(fixture["exit"]) +""") + binary.chmod(0o700) + return { + "worktree": str(self.root), + "binaries": {target: str(binary) for _, target, _, _, _ in PERF.WORKLOADS}, + } + + def test_existing_reports_are_preserved_and_never_used(self): + side = self.side([]) + report = self.root / "reports" + report.mkdir() + stale = report / "cap-verify-123.jsonl" + previous = json.dumps({"case": "export/basic/output/mp4/native", "outcome": "pass"}) + "\n" + stale.write_text(previous) + log = report / "test.log" + log.write_text("previous run\n") + with self.assertRaises(FileExistsError): + PERF.run_once(side, "export", "export/basic/*", "fast", self.root / "fixtures", report) + self.assertEqual(stale.read_text(), previous) + self.assertEqual(log.read_text(), "previous run\n") + self.assertFalse((self.root / "fixture.started").exists()) + + def test_fresh_reports_preserve_exit_status_and_outcomes(self): + outcomes = ["pass", "xfail", "xpass", "fail", "unsupported", "inconclusive"] + rows = [{"case": f"export/basic/{outcome}", "outcome": outcome} for outcome in outcomes] + result = PERF.run_once( + self.side(rows, 101), "export", "export/basic/*", "fast", + self.root / "fixtures", self.root / "reports", + ) + self.assertEqual(result["exit"], 101) + self.assertEqual(result["outcomes"], { + f"export/basic/{outcome}": "ran" if outcome in {"pass", "xfail", "xpass"} else outcome + for outcome in outcomes + }) + + def test_cli_rejects_fresh_runs_that_report_no_cases(self): + manifest = self.root / "binaries.json" + manifest.write_text(json.dumps(self.side([]))) + output = self.root / "output" + output.mkdir() + result = subprocess.run([ + sys.executable, "-B", str(SCRIPT), "--base", str(manifest), "--head", str(manifest), + "--fixtures", str(self.root / "fixtures"), "--out", str(output), "--pairs", "1", + ], capture_output=True, text=True, timeout=30) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("not measured:", result.stdout) + self.assertEqual((output / "perf-budget.md").read_text().count("BROKEN:"), len(PERF.WORKLOADS)) + + +if __name__ == "__main__": + unittest.main() From f6f86311b32d4c98cd0696eb7ef0c5acc2429e27 Mon Sep 17 00:00:00 2001 From: Akshar Patel <123344143+AksharP5@users.noreply.github.com> Date: Thu, 22 Oct 2026 10:46:08 -0400 Subject: [PATCH 2/2] test(verify): use kebab-case for the performance regression test --- .github/workflows/perf-budget.yml | 2 +- scripts/verify/{test_perf_ab.py => test-perf-ab.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename scripts/verify/{test_perf_ab.py => test-perf-ab.py} (100%) diff --git a/.github/workflows/perf-budget.yml b/.github/workflows/perf-budget.yml index f1629edb3b0..401f5a036ac 100644 --- a/.github/workflows/perf-budget.yml +++ b/.github/workflows/perf-budget.yml @@ -30,7 +30,7 @@ jobs: with: persist-credentials: false - name: Test performance report isolation - run: python3 -B scripts/verify/test_perf_ab.py + run: python3 -B scripts/verify/test-perf-ab.py - name: Scope id: scope env: diff --git a/scripts/verify/test_perf_ab.py b/scripts/verify/test-perf-ab.py similarity index 100% rename from scripts/verify/test_perf_ab.py rename to scripts/verify/test-perf-ab.py