diff --git a/evals/capability/cases/tier-hard/verify.sh b/evals/capability/cases/tier-hard/verify.sh index 381629762..f0eab1a95 100755 --- a/evals/capability/cases/tier-hard/verify.sh +++ b/evals/capability/cases/tier-hard/verify.sh @@ -13,6 +13,14 @@ TEST_SHA="a7ce0705273ad6b874b737d389db038745e5baec825fd757f42448cd1addb687" actual_sha=$(shasum -a 256 tests/report.test.ts | cut -d' ' -f1) if [[ "$actual_sha" != "$TEST_SHA" ]]; then + # A stale TEST_SHA is indistinguishable from an agent edit above, so check + # the fixture's own hash at setup before blaming the agent: the workdir is + # a git repo committed before the run, so HEAD holds the pristine file. + setup_sha=$(git show "HEAD:tests/report.test.ts" 2>/dev/null | shasum -a 256 | cut -d' ' -f1 || true) + if [[ -n "$setup_sha" && "$actual_sha" == "$setup_sha" ]]; then + echo "FAIL: tests/report.test.ts is unchanged from the fixture at setup ($setup_sha) but does not match TEST_SHA ($TEST_SHA): the case's locked hash is stale (broken case), not an agent edit" + exit 1 + fi echo "FAIL: tests/report.test.ts was modified (contract file must be byte-unchanged)" exit 1 fi diff --git a/evals/capability/cases/tier-med/verify.sh b/evals/capability/cases/tier-med/verify.sh index 825091f06..33c13781a 100755 --- a/evals/capability/cases/tier-med/verify.sh +++ b/evals/capability/cases/tier-med/verify.sh @@ -13,6 +13,14 @@ TEST_SHA="4b48fc362c6c3a72af405545ffa816f4a531bbed974d87b6f4d60cf5c96f06a0" actual_sha=$(shasum -a 256 tests/checkout.test.ts | cut -d' ' -f1) if [[ "$actual_sha" != "$TEST_SHA" ]]; then + # A stale TEST_SHA is indistinguishable from an agent edit above, so check + # the fixture's own hash at setup before blaming the agent: the workdir is + # a git repo committed before the run, so HEAD holds the pristine file. + setup_sha=$(git show "HEAD:tests/checkout.test.ts" 2>/dev/null | shasum -a 256 | cut -d' ' -f1 || true) + if [[ -n "$setup_sha" && "$actual_sha" == "$setup_sha" ]]; then + echo "FAIL: tests/checkout.test.ts is unchanged from the fixture at setup ($setup_sha) but does not match TEST_SHA ($TEST_SHA): the case's locked hash is stale (broken case), not an agent edit" + exit 1 + fi echo "FAIL: tests/checkout.test.ts was modified (contract file must be byte-unchanged)" exit 1 fi diff --git a/evals/capability/cases/tier-xhard/verify.sh b/evals/capability/cases/tier-xhard/verify.sh index 01aee06b6..a791926ad 100755 --- a/evals/capability/cases/tier-xhard/verify.sh +++ b/evals/capability/cases/tier-xhard/verify.sh @@ -5,13 +5,21 @@ # at green fails every check below. set -euo pipefail -TEST_SHA="e125ecc69453b923bb69b46e8411ca5b580e0afb81868a8e24f45a10d93e1d2a" +TEST_SHA="bd97182202b283a721f2972f1fba79e2b552a95a64f1639d077022aa95a157d5" [[ -f package.json ]] || { echo "FAIL: package.json missing in workdir"; exit 1; } [[ -f tests/notify.test.ts ]] || { echo "FAIL: tests/notify.test.ts is gone"; exit 1; } actual_sha=$(shasum -a 256 tests/notify.test.ts | cut -d' ' -f1) if [[ "$actual_sha" != "$TEST_SHA" ]]; then + # A stale TEST_SHA is indistinguishable from an agent edit above, so check + # the fixture's own hash at setup before blaming the agent: the workdir is + # a git repo committed before the run, so HEAD holds the pristine file. + setup_sha=$(git show "HEAD:tests/notify.test.ts" 2>/dev/null | shasum -a 256 | cut -d' ' -f1 || true) + if [[ -n "$setup_sha" && "$actual_sha" == "$setup_sha" ]]; then + echo "FAIL: tests/notify.test.ts is unchanged from the fixture at setup ($setup_sha) but does not match TEST_SHA ($TEST_SHA): the case's locked hash is stale (broken case), not an agent edit" + exit 1 + fi echo "FAIL: tests/notify.test.ts was modified (contract file must be byte-unchanged)" exit 1 fi diff --git a/evals/capability/locked-fixtures.test.ts b/evals/capability/locked-fixtures.test.ts new file mode 100644 index 000000000..ae60e1629 --- /dev/null +++ b/evals/capability/locked-fixtures.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { loadEvalCases, resolveFixturePath } from "./lib.js"; + +const capabilityDir = dirname(fileURLToPath(import.meta.url)); + +function parseLockedContract( + verifySrc: string, + caseId: string, +): { sha: string; path: string } | null { + const sha = verifySrc.match(/^TEST_SHA="([0-9a-f]{64})"$/m)?.[1]; + if (sha === undefined) return null; + const hashed = verifySrc.match(/shasum -a 256 (\S+)/)?.[1]; + const guarded = [...verifySrc.matchAll(/\[\[ -f (\S+) \]\]/g)].map( + (m) => m[1], + ); + if (hashed === undefined || !guarded.includes(hashed)) { + throw new Error( + `case ${caseId}: cannot determine the locked contract file from verify.sh`, + ); + } + return { sha, path: hashed }; +} + +describe("locked fixture hashes", () => { + test("every TEST_SHA literal matches its fixture file", async () => { + const repoRoot = resolve(capabilityDir, "..", ".."); + const cases = await loadEvalCases(join(capabilityDir, "cases")); + const failures: string[] = []; + let checked = 0; + for (const c of cases) { + const verifySrc = await readFile(join(c.caseDir, c.verify), "utf8"); + const locked = parseLockedContract(verifySrc, c.id); + if (locked === null) continue; + checked += 1; + const bytes = await readFile( + join(resolveFixturePath(repoRoot, c.fixture), locked.path), + ); + const actual = createHash("sha256").update(bytes).digest("hex"); + if (actual !== locked.sha) { + failures.push( + `case ${c.id}: ${locked.path} hashes to ${actual} but verify.sh pins ${locked.sha}`, + ); + } + } + expect(checked).toBeGreaterThan(0); + if (failures.length > 0) { + throw new Error(`stale locked hashes:\n${failures.join("\n")}`); + } + }); +});