diff --git a/evals/completion/lib.test.ts b/evals/completion/lib.test.ts index da36e948d..e43bfe963 100644 --- a/evals/completion/lib.test.ts +++ b/evals/completion/lib.test.ts @@ -77,6 +77,14 @@ describe("completion totals", () => { expect(totals.meanTurnsToCompletion).toBe(0); expect(totals.meanAgentDurationMs).toBe(0); }); + + test("aggregates gate suspensions across runs", () => { + const totals = computeTotals([ + result({ gateSuspensions: 2 }), + result({ taskId: "stall-read", gateSuspensions: 3 }), + ]); + expect(totals.totalGateSuspensions).toBe(5); + }); }); describe("boundary parsing", () => { @@ -136,4 +144,26 @@ describe("human summary", () => { expect(summary).toContain("version-endpoint r0: complete"); expect(summary).toContain("stall-read r0: incomplete"); }); + + test("prints the aggregated gate suspensions", () => { + const results = [ + result({ gateSuspensions: 2 }), + result({ taskId: "stall-read", gateSuspensions: 3 }), + ]; + const report: CompletionReportType = CompletionReport.assert({ + harness: "completion-baseline", + version: 1, + startedAt: "2026-09-14T00:00:00.000Z", + finishedAt: "2026-09-14T00:01:00.000Z", + commitSha: "deadbeef", + provider: "stub-scripted", + model: "completion-baseline-v1", + repeats: 1, + taskSetVersion: 1, + taskIds: ["version-endpoint", "stall-read"], + results, + totals: computeTotals(results), + }); + expect(formatSummary(report)).toContain("gate suspensions 5"); + }); }); diff --git a/evals/completion/lib.ts b/evals/completion/lib.ts index c9c04351e..d263c4b9a 100644 --- a/evals/completion/lib.ts +++ b/evals/completion/lib.ts @@ -85,6 +85,7 @@ export const CompletionTotals = type({ totalCompactionEvents: "number.integer >= 0", totalDoomLoopInterventions: "number.integer >= 0", totalThrashInterventions: "number.integer >= 0", + totalGateSuspensions: "number.integer >= 0", }); export type CompletionTotals = typeof CompletionTotals.infer; @@ -158,6 +159,7 @@ export function computeTotals( totalThrashInterventions: sum( results.map((result) => result.thrashInterventions), ), + totalGateSuspensions: sum(results.map((result) => result.gateSuspensions)), }; } @@ -172,7 +174,7 @@ export function formatSummary(report: CompletionReport): string { `task set v${report.taskSetVersion}: ${report.taskIds.join(", ")}`, `completion rate ${formatRate(report.totals.completionRate)} (${report.totals.completedRuns}/${report.totals.runsTotal} runs)`, `mean turns to completion ${report.totals.meanTurnsToCompletion.toFixed(1)} mean agent time ${Math.round(report.totals.meanAgentDurationMs)}ms`, - `retries ${report.totals.totalRetries} compaction events ${report.totals.totalCompactionEvents} doom-loop interventions ${report.totals.totalDoomLoopInterventions} thrash interventions ${report.totals.totalThrashInterventions}`, + `retries ${report.totals.totalRetries} compaction events ${report.totals.totalCompactionEvents} doom-loop interventions ${report.totals.totalDoomLoopInterventions} thrash interventions ${report.totals.totalThrashInterventions} gate suspensions ${report.totals.totalGateSuspensions}`, "", ...report.results.map( (result) => diff --git a/scripts/eval-completion.test.ts b/scripts/eval-completion.test.ts index 9832550a7..b0a231b8d 100644 --- a/scripts/eval-completion.test.ts +++ b/scripts/eval-completion.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { withTimeout } from "./eval-completion.js"; +import { resolveRunStatus, withTimeout } from "./eval-completion.js"; describe("withTimeout", () => { test("rejects a hung run after the timeout", async () => { @@ -28,4 +28,47 @@ describe("withTimeout", () => { withTimeout(Promise.reject(new Error("boom")), 1000, "task"), ).rejects.toThrow("boom"); }); + + test("invokes onTimeout when the deadline fires", async () => { + const hung = new Promise(() => undefined); + let calls = 0; + const onTimeout = () => { + calls += 1; + }; + const error = await withTimeout(hung, 50, "task", { onTimeout }).catch( + (err: unknown) => err, + ); + expect(error).toBeInstanceOf(Error); + expect(calls).toBe(1); + }); + + test("skips onTimeout when the inner promise settles first", async () => { + let calls = 0; + const onTimeout = () => { + calls += 1; + }; + await expect( + withTimeout(Promise.resolve("done"), 50, "task", { onTimeout }), + ).resolves.toBe("done"); + const failing = Promise.reject(new Error("boom")); + const rejected = withTimeout(failing, 50, "task", { onTimeout }); + await expect(rejected).rejects.toThrow("boom"); + expect(calls).toBe(0); + }); +}); + +describe("resolveRunStatus", () => { + test("timeout keeps status over a failure signal", () => { + const status = resolveRunStatus({ timedOut: true, failed: true }); + expect(status).toBe("timeout"); + }); + + test("resolves the remaining outcomes", () => { + const timeoutOnly = resolveRunStatus({ timedOut: true, failed: false }); + const failedOnly = resolveRunStatus({ timedOut: false, failed: true }); + const clean = resolveRunStatus({ timedOut: false, failed: false }); + expect(timeoutOnly).toBe("timeout"); + expect(failedOnly).toBe("failed"); + expect(clean).toBe("completed"); + }); }); diff --git a/scripts/eval-completion.ts b/scripts/eval-completion.ts index abfdce6fb..f8a9ae2a4 100644 --- a/scripts/eval-completion.ts +++ b/scripts/eval-completion.ts @@ -117,6 +117,7 @@ export async function withTimeout( promise: Promise, ms: number, label: string, + opts?: { onTimeout?: () => void }, ): Promise { let timer: ReturnType | undefined; try { @@ -126,10 +127,12 @@ export async function withTimeout( return await Promise.race([ promise, new Promise((_, reject) => { - timer = setTimeout( - () => reject(new Error(`${label} timed out after ${ms}ms`)), - ms, - ); + timer = setTimeout(() => { + // Cancel first so the caller's in-flight work settles instead of + // lingering past the deadline; the race rejects below regardless. + opts?.onTimeout?.(); + reject(new Error(`${label} timed out after ${ms}ms`)); + }, ms); }), ]); } finally { @@ -137,6 +140,26 @@ export async function withTimeout( } } +/** + * Timeout keeps status over failure signals: a run that timed out did not + * fail, it ran out of time — even when the partial stream already carries + * a failure event. + */ +export function resolveRunStatus(options: { + timedOut: boolean; + failed: boolean; +}): RunStatus { + if (options.timedOut) return "timeout"; + if (options.failed) return "failed"; + return "completed"; +} + +// Backstop for the post-timeout quiesce below: the abort plus agent close +// settle the live paths promptly, so this only bites when the mock pump +// itself is stuck — and then it keeps a stuck pump from re-hanging the +// harness at the deadline it just enforced. +const SETTLE_GRACE_MS = 5_000; + interface PersistedTurn { role: string; } @@ -241,31 +264,46 @@ async function runTask( if (turnComplete && event.type === "message.run.ended") return; } })().catch(() => undefined); + // Abort the in-flight send when the deadline fires so its promise + // settles instead of lingering past the timeout. + const controller = new AbortController(); + const runWork = (async () => { + const sendResult = await Promise.all([ + session.agent + .send(task.prompt, { signal: controller.signal }) + .then((result) => { + turnComplete = true; + return result; + }), + session.harness.run({ wallClockBudgetMs: Infinity }), + collect, + ]).then(([result]) => result); + if (sendResult.type !== "reply") { + throw new Error(`unexpected send outcome: ${sendResult.type}`); + } + })(); + let timedOut = false; try { - await withTimeout( - (async () => { - const sendResult = await Promise.all([ - session.agent.send(task.prompt).then((result) => { - turnComplete = true; - return result; - }), - session.harness.run({ wallClockBudgetMs: Infinity }), - collect, - ]).then(([result]) => result); - if (sendResult.type !== "reply") { - throw new Error(`unexpected send outcome: ${sendResult.type}`); - } - })(), - timeoutMs, - `task ${task.id}`, - ); + await withTimeout(runWork, timeoutMs, `task ${task.id}`, { + onTimeout: () => controller.abort(), + }); } catch (err) { - runStatus = - err instanceof Error && err.message.includes("timed out") - ? "timeout" - : "failed"; + timedOut = err instanceof Error && err.message.includes("timed out"); error = err instanceof Error ? err.message : String(err); } + if (timedOut) { + // Quiesce before grading: the aborted send settles at once, but the + // pump and collector lag behind. Closing the agent aborts the reactor + // and terminates the stream so the collector settles, then awaiting + // the inner work keeps verify below off partial state. + await session.agent.close().catch(() => undefined); + await Promise.race([ + runWork.catch(() => undefined), + new Promise((resolve) => { + setTimeout(resolve, SETTLE_GRACE_MS); + }), + ]); + } const agentDurationMs = Date.now() - agentStart; const signals = deriveSignals(events); // A guard trip rejects send() without a run.ended event; recover the @@ -277,7 +315,10 @@ async function runTask( ) { signals.doomLoopInterventions = 1; } - if (signals.runFailed) runStatus = "failed"; + runStatus = resolveRunStatus({ + timedOut, + failed: error !== undefined || signals.runFailed, + }); const verifyStart = Date.now(); const verify = spawnSync("bash", [resolve(COMPLETION_ROOT, task.verify)], {