fix(evals): keep timeout status, cancel in-flight work, aggregate gate suspensions

evals/completion/lib.test.ts

@@ -77,6 +77,14 @@ describe("completion totals", () => {
     expect(totals.meanTurnsToCompletion).toBe(0);
     expect(totals.meanAgentDurationMs).toBe(0);
   });
+
+  test("aggregates gate suspensions across runs", () => {
+    const totals = computeTotals([
+      result({ gateSuspensions: 2 }),
+      result({ taskId: "stall-read", gateSuspensions: 3 }),
+    ]);
+    expect(totals.totalGateSuspensions).toBe(5);
+  });
 });
 
 describe("boundary parsing", () => {
@@ -136,4 +144,26 @@ describe("human summary", () => {
     expect(summary).toContain("version-endpoint r0: complete");
     expect(summary).toContain("stall-read r0: incomplete");
   });
+
+  test("prints the aggregated gate suspensions", () => {
+    const results = [
+      result({ gateSuspensions: 2 }),
+      result({ taskId: "stall-read", gateSuspensions: 3 }),
+    ];
+    const report: CompletionReportType = CompletionReport.assert({
+      harness: "completion-baseline",
+      version: 1,
+      startedAt: "2026-09-14T00:00:00.000Z",
+      finishedAt: "2026-09-14T00:01:00.000Z",
+      commitSha: "deadbeef",
+      provider: "stub-scripted",
+      model: "completion-baseline-v1",
+      repeats: 1,
+      taskSetVersion: 1,
+      taskIds: ["version-endpoint", "stall-read"],
+      results,
+      totals: computeTotals(results),
+    });
+    expect(formatSummary(report)).toContain("gate suspensions 5");
+  });
 });

evals/completion/lib.ts

@@ -85,6 +85,7 @@ export const CompletionTotals = type({
   totalCompactionEvents: "number.integer >= 0",
   totalDoomLoopInterventions: "number.integer >= 0",
   totalThrashInterventions: "number.integer >= 0",
+  totalGateSuspensions: "number.integer >= 0",
 });
 export type CompletionTotals = typeof CompletionTotals.infer;
 
@@ -158,6 +159,7 @@ export function computeTotals(
     totalThrashInterventions: sum(
       results.map((result) => result.thrashInterventions),
     ),
+    totalGateSuspensions: sum(results.map((result) => result.gateSuspensions)),
   };
 }
 
@@ -172,7 +174,7 @@ export function formatSummary(report: CompletionReport): string {
     `task set v${report.taskSetVersion}: ${report.taskIds.join(", ")}`,
     `completion rate ${formatRate(report.totals.completionRate)} (${report.totals.completedRuns}/${report.totals.runsTotal} runs)`,
     `mean turns to completion ${report.totals.meanTurnsToCompletion.toFixed(1)} mean agent time ${Math.round(report.totals.meanAgentDurationMs)}ms`,
-    `retries ${report.totals.totalRetries} compaction events ${report.totals.totalCompactionEvents} doom-loop interventions ${report.totals.totalDoomLoopInterventions} thrash interventions ${report.totals.totalThrashInterventions}`,
+    `retries ${report.totals.totalRetries} compaction events ${report.totals.totalCompactionEvents} doom-loop interventions ${report.totals.totalDoomLoopInterventions} thrash interventions ${report.totals.totalThrashInterventions} gate suspensions ${report.totals.totalGateSuspensions}`,
     "",
     ...report.results.map(
       (result) =>

scripts/eval-completion.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test";
-import { withTimeout } from "./eval-completion.js";
+import { resolveRunStatus, withTimeout } from "./eval-completion.js";
 
 describe("withTimeout", () => {
   test("rejects a hung run after the timeout", async () => {
@@ -28,4 +28,47 @@ describe("withTimeout", () => {
       withTimeout(Promise.reject(new Error("boom")), 1000, "task"),
     ).rejects.toThrow("boom");
   });
+
+  test("invokes onTimeout when the deadline fires", async () => {
+    const hung = new Promise<never>(() => undefined);
+    let calls = 0;
+    const onTimeout = () => {
+      calls += 1;
+    };
+    const error = await withTimeout(hung, 50, "task", { onTimeout }).catch(
+      (err: unknown) => err,
+    );
+    expect(error).toBeInstanceOf(Error);
+    expect(calls).toBe(1);
+  });
+
+  test("skips onTimeout when the inner promise settles first", async () => {
+    let calls = 0;
+    const onTimeout = () => {
+      calls += 1;
+    };
+    await expect(
+      withTimeout(Promise.resolve("done"), 50, "task", { onTimeout }),
+    ).resolves.toBe("done");
+    const failing = Promise.reject(new Error("boom"));
+    const rejected = withTimeout(failing, 50, "task", { onTimeout });
+    await expect(rejected).rejects.toThrow("boom");
+    expect(calls).toBe(0);
+  });
+});
+
+describe("resolveRunStatus", () => {
+  test("timeout keeps status over a failure signal", () => {
+    const status = resolveRunStatus({ timedOut: true, failed: true });
+    expect(status).toBe("timeout");
+  });
+
+  test("resolves the remaining outcomes", () => {
+    const timeoutOnly = resolveRunStatus({ timedOut: true, failed: false });
+    const failedOnly = resolveRunStatus({ timedOut: false, failed: true });
+    const clean = resolveRunStatus({ timedOut: false, failed: false });
+    expect(timeoutOnly).toBe("timeout");
+    expect(failedOnly).toBe("failed");
+    expect(clean).toBe("completed");
+  });
 });

scripts/eval-completion.ts

@@ -117,6 +117,7 @@ export async function withTimeout<T>(
   promise: Promise<T>,
   ms: number,
   label: string,
+  opts?: { onTimeout?: () => void },
 ): Promise<T> {
   let timer: ReturnType<typeof setTimeout> | undefined;
   try {
@@ -126,17 +127,39 @@ export async function withTimeout<T>(
     return await Promise.race([
       promise,
       new Promise<never>((_, reject) => {
-        timer = setTimeout(
-          () => reject(new Error(`${label} timed out after ${ms}ms`)),
-          ms,
-        );
+        timer = setTimeout(() => {
+          // Cancel first so the caller's in-flight work settles instead of
+          // lingering past the deadline; the race rejects below regardless.
+          opts?.onTimeout?.();
+          reject(new Error(`${label} timed out after ${ms}ms`));
+        }, ms);
       }),
     ]);
   } finally {
     if (timer !== undefined) clearTimeout(timer);
   }
 }
 
+/**
+ * Timeout keeps status over failure signals: a run that timed out did not
+ * fail, it ran out of time — even when the partial stream already carries
+ * a failure event.
+ */
+export function resolveRunStatus(options: {
+  timedOut: boolean;
+  failed: boolean;
+}): RunStatus {
+  if (options.timedOut) return "timeout";
+  if (options.failed) return "failed";
+  return "completed";
+}
+
+// Backstop for the post-timeout quiesce below: the abort plus agent close
+// settle the live paths promptly, so this only bites when the mock pump
+// itself is stuck — and then it keeps a stuck pump from re-hanging the
+// harness at the deadline it just enforced.
+const SETTLE_GRACE_MS = 5_000;
+
 interface PersistedTurn {
   role: string;
 }
@@ -241,31 +264,46 @@ async function runTask(
         if (turnComplete && event.type === "message.run.ended") return;
       }
     })().catch(() => undefined);
+    // Abort the in-flight send when the deadline fires so its promise
+    // settles instead of lingering past the timeout.
+    const controller = new AbortController();
+    const runWork = (async () => {
+      const sendResult = await Promise.all([
+        session.agent
+          .send(task.prompt, { signal: controller.signal })
+          .then((result) => {
+            turnComplete = true;
+            return result;
+          }),
+        session.harness.run({ wallClockBudgetMs: Infinity }),
+        collect,
+      ]).then(([result]) => result);
+      if (sendResult.type !== "reply") {
+        throw new Error(`unexpected send outcome: ${sendResult.type}`);
+      }
+    })();
+    let timedOut = false;
     try {
-      await withTimeout(
-        (async () => {
-          const sendResult = await Promise.all([
-            session.agent.send(task.prompt).then((result) => {
-              turnComplete = true;
-              return result;
-            }),
-            session.harness.run({ wallClockBudgetMs: Infinity }),
-            collect,
-          ]).then(([result]) => result);
-          if (sendResult.type !== "reply") {
-            throw new Error(`unexpected send outcome: ${sendResult.type}`);
-          }
-        })(),
-        timeoutMs,
-        `task ${task.id}`,
-      );
+      await withTimeout(runWork, timeoutMs, `task ${task.id}`, {
+        onTimeout: () => controller.abort(),
+      });
     } catch (err) {
-      runStatus =
-        err instanceof Error && err.message.includes("timed out")
-          ? "timeout"
-          : "failed";
+      timedOut = err instanceof Error && err.message.includes("timed out");
       error = err instanceof Error ? err.message : String(err);
     }
+    if (timedOut) {
+      // Quiesce before grading: the aborted send settles at once, but the
+      // pump and collector lag behind. Closing the agent aborts the reactor
+      // and terminates the stream so the collector settles, then awaiting
+      // the inner work keeps verify below off partial state.
+      await session.agent.close().catch(() => undefined);
+      await Promise.race([
+        runWork.catch(() => undefined),
+        new Promise<void>((resolve) => {
+          setTimeout(resolve, SETTLE_GRACE_MS);
+        }),
+      ]);
+    }
     const agentDurationMs = Date.now() - agentStart;
     const signals = deriveSignals(events);
     // A guard trip rejects send() without a run.ended event; recover the
@@ -277,7 +315,10 @@ async function runTask(
     ) {
       signals.doomLoopInterventions = 1;
     }
-    if (signals.runFailed) runStatus = "failed";
+    runStatus = resolveRunStatus({
+      timedOut,
+      failed: error !== undefined || signals.runFailed,
+    });
 
     const verifyStart = Date.now();
     const verify = spawnSync("bash", [resolve(COMPLETION_ROOT, task.verify)], {