Add parallel test runner with stall watchdog

bun test --parallel on Bun 1.4.x intermittently livelocks: a worker
spins at 100% CPU holding a zombie git child while the main process
idles with no output, and bun test has no run-level timeout, so a
stalled run hangs forever (upstream oven-sh/bun bug, still open on
1.4.2, reproduced locally at ~50% of runs with 4 workers). A healthy
run never exceeds 0.64s of output silence, so the wrapper runs the
seeded suite in its own process group, kills the whole group after 90s
of silence, and retries up to 3 times. A child that exits on its own,
pass or fail, is never retried, so real failures still fail the gate.
CI keeps sharded sequential runs and is unaffected.

Author: Gustav-Simonsson

docs/IMPLEMENTATION.md

@@ -521,6 +521,7 @@ Run all three before declaring work complete.
 - **`tests/integration/`** holds the reactor permission / multi-turn harness (scripted models via `@intx/inference-testing`). **`tests/e2e/`** (fixture-repo runs) is still planned. Until e2e exists, broader harness coverage also lives in co-located `*.test.ts` files and `tests/unit/`.
 - **Capability evals** (`evals/capability/`) are **not** the integration harness: they drive the product path (`corbits exec` / `runExec`) with real models against fixture copies and objective `verify.sh` graders. Case format + loader tests live under `evals/capability/`; run with `bun run eval:capability` (see `evals/capability/README.md`). Use `--baseline` to detect improve/regress across models or commits.
 - **TUI tests** are co-located `*.test.ts` files under `src/tui/` (e.g. `shell.test.ts`, `runner-host.test.ts`, `stream.test.ts`), run as part of `bun test` along with everything else; there is no separate `test:tui` script or test-setup preload.
+- **Parallel local runs** — `bun run test:parallel [N]` (default 4 workers) runs the same seeded suite with `--parallel=N`, wrapped in an output-stall watchdog (`scripts/test-parallel.ts`). Bun 1.4.x intermittently livelocks under `--parallel` (one worker spins at 100 % CPU holding a zombie git child while the main process idles; no output, no summary — upstream oven-sh/bun#36235), and `bun test` has no run-level timeout, so a stalled run hangs forever. The watchdog kills the suite's own process group after 90 s of silence and retries up to 3 times; a child that exits on its own (pass or fail) is never retried. CI keeps sharded sequential runs (`test:paths`) and does not use `--parallel`.
 
 ## Deployment
 

package.json

@@ -34,6 +34,7 @@
     "typecheck": "tsc --noEmit",
     "test": "bun test ./src ./tests ./evals ./scripts --randomize --seed 424242",
     "test:paths": "bun scripts/test-paths.ts",
+    "test:parallel": "bun scripts/test-parallel.ts",
     "lint": "oxfmt --check . && oxlint",
     "check:projects-dir-guard": "bun scripts/guard-real-projects-dir.ts",
     "check": "bun run lint && bun run typecheck && bun run build && bun run check:projects-dir-guard",

scripts/test-parallel.test.ts

@@ -0,0 +1,135 @@
+import { describe, expect, test } from "bun:test";
+import { mkdtempSync, readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { runWithWatchdog } from "./test-parallel.js";
+
+function processAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// These probes are `bun -e` one-liners, so they never import project modules
+// and finish in milliseconds. Stall windows are tiny (300-500ms) to keep the
+// file fast while still exercising the watchdog's timing logic.
+
+describe("runWithWatchdog", () => {
+  test("passes through a successful run without retrying", async () => {
+    const chunks: string[] = [];
+    const result = await runWithWatchdog({
+      command: process.execPath,
+      args: ["-e", 'console.log("hello-parallel")'],
+      stallMs: 5_000,
+      onOutput: (chunk) => chunks.push(Buffer.from(chunk).toString("utf8")),
+    });
+    expect(result.exitCode).toBe(0);
+    expect(result.stalled).toBe(false);
+    expect(result.attempts).toBe(1);
+    expect(chunks.join("")).toContain("hello-parallel");
+  });
+
+  test("passes through a failing run without retrying", async () => {
+    const result = await runWithWatchdog({
+      command: process.execPath,
+      args: ["-e", "process.exit(3)"],
+      stallMs: 5_000,
+      onStall: () => {
+        throw new Error("must not stall on a clean failure");
+      },
+    });
+    expect(result.exitCode).toBe(3);
+    expect(result.stalled).toBe(false);
+    expect(result.attempts).toBe(1);
+  });
+
+  test("output resets the stall timer", async () => {
+    // Prints every 100ms for ~700ms against a 300ms stall window: a broken
+    // timer that never reset would fire on the second tick.
+    const result = await runWithWatchdog({
+      command: process.execPath,
+      args: [
+        "-e",
+        "for (let i = 0; i < 7; i++) { console.log('tick', i); await new Promise(r => setTimeout(r, 100)); }",
+      ],
+      stallMs: 300,
+    });
+    expect(result.exitCode).toBe(0);
+    expect(result.stalled).toBe(false);
+    expect(result.attempts).toBe(1);
+  });
+
+  test("retries after a stall and returns the second attempt's result", async () => {
+    const dir = mkdtempSync(join(tmpdir(), "test-parallel-retry-"));
+    const flag = join(dir, "attempted");
+    const code = `
+      const fs = require("node:fs");
+      if (fs.existsSync(${JSON.stringify(flag)})) {
+        console.log("second attempt");
+      } else {
+        fs.writeFileSync(${JSON.stringify(flag)}, "1");
+        setTimeout(() => {}, 30_000);
+      }
+    `;
+    const stalls: [number, number][] = [];
+    const result = await runWithWatchdog({
+      command: process.execPath,
+      args: ["-e", code],
+      stallMs: 300,
+      onStall: (attempt, max) => stalls.push([attempt, max]),
+    });
+    expect(result.exitCode).toBe(0);
+    expect(result.stalled).toBe(false);
+    expect(result.attempts).toBe(2);
+    expect(stalls).toEqual([[1, 3]]);
+  });
+
+  test("gives up after the final attempt with exit code 1", async () => {
+    const stalls: [number, number][] = [];
+    const result = await runWithWatchdog({
+      command: process.execPath,
+      args: ["-e", "setTimeout(() => {}, 30_000)"],
+      stallMs: 300,
+      attempts: 2,
+      onStall: (attempt, max) => stalls.push([attempt, max]),
+    });
+    expect(result.exitCode).toBe(1);
+    expect(result.stalled).toBe(true);
+    expect(result.attempts).toBe(2);
+    // The final stall is reported by the exit code, not a retry notice.
+    expect(stalls).toEqual([[1, 2]]);
+  });
+
+  test("kills the whole process group, including the child's own children", async () => {
+    const dir = mkdtempSync(join(tmpdir(), "test-parallel-group-"));
+    const pidFile = join(dir, "grandchild.pid");
+    const code = `
+      const { spawn } = require("node:child_process");
+      const fs = require("node:fs");
+      const grandchild = spawn("sleep", ["30"], { stdio: "ignore" });
+      fs.writeFileSync(${JSON.stringify(pidFile)}, String(grandchild.pid));
+      setTimeout(() => {}, 30_000);
+    `;
+    const result = await runWithWatchdog({
+      command: process.execPath,
+      args: ["-e", code],
+      stallMs: 300,
+      attempts: 1,
+    });
+    expect(result.exitCode).toBe(1);
+    expect(result.stalled).toBe(true);
+
+    const grandchildPid = Number(readFileSync(pidFile, "utf8"));
+    // Reaped processes disappear from the pid namespace, but allow a brief
+    // window for the kernel to reap before concluding the kill failed.
+    const deadline = Date.now() + 2_000;
+    while (processAlive(grandchildPid) && Date.now() < deadline) {
+      await new Promise((r) => setTimeout(r, 50));
+    }
+    expect(processAlive(grandchildPid)).toBe(false);
+  });
+});

scripts/test-parallel.ts

@@ -0,0 +1,189 @@
+#!/usr/bin/env bun
+// Parallel test runner with a stall watchdog.
+//
+// `bun test --parallel` intermittently livelocks on Bun 1.4.x: a worker
+// spins at 100% CPU while a git child it spawned is left as a zombie, and
+// the suite produces no further output (upstream Bun issue #36235, still
+// open as of 1.4.2). bun test has no run-level timeout, so this wrapper
+// runs the suite in its own process group, watches for an output stall
+// well beyond any healthy run's silence, kills the group, and retries.
+// A child that exits on its own (pass or fail) is never retried.
+//
+// Usage: bun scripts/test-parallel.ts [workers]   (default 4)
+
+export const DEFAULT_WORKERS = 4;
+export const DEFAULT_STALL_MS = 90_000;
+export const DEFAULT_ATTEMPTS = 3;
+// SIGKILL a group that ignores SIGTERM this long.
+const KILL_GRACE_MS = 5_000;
+
+export interface WatchdogOptions {
+  // Arguments after the bun binary, e.g. ["test", "./src", "--parallel", "4"].
+  args: string[];
+  // Bun binary to spawn. Defaults to the running binary.
+  command?: string;
+  stallMs?: number;
+  attempts?: number;
+  onOutput?: (chunk: Uint8Array, stream: "stdout" | "stderr") => void;
+  // Report a stall so the caller can log the retry. Defaults to stderr.
+  onStall?: (attempt: number, maxAttempts: number) => void;
+}
+
+export interface WatchdogResult {
+  exitCode: number;
+  stalled: boolean;
+  attempts: number;
+}
+
+const STALL_CODE = -1;
+
+function groupAlive(pid: number): boolean {
+  try {
+    process.kill(-pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function terminateGroup(pid: number): Promise<void> {
+  try {
+    process.kill(-pid, "SIGTERM");
+  } catch {
+    return;
+  }
+  const deadline = Date.now() + KILL_GRACE_MS;
+  while (Date.now() < deadline) {
+    if (!groupAlive(pid)) return;
+    await new Promise((r) => setTimeout(r, 200));
+  }
+  try {
+    process.kill(-pid, "SIGKILL");
+  } catch {
+    // group already gone
+  }
+}
+
+interface AttemptOutcome {
+  exitCode: number;
+  stalled: boolean;
+}
+
+async function runAttempt(
+  command: string,
+  args: string[],
+  stallMs: number,
+  onOutput: WatchdogOptions["onOutput"],
+): Promise<AttemptOutcome> {
+  const child = Bun.spawn([command, ...args], {
+    stdout: "pipe",
+    stderr: "pipe",
+    // Own process group so a stall kills bun test, its workers, and any
+    // grandchild (e.g. git) together.
+    detached: true,
+  });
+
+  let lastOutput = Date.now();
+  let stalled = false;
+  let exited = false;
+
+  const touch = (chunk: Uint8Array, stream: "stdout" | "stderr") => {
+    lastOutput = Date.now();
+    onOutput?.(chunk, stream);
+  };
+  const pumpStream = async (
+    stream: typeof child.stdout,
+    kind: "stdout" | "stderr",
+  ): Promise<void> => {
+    const reader = stream.getReader();
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      touch(value, kind);
+    }
+  };
+  const pumps = [
+    pumpStream(child.stdout, "stdout"),
+    pumpStream(child.stderr, "stderr"),
+  ];
+
+  const tickMs = Math.max(100, Math.min(5_000, stallMs));
+  const watchdog = setInterval(() => {
+    if (exited) return;
+    // Re-check liveness: the child may have exited between output and now.
+    if (!groupAlive(child.pid)) {
+      exited = true;
+      return;
+    }
+    if (Date.now() - lastOutput > stallMs) {
+      stalled = true;
+      clearInterval(watchdog);
+      void terminateGroup(child.pid);
+    }
+  }, tickMs);
+
+  const raw = await child.exited;
+  exited = true;
+  clearInterval(watchdog);
+  await Promise.allSettled(pumps);
+  // null means killed by a signal; the stalled flag says who did it.
+  const exitCode = raw === null ? (stalled ? STALL_CODE : 1) : raw;
+  return { exitCode, stalled };
+}
+
+export async function runWithWatchdog(
+  opts: WatchdogOptions,
+): Promise<WatchdogResult> {
+  const stallMs = opts.stallMs ?? DEFAULT_STALL_MS;
+  const maxAttempts = opts.attempts ?? DEFAULT_ATTEMPTS;
+  const command = opts.command ?? process.execPath;
+  const reportStall =
+    opts.onStall ??
+    ((attempt, max) =>
+      console.error(
+        `[test:parallel] no output for ${Math.round(stallMs / 1000)}s — ` +
+          `killing stalled run (attempt ${attempt}/${max}); ` +
+          `Bun --parallel livelock, see Bun issue #36235`,
+      ));
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    const { exitCode, stalled } = await runAttempt(
+      command,
+      opts.args,
+      stallMs,
+      opts.onOutput,
+    );
+    if (!stalled) {
+      return { exitCode, stalled: false, attempts: attempt };
+    }
+    if (attempt < maxAttempts) reportStall(attempt, maxAttempts);
+  }
+  return { exitCode: 1, stalled: true, attempts: maxAttempts };
+}
+
+if (import.meta.main) {
+  const arg = process.argv[2];
+  const workers = arg === undefined ? DEFAULT_WORKERS : Number(arg);
+  if (!Number.isInteger(workers) || workers < 1 || workers > 16) {
+    console.error("usage: bun scripts/test-parallel.ts [workers 1-16]");
+    process.exit(2);
+  }
+  const result = await runWithWatchdog({
+    args: [
+      "test",
+      "./src",
+      "./tests",
+      "./evals",
+      "./scripts",
+      "--randomize",
+      "--seed",
+      "424242",
+      "--parallel",
+      String(workers),
+    ],
+    onOutput: (chunk, stream) => {
+      (stream === "stdout" ? process.stdout : process.stderr).write(chunk);
+    },
+  });
+  process.exit(result.exitCode);
+}