Make the test suite runnable under bun test --parallel

docs/IMPLEMENTATION.md

@@ -521,6 +521,7 @@ Run all three before declaring work complete.
 - **`tests/integration/`** holds the reactor permission / multi-turn harness (scripted models via `@intx/inference-testing`). **`tests/e2e/`** (fixture-repo runs) is still planned. Until e2e exists, broader harness coverage also lives in co-located `*.test.ts` files and `tests/unit/`.
 - **Capability evals** (`evals/capability/`) are **not** the integration harness: they drive the product path (`corbits exec` / `runExec`) with real models against fixture copies and objective `verify.sh` graders. Case format + loader tests live under `evals/capability/`; run with `bun run eval:capability` (see `evals/capability/README.md`). Use `--baseline` to detect improve/regress across models or commits.
 - **TUI tests** are co-located `*.test.ts` files under `src/tui/` (e.g. `shell.test.ts`, `runner-host.test.ts`, `stream.test.ts`), run as part of `bun test` along with everything else; there is no separate `test:tui` script or test-setup preload.
+- **Parallel local runs** — `bun run test:parallel [N]` (default 4 workers) runs the same seeded suite with `--parallel=N`, wrapped in an output-stall watchdog (`scripts/test-parallel.ts`). Bun 1.4.x intermittently livelocks under `--parallel` (one worker spins at 100 % CPU holding a zombie git child while the main process idles; no output, no summary — upstream oven-sh/bun#36235), and `bun test` has no run-level timeout, so a stalled run hangs forever. The watchdog kills the suite's own process group after 90 s of silence and retries up to 3 times; a child that exits on its own (pass or fail) is never retried. CI keeps sharded sequential runs (`test:paths`) and does not use `--parallel`.
 
 ## Deployment
 

package.json

@@ -34,6 +34,7 @@
     "typecheck": "tsc --noEmit",
     "test": "bun test ./src ./tests ./evals ./scripts --randomize --seed 424242",
     "test:paths": "bun scripts/test-paths.ts",
+    "test:parallel": "bun scripts/test-parallel.ts",
     "lint": "oxfmt --check . && oxlint",
     "check:projects-dir-guard": "bun scripts/guard-real-projects-dir.ts",
     "check": "bun run lint && bun run typecheck && bun run build && bun run check:projects-dir-guard",

scripts/test-parallel.test.ts

@@ -0,0 +1,135 @@
+import { describe, expect, test } from "bun:test";
+import { mkdtempSync, readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { runWithWatchdog } from "./test-parallel.js";
+
+function processAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// These probes are `bun -e` one-liners, so they never import project modules
+// and finish in milliseconds. Stall windows are tiny (300-500ms) to keep the
+// file fast while still exercising the watchdog's timing logic.
+
+describe("runWithWatchdog", () => {
+  test("passes through a successful run without retrying", async () => {
+    const chunks: string[] = [];
+    const result = await runWithWatchdog({
+      command: process.execPath,
+      args: ["-e", 'console.log("hello-parallel")'],
+      stallMs: 5_000,
+      onOutput: (chunk) => chunks.push(Buffer.from(chunk).toString("utf8")),
+    });
+    expect(result.exitCode).toBe(0);
+    expect(result.stalled).toBe(false);
+    expect(result.attempts).toBe(1);
+    expect(chunks.join("")).toContain("hello-parallel");
+  });
+
+  test("passes through a failing run without retrying", async () => {
+    const result = await runWithWatchdog({
+      command: process.execPath,
+      args: ["-e", "process.exit(3)"],
+      stallMs: 5_000,
+      onStall: () => {
+        throw new Error("must not stall on a clean failure");
+      },
+    });
+    expect(result.exitCode).toBe(3);
+    expect(result.stalled).toBe(false);
+    expect(result.attempts).toBe(1);
+  });
+
+  test("output resets the stall timer", async () => {
+    // Prints every 100ms for ~700ms against a 300ms stall window: a broken
+    // timer that never reset would fire on the second tick.
+    const result = await runWithWatchdog({
+      command: process.execPath,
+      args: [
+        "-e",
+        "for (let i = 0; i < 7; i++) { console.log('tick', i); await new Promise(r => setTimeout(r, 100)); }",
+      ],
+      stallMs: 300,
+    });
+    expect(result.exitCode).toBe(0);
+    expect(result.stalled).toBe(false);
+    expect(result.attempts).toBe(1);
+  });
+
+  test("retries after a stall and returns the second attempt's result", async () => {
+    const dir = mkdtempSync(join(tmpdir(), "test-parallel-retry-"));
+    const flag = join(dir, "attempted");
+    const code = `
+      const fs = require("node:fs");
+      if (fs.existsSync(${JSON.stringify(flag)})) {
+        console.log("second attempt");
+      } else {
+        fs.writeFileSync(${JSON.stringify(flag)}, "1");
+        setTimeout(() => {}, 30_000);
+      }
+    `;
+    const stalls: [number, number][] = [];
+    const result = await runWithWatchdog({
+      command: process.execPath,
+      args: ["-e", code],
+      stallMs: 300,
+      onStall: (attempt, max) => stalls.push([attempt, max]),
+    });
+    expect(result.exitCode).toBe(0);
+    expect(result.stalled).toBe(false);
+    expect(result.attempts).toBe(2);
+    expect(stalls).toEqual([[1, 3]]);
+  });
+
+  test("gives up after the final attempt with exit code 1", async () => {
+    const stalls: [number, number][] = [];
+    const result = await runWithWatchdog({
+      command: process.execPath,
+      args: ["-e", "setTimeout(() => {}, 30_000)"],
+      stallMs: 300,
+      attempts: 2,
+      onStall: (attempt, max) => stalls.push([attempt, max]),
+    });
+    expect(result.exitCode).toBe(1);
+    expect(result.stalled).toBe(true);
+    expect(result.attempts).toBe(2);
+    // The final stall is reported by the exit code, not a retry notice.
+    expect(stalls).toEqual([[1, 2]]);
+  });
+
+  test("kills the whole process group, including the child's own children", async () => {
+    const dir = mkdtempSync(join(tmpdir(), "test-parallel-group-"));
+    const pidFile = join(dir, "grandchild.pid");
+    const code = `
+      const { spawn } = require("node:child_process");
+      const fs = require("node:fs");
+      const grandchild = spawn("sleep", ["30"], { stdio: "ignore" });
+      fs.writeFileSync(${JSON.stringify(pidFile)}, String(grandchild.pid));
+      setTimeout(() => {}, 30_000);
+    `;
+    const result = await runWithWatchdog({
+      command: process.execPath,
+      args: ["-e", code],
+      stallMs: 300,
+      attempts: 1,
+    });
+    expect(result.exitCode).toBe(1);
+    expect(result.stalled).toBe(true);
+
+    const grandchildPid = Number(readFileSync(pidFile, "utf8"));
+    // Reaped processes disappear from the pid namespace, but allow a brief
+    // window for the kernel to reap before concluding the kill failed.
+    const deadline = Date.now() + 2_000;
+    while (processAlive(grandchildPid) && Date.now() < deadline) {
+      await new Promise((r) => setTimeout(r, 50));
+    }
+    expect(processAlive(grandchildPid)).toBe(false);
+  });
+});

scripts/test-parallel.ts

@@ -0,0 +1,189 @@
+#!/usr/bin/env bun
+// Parallel test runner with a stall watchdog.
+//
+// `bun test --parallel` intermittently livelocks on Bun 1.4.x: a worker
+// spins at 100% CPU while a git child it spawned is left as a zombie, and
+// the suite produces no further output (upstream Bun issue #36235, still
+// open as of 1.4.2). bun test has no run-level timeout, so this wrapper
+// runs the suite in its own process group, watches for an output stall
+// well beyond any healthy run's silence, kills the group, and retries.
+// A child that exits on its own (pass or fail) is never retried.
+//
+// Usage: bun scripts/test-parallel.ts [workers]   (default 4)
+
+export const DEFAULT_WORKERS = 4;
+export const DEFAULT_STALL_MS = 90_000;
+export const DEFAULT_ATTEMPTS = 3;
+// SIGKILL a group that ignores SIGTERM this long.
+const KILL_GRACE_MS = 5_000;
+
+export interface WatchdogOptions {
+  // Arguments after the bun binary, e.g. ["test", "./src", "--parallel", "4"].
+  args: string[];
+  // Bun binary to spawn. Defaults to the running binary.
+  command?: string;
+  stallMs?: number;
+  attempts?: number;
+  onOutput?: (chunk: Uint8Array, stream: "stdout" | "stderr") => void;
+  // Report a stall so the caller can log the retry. Defaults to stderr.
+  onStall?: (attempt: number, maxAttempts: number) => void;
+}
+
+export interface WatchdogResult {
+  exitCode: number;
+  stalled: boolean;
+  attempts: number;
+}
+
+const STALL_CODE = -1;
+
+function groupAlive(pid: number): boolean {
+  try {
+    process.kill(-pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function terminateGroup(pid: number): Promise<void> {
+  try {
+    process.kill(-pid, "SIGTERM");
+  } catch {
+    return;
+  }
+  const deadline = Date.now() + KILL_GRACE_MS;
+  while (Date.now() < deadline) {
+    if (!groupAlive(pid)) return;
+    await new Promise((r) => setTimeout(r, 200));
+  }
+  try {
+    process.kill(-pid, "SIGKILL");
+  } catch {
+    // group already gone
+  }
+}
+
+interface AttemptOutcome {
+  exitCode: number;
+  stalled: boolean;
+}
+
+async function runAttempt(
+  command: string,
+  args: string[],
+  stallMs: number,
+  onOutput: WatchdogOptions["onOutput"],
+): Promise<AttemptOutcome> {
+  const child = Bun.spawn([command, ...args], {
+    stdout: "pipe",
+    stderr: "pipe",
+    // Own process group so a stall kills bun test, its workers, and any
+    // grandchild (e.g. git) together.
+    detached: true,
+  });
+
+  let lastOutput = Date.now();
+  let stalled = false;
+  let exited = false;
+
+  const touch = (chunk: Uint8Array, stream: "stdout" | "stderr") => {
+    lastOutput = Date.now();
+    onOutput?.(chunk, stream);
+  };
+  const pumpStream = async (
+    stream: typeof child.stdout,
+    kind: "stdout" | "stderr",
+  ): Promise<void> => {
+    const reader = stream.getReader();
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      touch(value, kind);
+    }
+  };
+  const pumps = [
+    pumpStream(child.stdout, "stdout"),
+    pumpStream(child.stderr, "stderr"),
+  ];
+
+  const tickMs = Math.max(100, Math.min(5_000, stallMs));
+  const watchdog = setInterval(() => {
+    if (exited) return;
+    // Re-check liveness: the child may have exited between output and now.
+    if (!groupAlive(child.pid)) {
+      exited = true;
+      return;
+    }
+    if (Date.now() - lastOutput > stallMs) {
+      stalled = true;
+      clearInterval(watchdog);
+      void terminateGroup(child.pid);
+    }
+  }, tickMs);
+
+  const raw = await child.exited;
+  exited = true;
+  clearInterval(watchdog);
+  await Promise.allSettled(pumps);
+  // null means killed by a signal; the stalled flag says who did it.
+  const exitCode = raw === null ? (stalled ? STALL_CODE : 1) : raw;
+  return { exitCode, stalled };
+}
+
+export async function runWithWatchdog(
+  opts: WatchdogOptions,
+): Promise<WatchdogResult> {
+  const stallMs = opts.stallMs ?? DEFAULT_STALL_MS;
+  const maxAttempts = opts.attempts ?? DEFAULT_ATTEMPTS;
+  const command = opts.command ?? process.execPath;
+  const reportStall =
+    opts.onStall ??
+    ((attempt, max) =>
+      console.error(
+        `[test:parallel] no output for ${Math.round(stallMs / 1000)}s — ` +
+          `killing stalled run (attempt ${attempt}/${max}); ` +
+          `Bun --parallel livelock, see Bun issue #36235`,
+      ));
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    const { exitCode, stalled } = await runAttempt(
+      command,
+      opts.args,
+      stallMs,
+      opts.onOutput,
+    );
+    if (!stalled) {
+      return { exitCode, stalled: false, attempts: attempt };
+    }
+    if (attempt < maxAttempts) reportStall(attempt, maxAttempts);
+  }
+  return { exitCode: 1, stalled: true, attempts: maxAttempts };
+}
+
+if (import.meta.main) {
+  const arg = process.argv[2];
+  const workers = arg === undefined ? DEFAULT_WORKERS : Number(arg);
+  if (!Number.isInteger(workers) || workers < 1 || workers > 16) {
+    console.error("usage: bun scripts/test-parallel.ts [workers 1-16]");
+    process.exit(2);
+  }
+  const result = await runWithWatchdog({
+    args: [
+      "test",
+      "./src",
+      "./tests",
+      "./evals",
+      "./scripts",
+      "--randomize",
+      "--seed",
+      "424242",
+      "--parallel",
+      String(workers),
+    ],
+    onOutput: (chunk, stream) => {
+      (stream === "stdout" ? process.stdout : process.stderr).write(chunk);
+    },
+  });
+  process.exit(result.exitCode);
+}