Skip to content

Commit 08e4cc2

Browse files
committed
Finalize run.json when the process crashes
Process-level uncaughtException/unhandledRejection handlers wrote a crash report but left run.json stuck at status: running, so crashed sessions kept reappearing in the resume picker as in-progress. runTUI now registers a narrow handle (session id, cwd, active flag) in a module-level slot the moment a run starts, clearing it on any finalize path it already owns. The top-level crash handler reads that slot and writes status: crashed plus finishedAt through a new saveCrashState that bypasses the per-session write chain entirely, so a write that never settles can't block process.exit.
1 parent c423b91 commit 08e4cc2

6 files changed

Lines changed: 183 additions & 11 deletions

File tree

src/index.ts

Lines changed: 49 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import { getLogger } from "@intx/log";
22
import { LOG_NAMESPACE_ROOT } from "./branding.js";
33
import { primeCrashReporting, writeCrashReport, type CrashKind } from "./crash/report.js";
4+
import { getActiveRun } from "./session/active-run.js";
5+
import { loadState, saveCrashState } from "./session/state.js";
46
import { loadConfig } from "./config/index.js";
57
import { ensureTelemetrySettings, globalSettingsPath } from "./config/settings.js";
68
import { flushPerfToOtel } from "./perf/index.js";
@@ -106,32 +108,69 @@ export async function main(argv: readonly string[]): Promise<number> {
106108
});
107109
}
108110

109-
async function handleFatal(kind: CrashKind, error: unknown): Promise<void> {
111+
// Exported so an integration test can register these process-level handlers
112+
// and inject a crash without spawning the full TUI stack.
113+
export async function handleFatal(kind: CrashKind, error: unknown): Promise<void> {
110114
process.stderr.write(`${kind}: ${error instanceof Error ? (error.stack ?? error.message) : String(error)}\n`);
111115
const file = await writeCrashReport(kind, error);
112116
if (file !== null) {
113117
process.stderr.write(`crash report written to ${file}\n`);
114118
} else {
115119
process.stderr.write("failed to write crash report\n");
116120
}
121+
await finalizeActiveRunOnCrash(error);
117122
process.exit(1);
118123
}
119124

120-
if (import.meta.main) {
121-
// OpenTUI installs a process-global uncaughtException/unhandledRejection
122-
// handler that only logs (opentui/core's Renderer.handleError), which
123-
// suppresses Bun's default print-and-exit. Combined with raw-mode stdin
124-
// holding the event loop open, an escaped throw would otherwise hang the
125-
// process forever with the terminal still in the alternate screen. Node
126-
// invokes every registered listener for the event regardless of order, so
127-
// these still run and terminate the process even though OpenTUI's own
128-
// listener never exits or rethrows.
125+
// A crash reaching here escaped without ever hitting runTUI's own try/catch
126+
// (e.g. a throw inside a fire-and-forget `void` call), so run.json was never
127+
// closed out. getActiveRun surfaces the in-flight session set by runTUI; the
128+
// write itself goes through saveCrashState, which bypasses the per-session
129+
// write chain in state.ts on purpose — chaining behind a write that never
130+
// settles (possibly the very write that triggered this crash) would block
131+
// process.exit indefinitely, defeating this handler's one job.
132+
async function finalizeActiveRunOnCrash(error: unknown): Promise<void> {
133+
const run = getActiveRun();
134+
if (run === null || !run.active) return;
135+
const message = error instanceof Error ? error.message : String(error);
136+
try {
137+
const prior = await loadState(run.cwd, run.sessionId);
138+
await saveCrashState(run.cwd, run.sessionId, {
139+
status: "crashed",
140+
turnsUsed: prior?.turnsUsed ?? 0,
141+
task: prior?.task ?? "(conversation)",
142+
startedAt: prior?.startedAt ?? Date.now(),
143+
finishedAt: Date.now(),
144+
error: message,
145+
...(prior?.model !== undefined ? { model: prior.model } : {}),
146+
...(prior?.mcpServers !== undefined ? { mcpServers: prior.mcpServers } : {}),
147+
});
148+
} catch (saveErr: unknown) {
149+
process.stderr.write(
150+
`failed to finalize run state after crash: ${saveErr instanceof Error ? saveErr.message : String(saveErr)}\n`,
151+
);
152+
}
153+
}
154+
155+
// OpenTUI installs a process-global uncaughtException/unhandledRejection
156+
// handler that only logs (opentui/core's Renderer.handleError), which
157+
// suppresses Bun's default print-and-exit. Combined with raw-mode stdin
158+
// holding the event loop open, an escaped throw would otherwise hang the
159+
// process forever with the terminal still in the alternate screen. Node
160+
// invokes every registered listener for the event regardless of order, so
161+
// these still run and terminate the process even though OpenTUI's own
162+
// listener never exits or rethrows.
163+
export function installCrashHandlers(): void {
129164
process.on("uncaughtException", (err) => {
130165
void handleFatal("uncaughtException", err);
131166
});
132167
process.on("unhandledRejection", (reason) => {
133168
void handleFatal("unhandledRejection", reason);
134169
});
170+
}
171+
172+
if (import.meta.main) {
173+
installCrashHandlers();
135174

136175
let code: number;
137176
try {

src/session/active-run.ts

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// A module-level slot the top-level uncaughtException/unhandledRejection
2+
// handler (src/index.ts) can reach even though persistRunSnapshot is a
3+
// closure local to runTUI. Only ever consulted from the crash path: a run
4+
// that never crashes never has this read.
5+
export type RunStateHandle = {
6+
sessionId: string;
7+
cwd: string;
8+
active: boolean;
9+
};
10+
11+
let activeRun: RunStateHandle | null = null;
12+
13+
export function setActiveRun(handle: RunStateHandle): void {
14+
activeRun = handle;
15+
}
16+
17+
export function clearActiveRun(): void {
18+
activeRun = null;
19+
}
20+
21+
export function getActiveRun(): RunStateHandle | null {
22+
return activeRun;
23+
}

src/session/state.ts

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ const ConnectedMcpServerSchema = type({
1414
export type ConnectedMcpServer = typeof ConnectedMcpServerSchema.infer;
1515

1616
const RunStateSchema = type({
17-
status: "'running' | 'done' | 'failed' | 'cancelled'",
17+
status: "'running' | 'done' | 'failed' | 'cancelled' | 'crashed'",
1818
turnsUsed: "number",
1919
task: "string",
2020
startedAt: "number",
@@ -92,6 +92,22 @@ export async function saveState(
9292
}
9393

9494

95+
// Crash-time terminal write. Deliberately bypasses writeChains: a hung or
96+
// still-pending write for this session (possibly the very write mid-flight
97+
// when the process crashed) must never be awaited here, or a queued write
98+
// that never settles would block the crash handler's process.exit forever.
99+
// There is no later write to order against once the process is exiting, so
100+
// per-session ordering has nothing left to protect.
101+
export async function saveCrashState(
102+
cwd: string,
103+
sessionId: string,
104+
state: RunState,
105+
home?: string,
106+
): Promise<void> {
107+
const path = statePath(cwd, sessionId, home);
108+
await atomicWrite(path, JSON.stringify(state, null, 2));
109+
}
110+
95111
// Returns the parsed state, or the arktype error summary when the shape is
96112
// invalid, so callers can surface a specific reason rather than "invalid shape".
97113
function parseRunState(data: unknown): RunState | { error: string } {

src/tui/runner.ts

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ import { createRunSink } from "../session/run-sink.js";
161161
import { generateSessionId, initSessionDir, renameSession, sessionContextDir, sessionDir } from "../session/index.js";
162162
import { resolveSessionLabel, truncateSessionLabel } from "../session/session-label.js";
163163
import { loadState, saveState, type ConnectedMcpServer, type RunState } from "../session/state.js";
164+
import { setActiveRun, clearActiveRun, type RunStateHandle } from "../session/active-run.js";
164165
import { openInBrowser } from "../auth/oauth/browser.js";
165166
import { pickSession } from "./pick-session.js";
166167
import { RESUME_TRANSCRIPT_BLOCK_LIMIT, turnsToContentBlocks } from "./turns-to-blocks.js";
@@ -468,6 +469,15 @@ export async function runTUI(initialConfig: Config): Promise<number> {
468469
mcpServers: resumeSeed.mcpServers,
469470
});
470471

472+
// Registered the moment a run starts so the top-level uncaughtException /
473+
// unhandledRejection handler in index.ts (which cannot see any local state
474+
// in this function) can finalize run.json for crashes that escape without
475+
// ever reaching this function's own try/catch — e.g. a throw inside a
476+
// fire-and-forget `void` call. Cleared wherever `finalized` below flips
477+
// true, since those paths already write a terminal run.json themselves.
478+
const activeRunHandle: RunStateHandle = { sessionId, cwd: config.cwd, active: true };
479+
setActiveRun(activeRunHandle);
480+
471481
// Crash guard: if anything from setup onward throws all the way out of
472482
// runTUI instead of reaching the normal finalize block, this still closes
473483
// out run.json so status and finishedAt never disagree. Declared before the
@@ -494,6 +504,8 @@ export async function runTUI(initialConfig: Config): Promise<number> {
494504
const finalizeOnCrash = async (err: unknown): Promise<void> => {
495505
if (finalized) return;
496506
finalized = true;
507+
activeRunHandle.active = false;
508+
clearActiveRun();
497509
await flushPartialOnCrash().catch((flushErr: unknown) => {
498510
// Best-effort only — still attempt saveState below. Log so a flush
499511
// failure is not invisible when diagnosing a crash exit.
@@ -1630,6 +1642,7 @@ export async function runTUI(initialConfig: Config): Promise<number> {
16301642
});
16311643
await persistRunSnapshot("done", { finishedAt: Date.now() });
16321644
sessionId = generateSessionId();
1645+
activeRunHandle.sessionId = sessionId;
16331646
startedAt = Date.now();
16341647
runTaskTitle = config.task;
16351648
emitter.emit("session.title", runTaskTitle.trim().length > 0 ? truncateSessionLabel(runTaskTitle) : "Untitled session");
@@ -2244,6 +2257,8 @@ export async function runTUI(initialConfig: Config): Promise<number> {
22442257
// finished run (finishedAt set) can be left reading as still in progress.
22452258
const persistedStatus: RunState["status"] = summaryStatus;
22462259
finalized = true;
2260+
activeRunHandle.active = false;
2261+
clearActiveRun();
22472262
await writeRunSnapshot(persistedStatus, {
22482263
finishedAt,
22492264
...(sinkError !== undefined ? { error: sinkError } : {}),
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// Spawned as a subprocess by tests/integration/crash-finalize.test.ts. Mimics
2+
// what runTUI does at startup (register the active run, write the initial
3+
// "running" run.json) and what index.ts does at process entry (install the
4+
// crash handlers), then throws asynchronously so it surfaces as a genuine
5+
// uncaughtException rather than a synchronous throw the caller could catch.
6+
import { installCrashHandlers } from "../../../src/index.js";
7+
import { setActiveRun } from "../../../src/session/active-run.js";
8+
import { sessionDir } from "../../../src/session/index.js";
9+
import { saveState } from "../../../src/session/state.js";
10+
11+
const cwd = process.cwd();
12+
const sessionId = process.env["CRASH_TEST_SESSION_ID"];
13+
if (sessionId === undefined) {
14+
throw new Error("CRASH_TEST_SESSION_ID must be set");
15+
}
16+
17+
await saveState(cwd, sessionId, {
18+
status: "running",
19+
turnsUsed: 3,
20+
task: "simulated crash task",
21+
startedAt: Date.now(),
22+
});
23+
24+
setActiveRun({ sessionId, cwd, active: true });
25+
installCrashHandlers();
26+
27+
process.stdout.write(`${sessionDir(cwd, sessionId)}\n`);
28+
29+
setImmediate(() => {
30+
throw new Error("simulated crash");
31+
});
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
2+
import { tmpdir } from "node:os";
3+
import { join } from "node:path";
4+
5+
import { describe, expect, test } from "bun:test";
6+
7+
import { generateSessionId } from "../../src/session/index.js";
8+
import type { RunState } from "../../src/session/state.js";
9+
import { isResumableByDefault } from "../../src/tui/pick-session.js";
10+
11+
const FIXTURE = join(import.meta.dirname, "../fixtures/crash-run/simulate-crash.ts");
12+
13+
describe("integration — crash finalizes run.json", () => {
14+
test("uncaughtException writes status: crashed with finishedAt", async () => {
15+
const cwd = mkdtempSync(join(tmpdir(), "corbits-crash-cwd-"));
16+
const home = mkdtempSync(join(tmpdir(), "corbits-crash-home-"));
17+
const sessionId = generateSessionId();
18+
19+
try {
20+
const proc = Bun.spawn(["bun", "run", FIXTURE], {
21+
cwd,
22+
env: { ...process.env, HOME: home, CRASH_TEST_SESSION_ID: sessionId },
23+
stdout: "pipe",
24+
stderr: "pipe",
25+
});
26+
27+
const exitCode = await proc.exited;
28+
const stdout = await new Response(proc.stdout).text();
29+
const stderr = await new Response(proc.stderr).text();
30+
31+
expect(exitCode).toBe(1);
32+
expect(stderr).toContain("uncaughtException: Error: simulated crash");
33+
34+
const runJsonPath = join(stdout.trim(), "run.json");
35+
const raw = readFileSync(runJsonPath, "utf8");
36+
const state = JSON.parse(raw) as RunState;
37+
38+
expect(state.status).toBe("crashed");
39+
expect(state.finishedAt).toBeGreaterThan(0);
40+
expect(state.error).toContain("simulated crash");
41+
expect(state.task).toBe("simulated crash task");
42+
expect(isResumableByDefault(state)).toBe(false);
43+
} finally {
44+
rmSync(cwd, { recursive: true, force: true });
45+
rmSync(home, { recursive: true, force: true });
46+
}
47+
}, 15_000);
48+
});

0 commit comments

Comments
 (0)