Skip to content

Commit f901295

Browse files
committed
Remove unbounded read and write-chain race from crash finalize
The crash handler awaited loadState (a plain readFile) to recover task/startedAt/model before writing, the same unbounded-I/O hazard primeCrashReporting exists to avoid for git. active-run.ts now carries those fields directly, updated by runTUI wherever it already tracks them, so the handler needs no read. Bypassing writeChains for the crash write also reopened the exact race CL-5567 closed: an in-flight progress snapshot for the same session could still land after the crash write and resurrect status: running. saveState now checks a synchronous isCrashed() flag right before each queued write fires, so anything still waiting in the chain when the crash handler marks the process crashed steps aside instead of racing it.
1 parent 50ca55f commit f901295

6 files changed

Lines changed: 115 additions & 24 deletions

File tree

src/index.ts

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import { getLogger } from "@intx/log";
22
import { LOG_NAMESPACE_ROOT } from "./branding.js";
33
import { primeCrashReporting, writeCrashReport, type CrashKind } from "./crash/report.js";
4-
import { getActiveRun } from "./session/active-run.js";
5-
import { loadState, saveCrashState } from "./session/state.js";
4+
import { getActiveRun, markCrashed } from "./session/active-run.js";
5+
import { saveCrashState } from "./session/state.js";
66
import { loadConfig } from "./config/index.js";
77
import { ensureTelemetrySettings, globalSettingsPath } from "./config/settings.js";
88
import { installFileLogSink } from "./logging/sink.js";
@@ -118,6 +118,12 @@ export async function main(argv: readonly string[]): Promise<number> {
118118
// Exported so an integration test can register these process-level handlers
119119
// and inject a crash without spawning the full TUI stack.
120120
export async function handleFatal(kind: CrashKind, error: unknown): Promise<void> {
121+
// Flip this before any awaits below so any snapshot write still queued
122+
// behind another one in state.ts's per-session chain sees it and steps
123+
// aside the moment it's next in line, rather than racing saveCrashState's
124+
// rename() below. See markCrashed's doc comment for the residual window
125+
// this cannot close.
126+
markCrashed();
121127
process.stderr.write(`${kind}: ${error instanceof Error ? (error.stack ?? error.message) : String(error)}\n`);
122128
const file = await writeCrashReport(kind, error);
123129
if (file !== null) {
@@ -131,26 +137,28 @@ export async function handleFatal(kind: CrashKind, error: unknown): Promise<void
131137

132138
// A crash reaching here escaped without ever hitting runTUI's own try/catch
133139
// (e.g. a throw inside a fire-and-forget `void` call), so run.json was never
134-
// closed out. getActiveRun surfaces the in-flight session set by runTUI; the
135-
// write itself goes through saveCrashState, which bypasses the per-session
136-
// write chain in state.ts on purpose — chaining behind a write that never
137-
// settles (possibly the very write that triggered this crash) would block
138-
// process.exit indefinitely, defeating this handler's one job.
140+
// closed out. getActiveRun surfaces the in-flight session set by runTUI, with
141+
// enough (task, startedAt, model) carried on the handle itself that no read
142+
// of run.json is needed — a readFile here would be exactly the kind of
143+
// unbounded crash-path I/O primeCrashReporting (src/crash/report.ts) exists
144+
// to avoid for git: a stalled disk or network mount would block process.exit
145+
// forever. The write itself goes through saveCrashState, which bypasses the
146+
// per-session write chain in state.ts on purpose — chaining behind a write
147+
// that never settles (possibly the very write that triggered this crash)
148+
// would block process.exit indefinitely, defeating this handler's one job.
139149
async function finalizeActiveRunOnCrash(error: unknown): Promise<void> {
140150
const run = getActiveRun();
141151
if (run === null || !run.active) return;
142152
const message = error instanceof Error ? error.message : String(error);
143153
try {
144-
const prior = await loadState(run.cwd, run.sessionId);
145154
await saveCrashState(run.cwd, run.sessionId, {
146155
status: "crashed",
147-
turnsUsed: prior?.turnsUsed ?? 0,
148-
task: prior?.task ?? "(conversation)",
149-
startedAt: prior?.startedAt ?? Date.now(),
156+
turnsUsed: 0,
157+
task: run.task,
158+
startedAt: run.startedAt,
150159
finishedAt: Date.now(),
151160
error: message,
152-
...(prior?.model !== undefined ? { model: prior.model } : {}),
153-
...(prior?.mcpServers !== undefined ? { mcpServers: prior.mcpServers } : {}),
161+
...(run.model !== undefined ? { model: run.model } : {}),
154162
});
155163
} catch (saveErr: unknown) {
156164
process.stderr.write(

src/session/active-run.ts

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,20 @@
22
// handler (src/index.ts) can reach even though persistRunSnapshot is a
33
// closure local to runTUI. Only ever consulted from the crash path: a run
44
// that never crashes never has this read.
5+
//
6+
// Carries enough of the live run state (task, startedAt, model) that the
7+
// crash handler can build a full RunState record itself. It must not read
8+
// run.json back off disk to fill these in — an unbounded readFile on the
9+
// crash path has the exact failure mode primeCrashReporting (src/crash/
10+
// report.ts) exists to avoid for git: a stalled disk or network mount would
11+
// block process.exit forever.
512
export type RunStateHandle = {
613
sessionId: string;
714
cwd: string;
815
active: boolean;
16+
task: string;
17+
startedAt: number;
18+
model?: string;
919
};
1020

1121
let activeRun: RunStateHandle | null = null;
@@ -21,3 +31,21 @@ export function clearActiveRun(): void {
2131
export function getActiveRun(): RunStateHandle | null {
2232
return activeRun;
2333
}
34+
35+
// Set once, by the crash handler, immediately before it writes the terminal
36+
// "crashed" record. saveState (src/session/state.ts) reads this synchronously
37+
// right before each queued write actually fires, so any snapshot write still
38+
// waiting behind another one in its per-session chain sees the flag and
39+
// no-ops instead of firing after (and clobbering) the crash write. It cannot
40+
// stop a write whose writeFile/rename has already been dispatched to the
41+
// kernel at the moment the flag flips — that window is one atomicWrite call
42+
// wide, not the full remaining lifetime of the process.
43+
let crashed = false;
44+
45+
export function markCrashed(): void {
46+
crashed = true;
47+
}
48+
49+
export function isCrashed(): boolean {
50+
return crashed;
51+
}

src/session/state.ts

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { dirname, join } from "node:path";
44
import { type } from "arktype";
55

66
import { sessionDir } from "./index.js";
7+
import { isCrashed } from "./active-run.js";
78
import { COMMAND_NAME } from "../branding.js";
89

910
const ConnectedMcpServerSchema = type({
@@ -65,6 +66,18 @@ export function warnUnreadableState(path: string, reason: string): void {
6566
// only ever address one file per session.
6667
const writeChains = new Map<string, Promise<void>>();
6768

69+
// Checked right before a chained write actually fires (not at saveState()
70+
// call time) so a snapshot write still queued behind another one, at the
71+
// moment the crash handler flips this flag, sees it and no-ops instead of
72+
// landing after (and clobbering) the crash write issued via saveCrashState.
73+
// This cannot recall a write whose writeFile/rename has already been
74+
// dispatched to the kernel — that residual window is one atomicWrite call
75+
// wide (a small local JSON write), not the remaining lifetime of the process.
76+
async function atomicWriteUnlessCrashed(path: string, content: string): Promise<void> {
77+
if (isCrashed()) return;
78+
await atomicWrite(path, content);
79+
}
80+
6881
export async function saveState(
6982
cwd: string,
7083
sessionId: string,
@@ -75,8 +88,8 @@ export async function saveState(
7588
const content = JSON.stringify(state, null, 2);
7689
const previous = writeChains.get(sessionId) ?? Promise.resolve();
7790
const write = previous.then(
78-
() => atomicWrite(path, content),
79-
() => atomicWrite(path, content),
91+
() => atomicWriteUnlessCrashed(path, content),
92+
() => atomicWriteUnlessCrashed(path, content),
8093
);
8194
// Swallow the error in the chain tail (not in `write`, which still rejects
8295
// for this caller) so one failed save doesn't permanently wedge later
@@ -96,8 +109,9 @@ export async function saveState(
96109
// still-pending write for this session (possibly the very write mid-flight
97110
// when the process crashed) must never be awaited here, or a queued write
98111
// that never settles would block the crash handler's process.exit forever.
99-
// There is no later write to order against once the process is exiting, so
100-
// per-session ordering has nothing left to protect.
112+
// Callers must call markCrashed() (src/session/active-run.ts) before this, so
113+
// any snapshot write still queued behind another one in the chain steps
114+
// aside instead of racing this write's rename().
101115
export async function saveCrashState(
102116
cwd: string,
103117
sessionId: string,

src/tui/runner.ts

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -477,7 +477,14 @@ export async function runTUI(initialConfig: Config): Promise<number> {
477477
// ever reaching this function's own try/catch — e.g. a throw inside a
478478
// fire-and-forget `void` call. Cleared wherever `finalized` below flips
479479
// true, since those paths already write a terminal run.json themselves.
480-
const activeRunHandle: RunStateHandle = { sessionId, cwd: config.cwd, active: true };
480+
const activeRunHandle: RunStateHandle = {
481+
sessionId,
482+
cwd: config.cwd,
483+
active: true,
484+
task: runTaskTitle.trim().length > 0 ? runTaskTitle.trim() : "(conversation)",
485+
startedAt,
486+
model: `${config.providerName}:${config.model}`,
487+
};
481488
setActiveRun(activeRunHandle);
482489

483490
// Crash guard: if anything from setup onward throws all the way out of
@@ -1387,12 +1394,19 @@ export async function runTUI(initialConfig: Config): Promise<number> {
13871394
status: RunState["status"],
13881395
extra?: Pick<RunState, "finishedAt" | "error">,
13891396
): Promise<void> => {
1397+
const task = runTaskTitle.trim().length > 0 ? runTaskTitle.trim() : "(conversation)";
1398+
const model = `${liveSource.id}:${liveSource.model}`;
1399+
// Kept in step with every persisted snapshot so the crash handler's copy
1400+
// (activeRunHandle, read by index.ts) never lags what's actually on disk.
1401+
activeRunHandle.task = task;
1402+
activeRunHandle.startedAt = startedAt;
1403+
activeRunHandle.model = model;
13901404
await saveState(config.cwd, sessionId, {
13911405
status,
13921406
turnsUsed: runSink.getTurnCount(),
1393-
task: runTaskTitle.trim().length > 0 ? runTaskTitle.trim() : "(conversation)",
1407+
task,
13941408
startedAt,
1395-
model: `${liveSource.id}:${liveSource.model}`,
1409+
model,
13961410
mcpServers: connectedMcpServers,
13971411
...extra,
13981412
});

tests/fixtures/crash-run/simulate-crash.ts

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,18 +14,40 @@ if (sessionId === undefined) {
1414
throw new Error("CRASH_TEST_SESSION_ID must be set");
1515
}
1616

17+
const startedAt = Date.now();
18+
const task = "simulated crash task";
19+
const model = "test-provider:test-model";
20+
1721
await saveState(cwd, sessionId, {
1822
status: "running",
1923
turnsUsed: 3,
20-
task: "simulated crash task",
21-
startedAt: Date.now(),
24+
task,
25+
startedAt,
26+
model,
2227
});
2328

24-
setActiveRun({ sessionId, cwd, active: true });
29+
setActiveRun({ sessionId, cwd, active: true, task, startedAt, model });
2530
installCrashHandlers();
2631

2732
process.stdout.write(`${sessionDir(cwd, sessionId)}\n`);
2833

34+
// Queue a burst of unawaited straggler snapshot writes (what
35+
// persistRunSnapshot does on every turn/model-switch/MCP-connect event) right
36+
// before crashing. Each is chained onto the previous one in state.ts's
37+
// per-session write queue, so most of these are still waiting their turn —
38+
// not yet dispatched to the kernel — at the moment the crash handler flips
39+
// the isCrashed() flag. Without that guard, one of these landing after
40+
// saveCrashState's rename() would resurrect status: "running".
41+
for (let i = 0; i < 50; i++) {
42+
void saveState(cwd, sessionId, {
43+
status: "running",
44+
turnsUsed: i,
45+
task,
46+
startedAt,
47+
model,
48+
});
49+
}
50+
2951
setImmediate(() => {
3052
throw new Error("simulated crash");
3153
});

tests/integration/crash-finalize.test.ts

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import { isResumableByDefault } from "../../src/tui/pick-session.js";
1111
const FIXTURE = join(import.meta.dirname, "../fixtures/crash-run/simulate-crash.ts");
1212

1313
describe("integration — crash finalizes run.json", () => {
14-
test("uncaughtException writes status: crashed with finishedAt", async () => {
14+
test("uncaughtException writes status: crashed with finishedAt, racing in-flight snapshot writes", async () => {
1515
const cwd = mkdtempSync(join(tmpdir(), "corbits-crash-cwd-"));
1616
const home = mkdtempSync(join(tmpdir(), "corbits-crash-home-"));
1717
const sessionId = generateSessionId();
@@ -35,10 +35,15 @@ describe("integration — crash finalizes run.json", () => {
3535
const raw = readFileSync(runJsonPath, "utf8");
3636
const state = JSON.parse(raw) as RunState;
3737

38+
// The fixture also fires 50 unawaited straggler "running" snapshot
39+
// writes for the same session immediately before crashing. Without the
40+
// isCrashed() guard in saveState (src/session/state.ts), one of those
41+
// could win the rename() race and this would read back "running".
3842
expect(state.status).toBe("crashed");
3943
expect(state.finishedAt).toBeGreaterThan(0);
4044
expect(state.error).toContain("simulated crash");
4145
expect(state.task).toBe("simulated crash task");
46+
expect(state.model).toBe("test-provider:test-model");
4247
expect(isResumableByDefault(state)).toBe(false);
4348
} finally {
4449
rmSync(cwd, { recursive: true, force: true });

0 commit comments

Comments
 (0)