diff --git a/CHANGELOG.md b/CHANGELOG.md index 222ed4936..f1e279a14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,15 @@ parallel copies under `docs/` or `scripts/notes/`. At cut time: rename - `--force` is no longer accepted. It had no runtime effect; resume and the session picker already include failed and done sessions without it. +### Fixed + +- Stale `running` sessions age to `interrupted` after two missed 5-minute + heartbeats, leftover newer parseable `run.json.*.tmp` files recover by mtime + over a stale `run.json`, and resume persists `interrupted` before reopening + as `running`. Signals stay `failed`; missing or unreadable state stays + `crashed`. + + ## [0.3.20] - 2026-09-10 ### Added diff --git a/docs/TUI.md b/docs/TUI.md index 21e54fd09..c8128e513 100644 --- a/docs/TUI.md +++ b/docs/TUI.md @@ -540,7 +540,7 @@ in `mouse-reporting-disabled.test.ts` for both `runListModal` and click-to-expand or drag-to-scroll, so leaving mouse reporting off lets the terminal's own text selection and copy work by default, with no Alt+M dance required. The resume picker lists the 10 most recently persisted sessions -for this checkout — completed, failed, and crashed included. Recency is +for this checkout — completed, failed, crashed, and interrupted included. Recency is the last write to `run.json`, not start time. Type to filter by name (printable keys claim the `>` row, same as the model picker). diff --git a/src/exec/runner.ts b/src/exec/runner.ts index 85270a054..13d71a570 100644 --- a/src/exec/runner.ts +++ b/src/exec/runner.ts @@ -75,6 +75,7 @@ import { syncRunStateHandle, type RunStateHandle, } from "../session/active-run.js"; +import { startRunHeartbeat } from "../session/run-liveness.js"; import { setActiveDisposeHost, clearActiveDisposeHost, @@ -389,13 +390,18 @@ export async function runExec(config: Config): Promise { turnsUsed: 0, model: `${config.providerName}:${config.model}`, }; + let stopHeartbeat: (() => void) | undefined; const persist = async ( status: "running" | "done" | "failed" | "cancelled", extra?: { error?: string }, ): Promise => { if (finalized && status === "running") return; - if (status !== "running") finalized = true; + if (status !== "running") { + finalized = true; + stopHeartbeat?.(); + stopHeartbeat = undefined; + } const model = `${config.providerName}:${config.model}`; const nextTurnsUsed = runSink?.getTurnCount() ?? turnsUsed; syncRunStateHandle(activeRunHandle, { @@ -437,6 +443,10 @@ export async function runExec(config: Config): Promise { }; await persist("running"); + stopHeartbeat = startRunHeartbeat({ + shouldTick: () => !finalized, + tick: () => persist("running"), + }); try { // Pricing seed is optional for exec; continue without rates rather than fail the run. diff --git a/src/session/list-sessions.test.ts b/src/session/list-sessions.test.ts index aebff3e5d..3f04c9040 100644 --- a/src/session/list-sessions.test.ts +++ b/src/session/list-sessions.test.ts @@ -275,3 +275,19 @@ test("listSessions reports updatedAt from run.json mtime", async () => { expect(row?.updatedAt).toBeGreaterThanOrEqual(stamp - 2000); expect(row?.updatedAt).toBeLessThanOrEqual(stamp + 2000); }); + +test("listSessions ages a stale running session to interrupted", async () => { + const sessionId = generateSessionId(); + const runPath = await writeRun(sessionId, { + status: "running", + task: "stale live", + startedAt: 1, + }); + const oldSec = Math.floor(Date.now() / 1000) - 20 * 60; + await utimes(runPath, oldSec, oldSec); + + const listed = await listSessions(cwd, home); + const row = listed.find((s) => s.sessionId === sessionId); + expect(row?.status).toBe("interrupted"); + expect(row?.status).not.toBe("running"); +}); diff --git a/src/session/run-liveness.test.ts b/src/session/run-liveness.test.ts new file mode 100644 index 000000000..9fee0657b --- /dev/null +++ b/src/session/run-liveness.test.ts @@ -0,0 +1,312 @@ +import { mkdtemp, rm, utimes, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; + +import { + clearActiveRun, + setActiveRun, + type RunStateHandle, +} from "./active-run.js"; +import { generateSessionId, initSessionDir, sessionDir } from "./index.js"; +import { + ageStaleRunningState, + RUN_STALE_THRESHOLD_MS, + startRunHeartbeat, +} from "./run-liveness.js"; +import { loadState, saveState, type RunState } from "./state.js"; + +function baseState(over: Partial = {}): RunState { + return { + status: "running", + turnsUsed: 2, + task: "liveness", + startedAt: 1_000, + model: "test:model", + ...over, + }; +} + +describe("ageStaleRunningState", () => { + test("ages parseable stale running to interrupted", () => { + const aged = ageStaleRunningState(baseState(), 1_000, { + nowMs: 1_000 + RUN_STALE_THRESHOLD_MS + 1, + sessionId: "other", + }); + expect(aged.status).toBe("interrupted"); + expect(aged.finishedAt).toBe(1_000); + }); + + test("does not age a fresh running record", () => { + const aged = ageStaleRunningState(baseState(), 1_000, { + nowMs: 1_000 + RUN_STALE_THRESHOLD_MS, + }); + expect(aged.status).toBe("running"); + }); + + test("does not age the active run owned by this process", () => { + clearActiveRun(); + const sessionId = "live-session"; + const handle: RunStateHandle = { + sessionId, + cwd: "/tmp", + task: "live", + startedAt: 1, + turnsUsed: 0, + }; + setActiveRun(handle); + try { + const aged = ageStaleRunningState(baseState(), 1_000, { + nowMs: 1_000 + RUN_STALE_THRESHOLD_MS + 1, + sessionId, + }); + expect(aged.status).toBe("running"); + } finally { + clearActiveRun(); + } + }); +}); + +describe("startRunHeartbeat", () => { + test("ticks while active and stops after teardown", async () => { + let ticks = 0; + let active = true; + const stop = startRunHeartbeat({ + intervalMs: 20, + shouldTick: () => active, + tick: () => { + ticks += 1; + }, + }); + await new Promise((resolve) => setTimeout(resolve, 55)); + expect(ticks).toBeGreaterThan(0); + const beforeStop = ticks; + active = false; + stop(); + await new Promise((resolve) => setTimeout(resolve, 45)); + expect(ticks).toBe(beforeStop); + }); +}); + +describe("loadState tmp recovery and stale aging", () => { + let cwd = ""; + let home = ""; + + beforeEach(async () => { + const stamp = `${Date.now()}-${Math.random().toString(16).slice(2)}`; + cwd = await mkdtemp(join(tmpdir(), `corbits-liveness-cwd-${stamp}-`)); + home = await mkdtemp(join(tmpdir(), `corbits-liveness-home-${stamp}-`)); + }); + + afterEach(async () => { + await rm(cwd, { recursive: true, force: true }); + await rm(home, { recursive: true, force: true }); + }); + + test("prefers a newer parseable tmp over stale run.json by mtime", async () => { + const sessionId = generateSessionId(); + await initSessionDir(cwd, sessionId, home); + const dir = sessionDir(cwd, sessionId, home); + const runPath = join(dir, "run.json"); + await saveState(cwd, sessionId, baseState({ task: "old-on-disk" }), home); + const newer = baseState({ task: "newer-tmp", turnsUsed: 9 }); + const tmpPath = join(dir, `run.json.${process.pid}.recovery.tmp`); + await writeFile(tmpPath, JSON.stringify(newer, null, 2)); + const older = Math.floor(Date.now() / 1000) - 20 * 60; + const newerSec = Math.floor(Date.now() / 1000); + await utimes(runPath, older, older); + await utimes(tmpPath, newerSec, newerSec); + + const loaded = await loadState(cwd, sessionId, home, { + persistAgeOut: false, + staleThresholdMs: 10 * 60_000, + }); + expect(loaded).toMatchObject({ + kind: "ok", + state: { task: "newer-tmp", turnsUsed: 9 }, + }); + }); + + test("does not prefer a newer parseable tmp over a fresh run.json", async () => { + const sessionId = generateSessionId(); + await initSessionDir(cwd, sessionId, home); + const dir = sessionDir(cwd, sessionId, home); + await saveState(cwd, sessionId, baseState({ task: "canonical" }), home); + const tmpPath = join(dir, `run.json.${process.pid}.inflight.tmp`); + await writeFile( + tmpPath, + JSON.stringify(baseState({ task: "in-flight", turnsUsed: 9 }), null, 2), + ); + const runPath = join(dir, "run.json"); + const older = Math.floor(Date.now() / 1000) - 30; + const newerSec = Math.floor(Date.now() / 1000); + await utimes(runPath, older, older); + await utimes(tmpPath, newerSec, newerSec); + + const loaded = await loadState(cwd, sessionId, home, { + persistAgeOut: false, + }); + expect(loaded).toMatchObject({ + kind: "ok", + state: { task: "canonical" }, + }); + }); + + test("does not resurrect a newer running tmp over a fresh terminal run.json", async () => { + const sessionId = generateSessionId(); + await initSessionDir(cwd, sessionId, home); + const dir = sessionDir(cwd, sessionId, home); + await saveState( + cwd, + sessionId, + baseState({ status: "done", finishedAt: 999, task: "landed" }), + home, + ); + const tmpPath = join(dir, `run.json.${process.pid}.straggler.tmp`); + await writeFile( + tmpPath, + JSON.stringify(baseState({ task: "straggler-running" }), null, 2), + ); + const runPath = join(dir, "run.json"); + const landed = Math.floor(Date.now() / 1000); + await utimes(runPath, landed, landed); + await utimes(tmpPath, landed + 2, landed + 2); + + const loaded = await loadState(cwd, sessionId, home, { + persistAgeOut: false, + }); + expect(loaded).toMatchObject({ + kind: "ok", + state: { status: "done", task: "landed" }, + }); + }); + + test("recovers a parseable tmp when run.json is missing", async () => { + const sessionId = generateSessionId(); + await initSessionDir(cwd, sessionId, home); + const dir = sessionDir(cwd, sessionId, home); + const tmpPath = join(dir, `run.json.${process.pid}.orphan.tmp`); + await writeFile( + tmpPath, + JSON.stringify(baseState({ task: "orphan-tmp", turnsUsed: 4 }), null, 2), + ); + + const loaded = await loadState(cwd, sessionId, home, { + persistAgeOut: false, + }); + expect(loaded).toMatchObject({ + kind: "ok", + state: { task: "orphan-tmp", turnsUsed: 4 }, + }); + }); + + test("does not resurrect an older or equal-mtime tmp", async () => { + const sessionId = generateSessionId(); + await initSessionDir(cwd, sessionId, home); + const dir = sessionDir(cwd, sessionId, home); + await saveState(cwd, sessionId, baseState({ task: "canonical" }), home); + const tmpPath = join(dir, `run.json.${process.pid}.older.tmp`); + await writeFile( + tmpPath, + JSON.stringify(baseState({ task: "stale-tmp" }), null, 2), + ); + const runPath = join(dir, "run.json"); + const stamp = Math.floor(Date.now() / 1000); + await utimes(runPath, stamp, stamp); + await utimes(tmpPath, stamp - 30, stamp - 30); + + const loaded = await loadState(cwd, sessionId, home, { + persistAgeOut: false, + }); + expect(loaded).toMatchObject({ + kind: "ok", + state: { task: "canonical" }, + }); + }); + + test("does not prefer a malformed newer tmp", async () => { + const sessionId = generateSessionId(); + await initSessionDir(cwd, sessionId, home); + const dir = sessionDir(cwd, sessionId, home); + await saveState(cwd, sessionId, baseState({ task: "canonical" }), home); + const tmpPath = join(dir, `run.json.${process.pid}.bad.tmp`); + await writeFile(tmpPath, "{ not-json"); + const runPath = join(dir, "run.json"); + const older = Math.floor(Date.now() / 1000) - 60; + const newerSec = Math.floor(Date.now() / 1000); + await utimes(runPath, older, older); + await utimes(tmpPath, newerSec, newerSec); + + const loaded = await loadState(cwd, sessionId, home, { + persistAgeOut: false, + }); + expect(loaded).toMatchObject({ + kind: "ok", + state: { task: "canonical" }, + }); + }); + + test("sweeps aged temps but keeps a fresh in-flight tmp", async () => { + const sessionId = generateSessionId(); + await initSessionDir(cwd, sessionId, home); + const dir = sessionDir(cwd, sessionId, home); + await saveState(cwd, sessionId, baseState(), home); + const agedTmp = join(dir, `run.json.${process.pid}.aged.tmp`); + const freshTmp = join(dir, `run.json.${process.pid}.fresh.tmp`); + await writeFile(agedTmp, "{"); + await writeFile(freshTmp, "{"); + const agedSec = Math.floor(Date.now() / 1000) - 20 * 60; + const freshSec = Math.floor(Date.now() / 1000); + await utimes(agedTmp, agedSec, agedSec); + await utimes(freshTmp, freshSec, freshSec); + + await loadState(cwd, sessionId, home, { + nowMs: Date.now(), + tmpSweepAgeMs: 10 * 60_000, + persistAgeOut: false, + }); + + const { existsSync } = await import("node:fs"); + expect(existsSync(agedTmp)).toBe(false); + expect(existsSync(freshTmp)).toBe(true); + }); + + test("ages stale running to interrupted and persists it", async () => { + const sessionId = generateSessionId(); + await initSessionDir(cwd, sessionId, home); + await saveState(cwd, sessionId, baseState(), home); + const runPath = join(sessionDir(cwd, sessionId, home), "run.json"); + const oldSec = Math.floor(Date.now() / 1000) - 20 * 60; + await utimes(runPath, oldSec, oldSec); + + const loaded = await loadState(cwd, sessionId, home, { + nowMs: Date.now(), + staleThresholdMs: 10 * 60_000, + }); + expect(loaded).toMatchObject({ + kind: "ok", + state: { status: "interrupted", turnsUsed: 2 }, + }); + const again = await loadState(cwd, sessionId, home, { + persistAgeOut: false, + }); + expect(again).toMatchObject({ + kind: "ok", + state: { status: "interrupted" }, + }); + }); + + test("missing and unreadable stay non-interrupted", async () => { + const missingId = generateSessionId(); + expect(await loadState(cwd, missingId, home)).toEqual({ kind: "missing" }); + + const badId = generateSessionId(); + await initSessionDir(cwd, badId, home); + await writeFile( + join(sessionDir(cwd, badId, home), "run.json"), + "{ turnsUsed", + ); + expect(await loadState(cwd, badId, home)).toEqual({ kind: "unreadable" }); + }); +}); diff --git a/src/session/run-liveness.ts b/src/session/run-liveness.ts new file mode 100644 index 000000000..35845897b --- /dev/null +++ b/src/session/run-liveness.ts @@ -0,0 +1,68 @@ +import { getActiveRun } from "./active-run.js"; +import type { RunState } from "./state.js"; + +/** Cadence for mid-run `running` heartbeats that keep session mtime fresh. */ +export const RUN_HEARTBEAT_INTERVAL_MS = 5 * 60_000; + +/** Two missed heartbeats: a `running` record older than this is stale. */ +export const RUN_STALE_THRESHOLD_MS = 2 * RUN_HEARTBEAT_INTERVAL_MS; + +/** Same window as the stale threshold: in-flight atomicWrite temps stay. */ +export const RUN_TMP_SWEEP_AGE_MS = RUN_STALE_THRESHOLD_MS; + +export function isStaleRunningMtime( + mtimeMs: number, + nowMs: number, + staleThresholdMs: number = RUN_STALE_THRESHOLD_MS, +): boolean { + return nowMs - mtimeMs > staleThresholdMs; +} + +/** Age a parseable stale `running` record to `interrupted` (resumable). */ +export function ageStaleRunningState( + state: RunState, + mtimeMs: number, + opts: { + nowMs?: number; + staleThresholdMs?: number; + sessionId?: string; + } = {}, +): RunState { + if (state.status !== "running") return state; + const nowMs = opts.nowMs ?? Date.now(); + const staleThresholdMs = opts.staleThresholdMs ?? RUN_STALE_THRESHOLD_MS; + if (!isStaleRunningMtime(mtimeMs, nowMs, staleThresholdMs)) return state; + // A live process owns this session — heartbeat/write path still has it. + const active = getActiveRun(); + if ( + opts.sessionId !== undefined && + active !== null && + active.sessionId === opts.sessionId + ) { + return state; + } + return { + ...state, + status: "interrupted", + finishedAt: state.finishedAt ?? mtimeMs, + }; +} + +/** + * Periodic running snapshot writer. `unref` so the timer cannot keep a + * process alive after the run ends; callers must stop it on terminal paths + * so no post-terminal writes fire. + */ +export function startRunHeartbeat(args: { + intervalMs?: number; + shouldTick: () => boolean; + tick: () => void | Promise; +}): () => void { + const intervalMs = args.intervalMs ?? RUN_HEARTBEAT_INTERVAL_MS; + const timer = setInterval(() => { + if (!args.shouldTick()) return; + void Promise.resolve(args.tick()).catch(() => undefined); + }, intervalMs); + timer.unref?.(); + return () => clearInterval(timer); +} diff --git a/src/session/state.ts b/src/session/state.ts index 52df6bb21..1ad2510aa 100644 --- a/src/session/state.ts +++ b/src/session/state.ts @@ -1,11 +1,25 @@ -import { mkdir, writeFile, readFile, rename } from "node:fs/promises"; -import { dirname, join } from "node:path"; +import { + mkdir, + writeFile, + readFile, + rename, + readdir, + stat, + unlink, +} from "node:fs/promises"; +import { basename, dirname, join } from "node:path"; import { type } from "arktype"; import { getLogger } from "@intx/log"; import { sessionDir } from "./index.js"; import { clearActiveRun, getTestWriteGate, isCrashed } from "./active-run.js"; +import { + ageStaleRunningState, + isStaleRunningMtime, + RUN_STALE_THRESHOLD_MS, + RUN_TMP_SWEEP_AGE_MS, +} from "./run-liveness.js"; import { LOG_NAMESPACE_ROOT } from "../branding.js"; const log = getLogger([LOG_NAMESPACE_ROOT, "session", "state"]); @@ -18,7 +32,8 @@ const ConnectedMcpServerSchema = type({ export type ConnectedMcpServer = typeof ConnectedMcpServerSchema.infer; const RunStateSchema = type({ - status: "'running' | 'done' | 'failed' | 'cancelled' | 'crashed'", + status: + "'running' | 'done' | 'failed' | 'cancelled' | 'crashed' | 'interrupted'", turnsUsed: "number", task: "string", startedAt: "number", @@ -39,6 +54,15 @@ function statePath(cwd: string, sessionId: string, home?: string): string { return join(sessionDir(cwd, sessionId, home), "run.json"); } +function isENOENT(err: unknown): boolean { + return ( + typeof err === "object" && + err !== null && + "code" in err && + (err as { code?: unknown }).code === "ENOENT" + ); +} + let tmpWriteCounter = 0; // Write atomically: serialize to a unique temp file, then rename into place so a @@ -177,40 +201,195 @@ export type LoadStateResult = | { kind: "missing" } | { kind: "unreadable" }; +export type LoadStateOptions = { + nowMs?: number; + staleThresholdMs?: number; + tmpSweepAgeMs?: number; + /** Persist an aged-out interrupted record. Default true. */ + persistAgeOut?: boolean; +}; + +type CandidateFile = { + path: string; + mtimeMs: number; + state: RunState; +}; + +type InspectedRunFile = + | { kind: "ok"; state: RunState; mtimeMs: number } + | { kind: "unreadable"; reason: string; mtimeMs: number } + | { kind: "missing" }; + +async function inspectRunStateFile(path: string): Promise { + let raw: string; + let fileStat: { mtimeMs: number }; + try { + [raw, fileStat] = await Promise.all([readFile(path, "utf8"), stat(path)]); + } catch (err) { + if (isENOENT(err)) return { kind: "missing" }; + throw err; + } + try { + const parsed = parseRunState(JSON.parse(raw)); + if (!parsed.ok) { + return { + kind: "unreadable", + reason: `invalid shape: ${parsed.reason}`, + mtimeMs: fileStat.mtimeMs, + }; + } + return { kind: "ok", state: parsed.state, mtimeMs: fileStat.mtimeMs }; + } catch (err) { + if (err instanceof SyntaxError) { + return { + kind: "unreadable", + reason: "corrupt JSON", + mtimeMs: fileStat.mtimeMs, + }; + } + throw err; + } +} + +function isRunJsonTmpName(name: string, runBase: string): boolean { + return name.startsWith(`${runBase}.`) && name.endsWith(".tmp"); +} + +async function recoverPreferredRunState( + path: string, + sessionId: string, + opts: { + nowMs: number; + tmpSweepAgeMs: number; + staleThresholdMs: number; + }, +): Promise< + | { kind: "ok"; state: RunState; mtimeMs: number } + | { kind: "missing" } + | { kind: "unreadable"; reason: string } +> { + const dir = dirname(path); + const runBase = basename(path); + let entries: string[] = []; + try { + entries = await readdir(dir); + } catch (err) { + if (isENOENT(err)) return { kind: "missing" }; + throw err; + } + + const primary = await inspectRunStateFile(path); + + const tmpCandidates: CandidateFile[] = []; + for (const name of entries) { + if (!isRunJsonTmpName(name, runBase)) continue; + const tmpPath = join(dir, name); + const parsed = await inspectRunStateFile(tmpPath); + if (parsed.kind !== "ok") continue; + tmpCandidates.push({ + path: tmpPath, + mtimeMs: parsed.mtimeMs, + state: parsed.state, + }); + } + + // Prefer a newer parseable tmp only over missing, unreadable, or stale + // run.json. A fresh canonical file is the in-flight save's destination; + // tmp is the normal artifact of every atomicWrite and must not win. + const primaryMtime = + primary.kind === "missing" ? Number.NEGATIVE_INFINITY : primary.mtimeMs; + const primaryAllowsTmp = + !writeChains.has(sessionId) && + (primary.kind !== "ok" || + isStaleRunningMtime(primaryMtime, opts.nowMs, opts.staleThresholdMs)); + let bestTmp: CandidateFile | null = null; + if (primaryAllowsTmp) { + for (const candidate of tmpCandidates) { + if (candidate.mtimeMs <= primaryMtime) continue; + if (bestTmp === null || candidate.mtimeMs > bestTmp.mtimeMs) { + bestTmp = candidate; + } + } + } + + if (bestTmp !== null) { + try { + await rename(bestTmp.path, path); + } catch { + // Another reader may have won the rename; fall through to re-read. + } + } + + // Sweep aged temps so in-flight atomicWrite files under the age gate survive. + for (const name of entries) { + if (!isRunJsonTmpName(name, runBase)) continue; + const tmpPath = join(dir, name); + if (bestTmp !== null && tmpPath === bestTmp.path) continue; + try { + const tmpStat = await stat(tmpPath); + if (opts.nowMs - tmpStat.mtimeMs > opts.tmpSweepAgeMs) { + await unlink(tmpPath).catch(() => undefined); + } + } catch { + // Gone already. + } + } + + const recovered = await inspectRunStateFile(path); + if (recovered.kind === "ok") { + return recovered; + } + if (recovered.kind === "unreadable") { + return { kind: "unreadable", reason: recovered.reason }; + } + if (primary.kind === "unreadable") { + return { kind: "unreadable", reason: primary.reason }; + } + return { kind: "missing" }; +} + export async function loadState( cwd: string, sessionId: string, home?: string, + options: LoadStateOptions = {}, ): Promise { const path = statePath(cwd, sessionId, home); + const nowMs = options.nowMs ?? Date.now(); + const tmpSweepAgeMs = options.tmpSweepAgeMs ?? RUN_TMP_SWEEP_AGE_MS; + const staleThresholdMs = options.staleThresholdMs ?? RUN_STALE_THRESHOLD_MS; + const persistAgeOut = options.persistAgeOut !== false; - try { - const raw = await readFile(path, "utf8"); - const parsed = parseRunState(JSON.parse(raw)); - if (!parsed.ok) { + const recovered = await recoverPreferredRunState(path, sessionId, { + nowMs, + tmpSweepAgeMs, + staleThresholdMs, + }); + if (recovered.kind !== "ok") { + if (recovered.kind === "unreadable") { log.warn("unreadable session state at {path}: {reason}", { path, - reason: `invalid shape: ${parsed.reason}`, + reason: recovered.reason, }); return { kind: "unreadable" }; } - return { kind: "ok", state: parsed.state }; - } catch (err) { - if (err instanceof SyntaxError) { - log.warn("unreadable session state at {path}: {reason}", { + return recovered; + } + + const aged = ageStaleRunningState(recovered.state, recovered.mtimeMs, { + nowMs, + staleThresholdMs, + sessionId, + }); + if (aged.status !== recovered.state.status && persistAgeOut) { + try { + await saveState(cwd, sessionId, aged, home); + } catch (err: unknown) { + log.warn("failed to persist aged-out run state at {path}: {error}", { path, - reason: "corrupt JSON", + error: err instanceof Error ? err.message : String(err), }); - return { kind: "unreadable" }; } - if ( - typeof err === "object" && - err !== null && - "code" in err && - (err as { code?: unknown }).code === "ENOENT" - ) { - return { kind: "missing" }; - } - throw err; } + return { kind: "ok", state: aged }; } diff --git a/src/tui/pick-session.test.ts b/src/tui/pick-session.test.ts index 74a34b5d8..5a6b105ca 100644 --- a/src/tui/pick-session.test.ts +++ b/src/tui/pick-session.test.ts @@ -38,6 +38,9 @@ describe("sessionResumeLabel", () => { expect(sessionResumeLabel(summary({ status: "crashed" }))).toContain( "crashed", ); + expect(sessionResumeLabel(summary({ status: "interrupted" }))).toContain( + "interrupted", + ); }); test("falls back to Untitled session when the task is blank", () => { diff --git a/src/tui/runner/exit.ts b/src/tui/runner/exit.ts index 7e80c8751..9fb640bbb 100644 --- a/src/tui/runner/exit.ts +++ b/src/tui/runner/exit.ts @@ -30,6 +30,7 @@ import { } from "../../session/session-label.js"; import { clearActiveDisposeHost } from "../../session/active-host.js"; import { syncRunStateHandle } from "../../session/active-run.js"; +import { startRunHeartbeat } from "../../session/run-liveness.js"; import { getValidCodexToken } from "../../auth/codex/session.js"; import { getValidXaiToken } from "../../auth/xai/session.js"; import { suppressProviderFailurePresentation } from "../provider/failure-attempt.js"; @@ -246,6 +247,11 @@ export async function createRunLifecycle( }> { const { persistRunSnapshot } = createRunPersistence(state, services); state.persistRunSnapshot = persistRunSnapshot; + const stopHeartbeat = startRunHeartbeat({ + shouldTick: () => !services.crashGuard.isFinalized(), + tick: () => persistRunSnapshot("running"), + }); + state.stopRunHeartbeat = stopHeartbeat; // Cycles persist to the context store only on inference.done; the assembled // recorder keeps the in-flight cycle's text so an errored or interrupted @@ -658,6 +664,8 @@ export async function finalizeTUIRun( services: RunnerServices, ): Promise { await hostOf(state).waitUntilExit(); + state.stopRunHeartbeat?.(); + delete state.stopRunHeartbeat; // Stop workers before awaiting the session-op tail so a hung enqueue cannot // delay abort/reap. Persistence, hooks, and telemetry stay after stop. // Toolset dispose lives inside shutdownRuntime so quit, crash, and signals diff --git a/src/tui/runner/state.ts b/src/tui/runner/state.ts index 59e7bb630..c07e9d5b0 100644 --- a/src/tui/runner/state.ts +++ b/src/tui/runner/state.ts @@ -285,6 +285,7 @@ export interface RunnerState { ) => Promise; shutdownRuntime?: () => Promise; stopFleetReporting?: () => void; + stopRunHeartbeat?: () => void; withFleetPublicationSuspended?: (reset: () => void) => void; /** TUI primary: true when a queued Enter steer should yield in-flight wait_agents. */ hasQueuedSteer?: () => boolean; diff --git a/tests/unit/session/resume-interrupted.test.ts b/tests/unit/session/resume-interrupted.test.ts new file mode 100644 index 000000000..3770b8f7d --- /dev/null +++ b/tests/unit/session/resume-interrupted.test.ts @@ -0,0 +1,77 @@ +import { mkdtemp, rm, utimes } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; + +import { + generateSessionId, + initSessionDir, + sessionDir, +} from "../../../src/session/index.js"; +import { + loadState, + saveState, + type RunState, +} from "../../../src/session/state.js"; + +describe("resume persists interrupted before reopening running", () => { + let cwd = ""; + let home = ""; + + beforeEach(async () => { + cwd = await mkdtemp(join(tmpdir(), "corbits-resume-int-cwd-")); + home = await mkdtemp(join(tmpdir(), "corbits-resume-int-home-")); + }); + + afterEach(async () => { + await rm(cwd, { recursive: true, force: true }); + await rm(home, { recursive: true, force: true }); + }); + + test("stale running is persisted interrupted, then a resume reopen writes running", async () => { + const sessionId = generateSessionId(); + await initSessionDir(cwd, sessionId, home); + const original: RunState = { + status: "running", + turnsUsed: 6, + task: "resume me", + startedAt: 42, + model: "test:model", + }; + await saveState(cwd, sessionId, original, home); + const runPath = join(sessionDir(cwd, sessionId, home), "run.json"); + const oldSec = Math.floor(Date.now() / 1000) - 20 * 60; + await utimes(runPath, oldSec, oldSec); + + const aged = await loadState(cwd, sessionId, home, { + nowMs: Date.now(), + staleThresholdMs: 10 * 60_000, + }); + expect(aged).toMatchObject({ + kind: "ok", + state: { status: "interrupted", turnsUsed: 6 }, + }); + + // Mirror prepareTUISession's reopen-as-running write after resume pick. + await saveState( + cwd, + sessionId, + { + status: "running", + turnsUsed: 6, + task: "resume me", + startedAt: 42, + model: "test:model", + }, + home, + ); + const reopened = await loadState(cwd, sessionId, home, { + persistAgeOut: false, + }); + expect(reopened).toMatchObject({ + kind: "ok", + state: { status: "running", turnsUsed: 6 }, + }); + }); +});