Skip to content

Commit 092bb0e

Browse files
Merge pull request #712 from corbitsdev/cl-7175-drop-interrupted-workers-from-the-live-agents-list
Drop interrupted workers from live agents
2 parents 1cfffe3 + a31bf68 commit 092bb0e

17 files changed

Lines changed: 442 additions & 42 deletions

CHANGELOG.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@ parallel copies under `docs/` or `scripts/notes/`. At cut time: rename
1919
the operator answers.
2020
- The `full_shell` overlay mode is removed; every overlay is inset.
2121

22+
### Fixed
23+
24+
- Interrupted workers linger on the agents strip for 4s then drop, instead of
25+
staying in the live list while leftover tools finish.
26+
2227
## [0.3.7] - 2026-08-27
2328

2429
### Fixed

docs/TUI.md

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -220,9 +220,10 @@ status / current tool) — Amp/Codex-style lanes without a FLEET header board:
220220

221221
`formatChromeZones``formatAgentsPanel` owns that paint. Geometry stays
222222
stack-only (`layoutMode: "stack"`, `railWidth: 0`); the zone max is
223-
`AGENTS_PANEL_MAX_VISIBLE + 1` (lanes plus a trailing `+N more`). Terminal
224-
lanes (done / failed / cancelled) linger for `AGENTS_PANEL_LINGER_MS` (4s)
225-
after `finishedAt`, then drop. Product-host sticky poll uses
223+
`AGENTS_PANEL_MAX_VISIBLE + 1` (lanes plus a trailing `+N more`). Finished
224+
lanes (done / failed / cancelled / interrupted) linger for
225+
`AGENTS_PANEL_LINGER_MS` (4s) after `finishedAt`, then drop. Product-host sticky
226+
poll uses
226227
`agentsChromeNeedsSticky` so clocks and linger stay fresh; while sticky is
227228
needed it **does not** call `bridge.syncAgentProgress` — chrome owns the live
228229
clocks.

src/inference-error-message.ts

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,11 @@ import {
2020
type InferenceErrorLike,
2121
} from "./inference-gateway-error.js";
2222

23+
/** Committed auth death — do not claim a refresh is in flight. */
24+
export const CREDENTIAL_FAILURE_USER_MESSAGE = "Authentication failed — log in again.";
25+
2326
const FRIENDLY_BY_CATEGORY: Record<string, string> = {
24-
// Committed auth death — do not claim a refresh is in flight.
25-
credential_failure: "Authentication failed — log in again.",
27+
credential_failure: CREDENTIAL_FAILURE_USER_MESSAGE,
2628
quota_exhausted: "Quota exhausted — usage limit reached.",
2729
context_overflow:
2830
"Context window full — compaction could not keep up. Try /clear to start fresh.",

src/subagent/agent-fleet.test.ts

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ import {
1515
} from "./lifecycle-tools.js";
1616
import { createSubAgentSessionStore } from "./session-store.js";
1717
import { createPermissionGate } from "../permission/gate.js";
18+
import { agentLaneIsLive, fleetProgress } from "../tui/agent-progress.js";
19+
import { AGENTS_PANEL_LINGER_MS, formatAgentsPanel } from "../tui/chrome-state.js";
1820
import { forcedStopReport } from "./stop-policy.js";
1921
import type { RunSubAgentParams, RunSubAgentResult } from "./types.js";
2022

@@ -921,6 +923,48 @@ describe("list_agents", () => {
921923
expect(parsed.agents[0]!.lifecycle).toBe("pending_init");
922924
gate.resolve({ report: "done" });
923925
});
926+
927+
test("interrupt_agent leaves the strip after the linger window", async () => {
928+
const gate = deferred<RunSubAgentResult>();
929+
const deps = makeDeps(async (params) => {
930+
params.onAgentReady?.({
931+
close: async () => {},
932+
interrupt: () => {},
933+
followup: async () => "",
934+
deliver: () => {},
935+
});
936+
return gate.promise;
937+
});
938+
const spawn = createSpawnAgentTool(deps);
939+
const interrupt = createInterruptAgentTool({
940+
sessions: deps.sessions,
941+
fleetRecords: deps.fleetRecords,
942+
});
943+
const spawned = await callTool(spawn, {
944+
description: "looping",
945+
prompt: "do it",
946+
intent: "explore",
947+
});
948+
const id = spawned.agent_id as string;
949+
await new Promise((resolve) => setTimeout(resolve, 20));
950+
if (interrupt.kind !== "full") throw new Error("expected full tool");
951+
await interrupt.handler(
952+
{ id: "int-strip", name: "interrupt_agent", arguments: { target: id } },
953+
new AbortController().signal,
954+
);
955+
const agents = deps.sessions.list();
956+
const session = agents[0]!;
957+
expect(session.status).toBe("running");
958+
expect(session.lifecycleStatus).toBe("interrupted");
959+
expect(agentLaneIsLive(session)).toBe(false);
960+
const finishedAt = session.finishedAt!;
961+
expect(finishedAt).toBeNumber();
962+
const inside = finishedAt + 1_000;
963+
expect(fleetProgress(agents, inside).running).toBe(0);
964+
expect(formatAgentsPanel(agents, undefined, inside)?.[0]?.status).toBe("interrupted");
965+
expect(formatAgentsPanel(agents, undefined, finishedAt + AGENTS_PANEL_LINGER_MS)).toBeNull();
966+
gate.resolve({ report: "done", interrupted: true });
967+
});
924968
});
925969

926970
describe("spawn_agent parity with task", () => {

src/subagent/agent-fleet.ts

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -546,14 +546,15 @@ export function createSpawnAgentTool(deps: AgentFleetDeps): AgentTool {
546546
const startedAt = Date.now();
547547
let settlement: Readonly<SubAgentRunSettlement> | undefined;
548548
let endFinalized = false;
549+
let runInterrupted = false;
549550
const finalizeEnd = (setupFailed = false): void => {
550551
if (endFinalized) return;
551552
endFinalized = true;
552553
const terminalSession = deps.sessions.get(session.id);
553554
const status =
554555
terminalSession?.status === "cancelled"
555556
? "cancelled"
556-
: terminalSession?.lifecycleStatus === "interrupted"
557+
: runInterrupted || terminalSession?.lifecycleStatus === "interrupted"
557558
? "interrupted"
558559
: (terminalSession?.status ?? "completed");
559560
captureSubagentEnd(telemetry, {
@@ -731,14 +732,16 @@ export function createSpawnAgentTool(deps: AgentFleetDeps): AgentTool {
731732
deps
732733
.run(params)
733734
.then((result) => {
734-
// interrupt_agent already flipped this session to "interrupted"
735-
// synchronously (session-store.interruptOne) — do not let the
736-
// settling promise's normal bookkeeping overwrite that with a
737-
// "completed" status. Still terminalize fleetRecords so a waiter
738-
// that never saw interrupt_agent (or raced it) cannot hang.
735+
// interrupt_agent / send_input already flipped this session
736+
// synchronously (session-store.interruptOne / sendInputOne) — do not
737+
// let the settling promise's normal bookkeeping overwrite that with
738+
// a "completed" status, and do not re-stamp the interrupt either: a
739+
// follow-up turn may already be live on this lane. Still terminalize
740+
// fleetRecords so a waiter that never saw interrupt_agent (or raced
741+
// it) cannot hang.
739742
if (result.interrupted === true) {
740743
keepWorktreeAlive = true;
741-
deps.sessions.interruptOne(session.id);
744+
runInterrupted = true;
742745
deps.fleetRecords.interrupt(session.id, result.report);
743746
return;
744747
}

src/subagent/lifecycle-tools.test.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,8 @@ describe("send_input", () => {
330330
expect(result).toEqual({ agent_id: worker.id, status: "interrupted" });
331331
expect(interrupted).toBe(true);
332332
expect(followupStarted).toBe(true);
333-
expect(sessions.get(worker.id)?.lifecycleStatus).toBe("interrupted");
333+
expect(sessions.get(worker.id)?.lifecycleStatus).toBe("running");
334+
expect(sessions.get(worker.id)?.finishedAt).toBeUndefined();
334335

335336
const missing = sessions.start({
336337
description: "no-followup",

src/subagent/session-store.test.ts

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ import { describe, expect, test } from "bun:test";
22

33
import { createSubAgentSessionStore } from "./session-store.js";
44
import { forcedStopReport } from "./stop-policy.js";
5+
import { agentLaneIsLive, fleetProgress } from "../tui/agent-progress.js";
6+
import { formatAgentsPanel } from "../tui/chrome-state.js";
57

68
import type { ReactorEmittedEvent } from "@intx/inference";
79

@@ -519,3 +521,132 @@ describe("CL-6943 reusable worker sessions", () => {
519521
expect(store.get(retained.id)).toBeUndefined();
520522
});
521523
});
524+
525+
describe("interrupt stamps finishedAt once", () => {
526+
test("interruptOne sets finishedAt, keeps status running, and preserves tools", () => {
527+
let t = 1000;
528+
const store = createSubAgentSessionStore({
529+
now: () => t,
530+
createId: () => "s-int",
531+
});
532+
const session = store.start({
533+
description: "looping",
534+
agentId: "explorer",
535+
brief: "b",
536+
retained: true,
537+
});
538+
store.markRunning(session.id);
539+
store.appendEvent(session.id, startCall(1, "call-1", "run_shell"));
540+
store.registerInterrupt(session.id, () => {});
541+
542+
t = 2000;
543+
expect(store.interruptOne(session.id).ok).toBe(true);
544+
const after = store.get(session.id);
545+
expect(after?.status).toBe("running");
546+
expect(after?.lifecycleStatus).toBe("interrupted");
547+
expect(after?.finishedAt).toBe(2000);
548+
expect(after?.outstandingTools).toHaveLength(1);
549+
expect(after?.currentToolName).toBe("run_shell");
550+
551+
t = 3500;
552+
expect(store.interruptOne(session.id).ok).toBe(true);
553+
expect(store.get(session.id)?.finishedAt).toBe(2000);
554+
expect(store.get(session.id)?.status).toBe("running");
555+
expect(store.get(session.id)?.outstandingTools).toHaveLength(1);
556+
});
557+
558+
test("sendInputOne interrupt starts a live follow-up turn and keeps tools", async () => {
559+
let t = 1000;
560+
let finish: (reply: string) => void = () => {};
561+
const store = createSubAgentSessionStore({
562+
now: () => t,
563+
createId: () => "s-send",
564+
});
565+
const session = store.start({
566+
description: "looping",
567+
agentId: "explorer",
568+
brief: "b",
569+
retained: true,
570+
});
571+
store.markRunning(session.id);
572+
store.appendEvent(session.id, startCall(1, "call-1", "run_shell"));
573+
store.registerInterrupt(session.id, () => {});
574+
store.registerFollowup(
575+
session.id,
576+
() =>
577+
new Promise<string>((resolve) => {
578+
finish = resolve;
579+
}),
580+
);
581+
582+
t = 2500;
583+
const outcome = store.sendInputOne(session.id, "stop that", { interrupt: true });
584+
expect(outcome).toEqual({ ok: true, status: "interrupted" });
585+
const after = store.get(session.id);
586+
expect(after?.status).toBe("running");
587+
expect(after?.lifecycleStatus).toBe("running");
588+
expect(after?.finishedAt).toBeUndefined();
589+
expect(after?.outstandingTools).toHaveLength(1);
590+
591+
t = 4000;
592+
expect(store.interruptOne(session.id).ok).toBe(true);
593+
expect(store.get(session.id)?.finishedAt).toBe(4000);
594+
595+
t = 5000;
596+
finish("later");
597+
await new Promise((resolve) => setTimeout(resolve, 0));
598+
expect(store.get(session.id)?.status).toBe("done");
599+
expect(store.get(session.id)?.lifecycleStatus).toBe("completed");
600+
expect(store.get(session.id)?.finishedAt).toBe(5000);
601+
});
602+
603+
test("a follow-up turn keeps the lane live past the linger window until it completes", async () => {
604+
let t = 1000;
605+
let finish: (reply: string) => void = () => {};
606+
const store = createSubAgentSessionStore({
607+
now: () => t,
608+
createId: () => "s-followup",
609+
});
610+
const session = store.start({
611+
description: "looping",
612+
agentId: "explorer",
613+
brief: "b",
614+
retained: true,
615+
});
616+
store.markRunning(session.id);
617+
store.registerInterrupt(session.id, () => {});
618+
store.registerFollowup(
619+
session.id,
620+
() =>
621+
new Promise<string>((resolve) => {
622+
finish = resolve;
623+
}),
624+
);
625+
626+
t = 2000;
627+
expect(store.interruptOne(session.id).ok).toBe(true);
628+
expect(store.get(session.id)?.finishedAt).toBe(2000);
629+
630+
t = 3000;
631+
const pending = store.followupOne(session.id, "keep going");
632+
633+
t = 11_000;
634+
store.appendEvent(session.id, startCall(1, "call-1", "run_shell"));
635+
const live = store.list();
636+
expect(live[0]?.lifecycleStatus).toBe("running");
637+
expect(live[0]?.finishedAt).toBeUndefined();
638+
expect(agentLaneIsLive(live[0]!)).toBe(true);
639+
expect(formatAgentsPanel(live, undefined, t)?.[0]?.status).toBe("running");
640+
expect(fleetProgress(live, t).running).toBe(1);
641+
642+
t = 12_000;
643+
finish("done");
644+
expect(await pending).toEqual({ ok: true, reply: "done" });
645+
const terminal = store.list();
646+
expect(terminal[0]?.status).toBe("done");
647+
expect(terminal[0]?.lifecycleStatus).toBe("completed");
648+
expect(terminal[0]?.finishedAt).toBe(12_000);
649+
expect(agentLaneIsLive(terminal[0]!)).toBe(false);
650+
expect(fleetProgress(terminal, t).running).toBe(0);
651+
});
652+
});

src/subagent/session-store.ts

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,9 @@ export interface SubAgentSession {
7979
// start/end, a status change). Distinct from startedAt so the strip can
8080
// tell a worker mid-turn from one that has gone silent.
8181
lastActivityAt: number;
82+
// Clock the live turn ended (complete/fail/cancel, and interrupt while TUI
83+
// status may still be "running"). Drives chrome linger; leftover tools may
84+
// still be outstanding after this stamp.
8285
finishedAt?: number;
8386
report?: string;
8487
error?: string;
@@ -594,6 +597,23 @@ export function createSubAgentSessionStore(
594597
notify();
595598
};
596599

600+
// A follow-up turn takes the lane back over: the worker is live again, so
601+
// the interrupt's linger stamp must not outlive the new turn. Completion
602+
// re-stamps through the caller's own mutate; a rejected turn restores the
603+
// addressable state it started from so followup_task can retry.
604+
const beginFollowupTurn = (id: string): void => {
605+
mutate(id, (s) => {
606+
s.lifecycleStatus = "running";
607+
delete s.finishedAt;
608+
});
609+
};
610+
const endFollowupTurn = (id: string, lifecycleStatus: AgentLifecycleStatus): void => {
611+
mutate(id, (s) => {
612+
s.lifecycleStatus = lifecycleStatus;
613+
s.finishedAt = now();
614+
});
615+
};
616+
597617
return {
598618
list(): readonly SubAgentSession[] {
599619
return [...sessions.values()].map(snapshotOf);
@@ -972,9 +992,7 @@ export function createSubAgentSessionStore(
972992
return { ok: false, status: session.lifecycleStatus };
973993
}
974994
interrupt();
975-
mutate(id, (s) => {
976-
s.lifecycleStatus = "interrupted";
977-
});
995+
beginFollowupTurn(id);
978996
void followup(message)
979997
.then((reply) => {
980998
const still = sessions.get(id);
@@ -990,6 +1008,7 @@ export function createSubAgentSessionStore(
9901008
pruneRetained();
9911009
})
9921010
.catch((err: unknown) => {
1011+
endFollowupTurn(id, "interrupted");
9931012
log.error("send_input followup failed for {id}: {error}", {
9941013
id,
9951014
error: err instanceof Error ? err.message : String(err),
@@ -1014,6 +1033,7 @@ export function createSubAgentSessionStore(
10141033
interrupt();
10151034
mutate(id, (s) => {
10161035
s.lifecycleStatus = "interrupted";
1036+
s.finishedAt = s.finishedAt ?? now();
10171037
});
10181038
pruneRetained();
10191039
return { ok: true };
@@ -1041,7 +1061,15 @@ export function createSubAgentSessionStore(
10411061
}
10421062
const followup = followupHandles.get(id);
10431063
if (followup === undefined) return { ok: false, status: session.lifecycleStatus };
1044-
const reply = await followup(message);
1064+
const priorLifecycle = session.lifecycleStatus;
1065+
beginFollowupTurn(id);
1066+
let reply: string;
1067+
try {
1068+
reply = await followup(message);
1069+
} catch (err) {
1070+
endFollowupTurn(id, priorLifecycle);
1071+
throw err;
1072+
}
10451073
mutate(id, (s) => {
10461074
s.status = "done";
10471075
s.lifecycleStatus = "completed";

src/subagent/spawn-agent-worktree.test.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,8 @@ describe("spawn_agent worktree isolation", () => {
347347
const content = typeof spawned.content === "string" ? spawned.content : "";
348348
const agentId = (JSON.parse(content) as { agent_id: string }).agent_id;
349349

350+
await waitFor(() => sessions.get(agentId)?.lifecycleStatus === "running");
351+
expect(sessions.interruptOne(agentId).ok).toBe(true);
350352
settle.resolve({
351353
report: "## Summary\nStopped.\n## Findings\npartial\n## Blockers\ninterrupted\n## Paths\n",
352354
stopReason: "cancelled",

0 commit comments

Comments
 (0)