Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions scripts/intervention-forensics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -151,15 +151,19 @@ for (const file of files) {
bucket = emptyBucket();
buckets.set(key, bucket);
}
bucket.count++;
const occurrences = record.count ?? 1;
bucket.count += occurrences;
const family = record.family ?? record.model ?? "unknown";
bucket.byFamily.set(family, (bucket.byFamily.get(family) ?? 0) + 1);
bucket.byFamily.set(
family,
(bucket.byFamily.get(family) ?? 0) + occurrences,
);
const model = record.model ?? "unknown";
bucket.byModel.set(model, (bucket.byModel.get(model) ?? 0) + 1);
bucket.byModel.set(model, (bucket.byModel.get(model) ?? 0) + occurrences);
if (record.class === "stop" || record.class === "nudge") {
interventionsByModel.set(
model,
(interventionsByModel.get(model) ?? 0) + 1,
(interventionsByModel.get(model) ?? 0) + occurrences,
);
}
if (record.measurement !== undefined) {
Expand Down
5 changes: 5 additions & 0 deletions src/subagent/intervention-log.ts
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,11 @@ export interface InterventionRecord {
};
/** Free-form specifics, kept short (a looped window, a refused fingerprint). */
detail?: string;
/**
* How many consecutive same-(id, state) firings this row stands for.
* Absent means 1 — older records and one-shot interventions omit it.
*/
count?: number;
}

/** Fields every record from one run shares, supplied once at construction. */
Expand Down
107 changes: 107 additions & 0 deletions src/subagent/nudge-director.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,77 @@ describe("SubAgentDirector tool failure recovery", () => {
expect(texts?.[0]).toContain("report the blocker");
});

test("N consecutive failed tools in one burst produce one coalesced intervention record", async () => {
const director = new SubAgentDirector("system", [], undefined, 30);
const caps = capabilities();
const records: { id: string; count?: number }[] = [];
director.observeInterventions((event) => {
records.push(
event.count === undefined
? { id: event.id }
: { id: event.id, count: event.count },
);
});

await director.decide(
inferenceDone(["fail-a", "fail-b", "fail-c"]),
state,
caps,
);
await director.decide(toolDone("fail-a", true), state, caps);
await director.decide(toolDone("fail-b", true), state, caps);
expect(records).toEqual([]);

const texts = ephemeralTexts(
inferAction(await director.decide(toolDone("fail-c", true), state, caps)),
);
expect(texts).toHaveLength(1);
expect(texts?.[0]).toContain("A tool call failed");
expect(records).toEqual([{ id: "tool-failure-recovery", count: 3 }]);
});

test("a single failed tool writes one recovery record with count omitted", async () => {
const director = new SubAgentDirector("system", [], undefined, 30);
const caps = capabilities();
const records: { id: string; class?: string; count?: number }[] = [];
director.observeInterventions((event) => {
records.push(
event.count === undefined
? { id: event.id, class: event.class }
: { id: event.id, class: event.class, count: event.count },
);
});

await director.decide(inferenceDone(["failed-call"]), state, caps);
await director.decide(toolDone("failed-call", true), state, caps);
expect(records).toEqual([{ id: "tool-failure-recovery", class: "nudge" }]);
});

test("a later failure burst writes a new coalesced record instead of stacking", async () => {
const director = new SubAgentDirector("system", [], undefined, 30);
const caps = capabilities();
const records: { id: string; count?: number }[] = [];
director.observeInterventions((event) => {
records.push(
event.count === undefined
? { id: event.id }
: { id: event.id, count: event.count },
);
});

await director.decide(inferenceDone(["fail-a", "fail-b"]), state, caps);
await director.decide(toolDone("fail-a", true), state, caps);
await director.decide(toolDone("fail-b", true), state, caps);
expect(records).toEqual([{ id: "tool-failure-recovery", count: 2 }]);

await director.decide(inferenceDone(["fail-c"]), state, caps);
await director.decide(toolDone("fail-c", true), state, caps);
expect(records).toEqual([
{ id: "tool-failure-recovery", count: 2 },
{ id: "tool-failure-recovery" },
]);
});

test("successful tool result has no ephemeral recovery turn", async () => {
const director = new SubAgentDirector("system", [], undefined, 30);
const caps = capabilities();
Expand Down Expand Up @@ -221,6 +292,42 @@ describe("SubAgentDirector tool failure recovery", () => {
expect(ephemeralTexts(infer)).toBeUndefined();
});

test("compaction defers the coalesced recovery record until continuation infer", async () => {
const director = new SubAgentDirector(
"system",
[],
() => undefined,
30,
);
const caps = capabilities();
const records: { id: string; count?: number }[] = [];
director.observeInterventions((event) => {
records.push(
event.count === undefined
? { id: event.id }
: { id: event.id, count: event.count },
);
});

await director.decide(
inferenceDone(["fail-a", "fail-b"], 999_999),
longState,
caps,
);
await director.decide(toolDone("fail-a", true), longState, caps);
const compact = actions(
await director.decide(toolDone("fail-b", true), longState, caps),
);
expect(compact.some((action) => action.type === "infer")).toBe(false);
expect(records).toEqual([]);

const resumed = inferAction(
await director.decide(messageReceived(""), longState, caps),
);
expect(ephemeralTexts(resumed)?.[0]).toContain("A tool call failed");
expect(records).toEqual([{ id: "tool-failure-recovery", count: 2 }]);
});

test("retains recovery through compaction and consumes it once on continuation infer", async () => {
let continuations = 0;
const director = new SubAgentDirector(
Expand Down
28 changes: 22 additions & 6 deletions src/subagent/nudge-director.ts
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,12 @@ export class SubAgentDirector extends DefaultDirector {
private lastActivityAt: number;
private consecutiveStalls = 0;
private lastAssistantText = "";
// Consecutive failed tool.done events that armed the single pending
// recovery nudge. The jsonl sink is append-only, so the director holds
// the count until applyPendingNudge actually attaches the nudge, then
// writes one tool-failure-recovery row. Same (id + state) burst → one
// record; a missing count still means 1.
private pendingToolFailureRecoveries = 0;
// Every stop and nudge is recorded with its measured value beside its
// threshold, so a later threshold change can cite data instead of judgment
//. Defaults to a no-op: logging is diagnostic, never required.
Expand Down Expand Up @@ -361,13 +367,10 @@ export class SubAgentDirector extends DefaultDirector {
this.lastActivityAt = this.now();
this.consecutiveStalls = 0;
if (event.result.isError === true) {
// Failed-tool recovery guidance.
// Failed-tool recovery guidance. Text stays a single pending slot;
// the audit count is the only thing that accumulates.
this.pendingNudgeText = TOOL_FAILURE_RECOVERY_NUDGE;
this.interventions({
id: "tool-failure-recovery",
class: "nudge",
state: this.interventionState(),
});
this.pendingToolFailureRecoveries += 1;
}
}
const base = await super.decide(event, state, capabilities);
Expand Down Expand Up @@ -465,6 +468,19 @@ export class SubAgentDirector extends DefaultDirector {
const text = this.pendingNudgeText;
this.pendingNudgeText = null;
this.lastConsumedNudgeText = text;
if (
text === TOOL_FAILURE_RECOVERY_NUDGE &&
this.pendingToolFailureRecoveries > 0
) {
const count = this.pendingToolFailureRecoveries;
this.pendingToolFailureRecoveries = 0;
this.interventions({
id: "tool-failure-recovery",
class: "nudge",
state: this.interventionState(),
...(count > 1 ? { count } : {}),
});
}
const existing = actions[inferIndex] as Extract<
ReactorAction,
{ type: "infer" }
Expand Down
Loading