From 3571dc213450bca81b79e25674ba4c6b7ad56170 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 10 Sep 2026 18:01:13 -0700 Subject: [PATCH] Coalesce consecutive tool-failure recovery audits into one count The audit sink is append-only, so one failed-tool recovery per result looked like distinct recoveries. Hold a burst count until the single pending nudge is applied. Forensics treat a missing count as one. --- scripts/intervention-forensics.ts | 12 ++-- src/subagent/intervention-log.ts | 5 ++ src/subagent/nudge-director.test.ts | 107 ++++++++++++++++++++++++++++ src/subagent/nudge-director.ts | 28 ++++++-- 4 files changed, 142 insertions(+), 10 deletions(-) diff --git a/scripts/intervention-forensics.ts b/scripts/intervention-forensics.ts index 314da1617..f75a060cb 100644 --- a/scripts/intervention-forensics.ts +++ b/scripts/intervention-forensics.ts @@ -151,15 +151,19 @@ for (const file of files) { bucket = emptyBucket(); buckets.set(key, bucket); } - bucket.count++; + const occurrences = record.count ?? 1; + bucket.count += occurrences; const family = record.family ?? record.model ?? "unknown"; - bucket.byFamily.set(family, (bucket.byFamily.get(family) ?? 0) + 1); + bucket.byFamily.set( + family, + (bucket.byFamily.get(family) ?? 0) + occurrences, + ); const model = record.model ?? "unknown"; - bucket.byModel.set(model, (bucket.byModel.get(model) ?? 0) + 1); + bucket.byModel.set(model, (bucket.byModel.get(model) ?? 0) + occurrences); if (record.class === "stop" || record.class === "nudge") { interventionsByModel.set( model, - (interventionsByModel.get(model) ?? 0) + 1, + (interventionsByModel.get(model) ?? 0) + occurrences, ); } if (record.measurement !== undefined) { diff --git a/src/subagent/intervention-log.ts b/src/subagent/intervention-log.ts index d15133bf5..0538f1697 100644 --- a/src/subagent/intervention-log.ts +++ b/src/subagent/intervention-log.ts @@ -90,6 +90,11 @@ export interface InterventionRecord { }; /** Free-form specifics, kept short (a looped window, a refused fingerprint). */ detail?: string; + /** + * How many consecutive same-(id, state) firings this row stands for. + * Absent means 1 — older records and one-shot interventions omit it. + */ + count?: number; } /** Fields every record from one run shares, supplied once at construction. */ diff --git a/src/subagent/nudge-director.test.ts b/src/subagent/nudge-director.test.ts index 848553c62..e8c9be74d 100644 --- a/src/subagent/nudge-director.test.ts +++ b/src/subagent/nudge-director.test.ts @@ -172,6 +172,77 @@ describe("SubAgentDirector tool failure recovery", () => { expect(texts?.[0]).toContain("report the blocker"); }); + test("N consecutive failed tools in one burst produce one coalesced intervention record", async () => { + const director = new SubAgentDirector("system", [], undefined, 30); + const caps = capabilities(); + const records: { id: string; count?: number }[] = []; + director.observeInterventions((event) => { + records.push( + event.count === undefined + ? { id: event.id } + : { id: event.id, count: event.count }, + ); + }); + + await director.decide( + inferenceDone(["fail-a", "fail-b", "fail-c"]), + state, + caps, + ); + await director.decide(toolDone("fail-a", true), state, caps); + await director.decide(toolDone("fail-b", true), state, caps); + expect(records).toEqual([]); + + const texts = ephemeralTexts( + inferAction(await director.decide(toolDone("fail-c", true), state, caps)), + ); + expect(texts).toHaveLength(1); + expect(texts?.[0]).toContain("A tool call failed"); + expect(records).toEqual([{ id: "tool-failure-recovery", count: 3 }]); + }); + + test("a single failed tool writes one recovery record with count omitted", async () => { + const director = new SubAgentDirector("system", [], undefined, 30); + const caps = capabilities(); + const records: { id: string; class?: string; count?: number }[] = []; + director.observeInterventions((event) => { + records.push( + event.count === undefined + ? { id: event.id, class: event.class } + : { id: event.id, class: event.class, count: event.count }, + ); + }); + + await director.decide(inferenceDone(["failed-call"]), state, caps); + await director.decide(toolDone("failed-call", true), state, caps); + expect(records).toEqual([{ id: "tool-failure-recovery", class: "nudge" }]); + }); + + test("a later failure burst writes a new coalesced record instead of stacking", async () => { + const director = new SubAgentDirector("system", [], undefined, 30); + const caps = capabilities(); + const records: { id: string; count?: number }[] = []; + director.observeInterventions((event) => { + records.push( + event.count === undefined + ? { id: event.id } + : { id: event.id, count: event.count }, + ); + }); + + await director.decide(inferenceDone(["fail-a", "fail-b"]), state, caps); + await director.decide(toolDone("fail-a", true), state, caps); + await director.decide(toolDone("fail-b", true), state, caps); + expect(records).toEqual([{ id: "tool-failure-recovery", count: 2 }]); + + await director.decide(inferenceDone(["fail-c"]), state, caps); + await director.decide(toolDone("fail-c", true), state, caps); + expect(records).toEqual([ + { id: "tool-failure-recovery", count: 2 }, + { id: "tool-failure-recovery" }, + ]); + }); + test("successful tool result has no ephemeral recovery turn", async () => { const director = new SubAgentDirector("system", [], undefined, 30); const caps = capabilities(); @@ -221,6 +292,42 @@ describe("SubAgentDirector tool failure recovery", () => { expect(ephemeralTexts(infer)).toBeUndefined(); }); + test("compaction defers the coalesced recovery record until continuation infer", async () => { + const director = new SubAgentDirector( + "system", + [], + () => undefined, + 30, + ); + const caps = capabilities(); + const records: { id: string; count?: number }[] = []; + director.observeInterventions((event) => { + records.push( + event.count === undefined + ? { id: event.id } + : { id: event.id, count: event.count }, + ); + }); + + await director.decide( + inferenceDone(["fail-a", "fail-b"], 999_999), + longState, + caps, + ); + await director.decide(toolDone("fail-a", true), longState, caps); + const compact = actions( + await director.decide(toolDone("fail-b", true), longState, caps), + ); + expect(compact.some((action) => action.type === "infer")).toBe(false); + expect(records).toEqual([]); + + const resumed = inferAction( + await director.decide(messageReceived(""), longState, caps), + ); + expect(ephemeralTexts(resumed)?.[0]).toContain("A tool call failed"); + expect(records).toEqual([{ id: "tool-failure-recovery", count: 2 }]); + }); + test("retains recovery through compaction and consumes it once on continuation infer", async () => { let continuations = 0; const director = new SubAgentDirector( diff --git a/src/subagent/nudge-director.ts b/src/subagent/nudge-director.ts index fb1ec456e..72e22e9fd 100644 --- a/src/subagent/nudge-director.ts +++ b/src/subagent/nudge-director.ts @@ -152,6 +152,12 @@ export class SubAgentDirector extends DefaultDirector { private lastActivityAt: number; private consecutiveStalls = 0; private lastAssistantText = ""; + // Consecutive failed tool.done events that armed the single pending + // recovery nudge. The jsonl sink is append-only, so the director holds + // the count until applyPendingNudge actually attaches the nudge, then + // writes one tool-failure-recovery row. Same (id + state) burst → one + // record; a missing count still means 1. + private pendingToolFailureRecoveries = 0; // Every stop and nudge is recorded with its measured value beside its // threshold, so a later threshold change can cite data instead of judgment //. Defaults to a no-op: logging is diagnostic, never required. @@ -361,13 +367,10 @@ export class SubAgentDirector extends DefaultDirector { this.lastActivityAt = this.now(); this.consecutiveStalls = 0; if (event.result.isError === true) { - // Failed-tool recovery guidance. + // Failed-tool recovery guidance. Text stays a single pending slot; + // the audit count is the only thing that accumulates. this.pendingNudgeText = TOOL_FAILURE_RECOVERY_NUDGE; - this.interventions({ - id: "tool-failure-recovery", - class: "nudge", - state: this.interventionState(), - }); + this.pendingToolFailureRecoveries += 1; } } const base = await super.decide(event, state, capabilities); @@ -465,6 +468,19 @@ export class SubAgentDirector extends DefaultDirector { const text = this.pendingNudgeText; this.pendingNudgeText = null; this.lastConsumedNudgeText = text; + if ( + text === TOOL_FAILURE_RECOVERY_NUDGE && + this.pendingToolFailureRecoveries > 0 + ) { + const count = this.pendingToolFailureRecoveries; + this.pendingToolFailureRecoveries = 0; + this.interventions({ + id: "tool-failure-recovery", + class: "nudge", + state: this.interventionState(), + ...(count > 1 ? { count } : {}), + }); + } const existing = actions[inferIndex] as Extract< ReactorAction, { type: "infer" }