diff --git a/scripts/intervention-forensics.ts b/scripts/intervention-forensics.ts index 314da1617..de9742ef5 100644 --- a/scripts/intervention-forensics.ts +++ b/scripts/intervention-forensics.ts @@ -151,15 +151,19 @@ for (const file of files) { bucket = emptyBucket(); buckets.set(key, bucket); } - bucket.count++; + const occurrence = record.count ?? 1; + bucket.count += occurrence; const family = record.family ?? record.model ?? "unknown"; - bucket.byFamily.set(family, (bucket.byFamily.get(family) ?? 0) + 1); + bucket.byFamily.set( + family, + (bucket.byFamily.get(family) ?? 0) + occurrence, + ); const model = record.model ?? "unknown"; - bucket.byModel.set(model, (bucket.byModel.get(model) ?? 0) + 1); + bucket.byModel.set(model, (bucket.byModel.get(model) ?? 0) + occurrence); if (record.class === "stop" || record.class === "nudge") { interventionsByModel.set( model, - (interventionsByModel.get(model) ?? 0) + 1, + (interventionsByModel.get(model) ?? 0) + occurrence, ); } if (record.measurement !== undefined) { diff --git a/src/subagent/intervention-log.test.ts b/src/subagent/intervention-log.test.ts index cf17cc6a7..f99e25f34 100644 --- a/src/subagent/intervention-log.test.ts +++ b/src/subagent/intervention-log.test.ts @@ -76,6 +76,16 @@ describe("intervention log", () => { expect(records.map((r) => r.id)).toEqual(["report-forced", "turn-budget"]); }); + test("preserves an optional coalesced count on the record", async () => { + const dir = await mkdtemp(join(tmpdir(), "intervention-log-")); + const sink = createInterventionLog(dir, { role: "leaf" }); + sink({ id: "tool-failure-recovery", class: "nudge", count: 3 }); + await flush(); + + const [record] = await readRecords(dir); + expect(record?.count).toBe(3); + }); + test("a write failure never throws into the caller", async () => { const sink = createInterventionLog( join(tmpdir(), "intervention-log-missing-dir-xyz"), diff --git a/src/subagent/intervention-log.ts b/src/subagent/intervention-log.ts index d15133bf5..6d4d36668 100644 --- a/src/subagent/intervention-log.ts +++ b/src/subagent/intervention-log.ts @@ -90,6 +90,12 @@ export interface InterventionRecord { }; /** Free-form specifics, kept short (a looped window, a refused fingerprint). */ detail?: string; + /** + * How many consecutive same-trigger audits this record represents. Present when + * the director coalesced a burst (e.g. several failed tool.done events before + * the pending recovery nudge was consumed) into one flush. Absent means one. + */ + count?: number; } /** Fields every record from one run shares, supplied once at construction. */ diff --git a/src/subagent/nudge-director.test.ts b/src/subagent/nudge-director.test.ts index 19dfb2720..9443a4462 100644 --- a/src/subagent/nudge-director.test.ts +++ b/src/subagent/nudge-director.test.ts @@ -179,6 +179,83 @@ describe("SubAgentDirector tool failure recovery", () => { expect(texts?.[0]).toContain("report the blocker"); }); + test("coalesces consecutive failed tool audits into one counted intervention", async () => { + const director = new SubAgentDirector("system", [], undefined, 30); + const caps = capabilities(); + const records: { id: string; count?: number }[] = []; + director.observeInterventions((event) => { + records.push( + event.count === undefined + ? { id: event.id } + : { id: event.id, count: event.count }, + ); + }); + + await director.decide( + inferenceDone(["fail-a", "fail-b", "ok-c"]), + state, + caps, + ); + await director.decide(toolDone("fail-a", true), state, caps); + expect(records).toEqual([]); + await director.decide(toolDone("fail-b", true), state, caps); + expect(records).toEqual([]); + + const texts = ephemeralTexts( + inferAction(await director.decide(toolDone("ok-c"), state, caps)), + ); + expect(texts).toHaveLength(1); + expect(texts?.[0]).toContain("A tool call failed"); + expect(records).toEqual([{ id: "tool-failure-recovery", count: 2 }]); + }); + + test("a single failed tool audit omits the count field", async () => { + const director = new SubAgentDirector("system", [], undefined, 30); + const caps = capabilities(); + const records: { id: string; count: number | null }[] = []; + director.observeInterventions((event) => { + records.push({ id: event.id, count: event.count ?? null }); + }); + + await director.decide(inferenceDone(["fail-a"]), state, caps); + await director.decide(toolDone("fail-a", true), state, caps); + await director.decide(inferenceDoneText(REPORT_ENVELOPE), state, caps); + + expect(records).toEqual([{ id: "tool-failure-recovery", count: null }]); + }); + + test("flushes an undelivered recovery burst when the run goes terminal", async () => { + const director = new SubAgentDirector("system", [], undefined, 30); + const caps = capabilities(); + const records: { id: string; count?: number }[] = []; + director.observeInterventions((event) => { + records.push( + event.count === undefined + ? { id: event.id } + : { id: event.id, count: event.count }, + ); + }); + + // ok-c stays pending so the armed recovery nudge never reaches an infer. + await director.decide( + inferenceDone(["fail-a", "fail-b", "ok-c"]), + state, + caps, + ); + await director.decide(toolDone("fail-a", true), state, caps); + await director.decide(toolDone("fail-b", true), state, caps); + expect(records).toEqual([]); + + const result = actions( + await director.decide(inferenceDoneText(REPORT_ENVELOPE), state, caps), + ); + expect(result).toContainEqual({ + type: "checkpoint", + message: "subagent-complete", + }); + expect(records).toEqual([{ id: "tool-failure-recovery", count: 2 }]); + }); + test("successful tool result has no ephemeral recovery turn", async () => { const director = new SubAgentDirector("system", [], undefined, 30); const caps = capabilities(); diff --git a/src/subagent/nudge-director.ts b/src/subagent/nudge-director.ts index 3e9470f1b..c3cb9891a 100644 --- a/src/subagent/nudge-director.ts +++ b/src/subagent/nudge-director.ts @@ -132,6 +132,10 @@ export class SubAgentDirector extends DefaultDirector { // overflow compact (interceptOverflow re-arms from lastConsumedNudgeText if // the infer that consumed pending never completed). private pendingNudgeText: string | null = null; + // How many consecutive failed tool.done audits are waiting to be flushed as + // one tool-failure-recovery intervention when applyPendingNudge consumes the + // pending recovery nudge. Coalesces the audit trail without changing nudge text. + private pendingToolFailureRecoveryCount = 0; // The text applyPendingNudge last attached to a returned infer. Overflow of // that infer means the model never saw it, so interceptOverflow re-arms // pending from this when pending is still null. Cleared on a successful @@ -321,6 +325,7 @@ export class SubAgentDirector extends DefaultDirector { if (stop === "complete") { this.reportReplied = true; + this.flushToolFailureRecoveryAudit(); const terminal: ReactorAction[] = [ capabilities.checkpoint("subagent-complete"), capabilities.reply(lastText(content)), @@ -380,6 +385,7 @@ export class SubAgentDirector extends DefaultDirector { }); this.onForcedStop("incomplete-report"); this.reportReplied = true; + this.flushToolFailureRecoveryAudit(); const terminal: ReactorAction[] = [ capabilities.checkpoint("subagent-incomplete-report"), capabilities.reply( @@ -402,13 +408,10 @@ export class SubAgentDirector extends DefaultDirector { this.lastActivityAt = this.now(); this.consecutiveStalls = 0; if (event.result.isError === true) { - // Failed-tool recovery guidance. + // Failed-tool recovery guidance. Arm once; coalesce consecutive failure + // audits until applyPendingNudge flushes a single counted record. this.pendingNudgeText = TOOL_FAILURE_RECOVERY_NUDGE; - this.interventions({ - id: "tool-failure-recovery", - class: "nudge", - state: this.interventionState(), - }); + this.pendingToolFailureRecoveryCount += 1; } } const base = await super.decide(event, state, capabilities); @@ -479,6 +482,7 @@ export class SubAgentDirector extends DefaultDirector { }); this.onForcedStop("stalled"); this.reportReplied = true; + this.flushToolFailureRecoveryAudit(); const terminal: ReactorAction[] = [ capabilities.checkpoint("subagent-stalled"), capabilities.reply( @@ -491,6 +495,25 @@ export class SubAgentDirector extends DefaultDirector { return terminal; } + /** + * Write the coalesced tool-failure-recovery audit once the burst ends — + * when the armed nudge lands on an infer, or when the run goes terminal + * (complete / forced stop / stalled) with the nudge still undelivered. + * Without the terminal-path flush a burst that is never followed by an + * infer would vanish from the audit trail entirely. + */ + private flushToolFailureRecoveryAudit(): void { + if (this.pendingToolFailureRecoveryCount === 0) return; + const count = this.pendingToolFailureRecoveryCount; + this.pendingToolFailureRecoveryCount = 0; + this.interventions({ + id: "tool-failure-recovery", + class: "nudge", + ...(count > 1 ? { count } : {}), + state: this.interventionState(), + }); + } + /** * Rewrite the infer action in a fall-through actions batch to carry the * armed nudge, once — this matches the infer after report-forced or @@ -506,6 +529,7 @@ export class SubAgentDirector extends DefaultDirector { const text = this.pendingNudgeText; this.pendingNudgeText = null; this.lastConsumedNudgeText = text; + this.flushToolFailureRecoveryAudit(); const existing = actions[inferIndex] as Extract< ReactorAction, { type: "infer" }