Skip to content

Commit 1a2925f

Browse files
committed
Coalesce consecutive tool-failure recovery audits into one count
Several failed tool.done events before the pending recovery nudge is consumed were each writing a separate intervention line. Keep the recovery nudge text and arming behavior the same, but count the burst in director memory and flush a single record with count when the nudge is applied. Forensics treats missing count as one.
1 parent a7919d0 commit 1a2925f

5 files changed

Lines changed: 70 additions & 10 deletions

File tree

scripts/intervention-forensics.ts

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -151,15 +151,19 @@ for (const file of files) {
151151
bucket = emptyBucket();
152152
buckets.set(key, bucket);
153153
}
154-
bucket.count++;
154+
const occurrence = record.count ?? 1;
155+
bucket.count += occurrence;
155156
const family = record.family ?? record.model ?? "unknown";
156-
bucket.byFamily.set(family, (bucket.byFamily.get(family) ?? 0) + 1);
157+
bucket.byFamily.set(
158+
family,
159+
(bucket.byFamily.get(family) ?? 0) + occurrence,
160+
);
157161
const model = record.model ?? "unknown";
158-
bucket.byModel.set(model, (bucket.byModel.get(model) ?? 0) + 1);
162+
bucket.byModel.set(model, (bucket.byModel.get(model) ?? 0) + occurrence);
159163
if (record.class === "stop" || record.class === "nudge") {
160164
interventionsByModel.set(
161165
model,
162-
(interventionsByModel.get(model) ?? 0) + 1,
166+
(interventionsByModel.get(model) ?? 0) + occurrence,
163167
);
164168
}
165169
if (record.measurement !== undefined) {

src/subagent/intervention-log.test.ts

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,16 @@ describe("intervention log", () => {
7676
expect(records.map((r) => r.id)).toEqual(["report-forced", "turn-budget"]);
7777
});
7878

79+
test("preserves an optional coalesced count on the record", async () => {
80+
const dir = await mkdtemp(join(tmpdir(), "intervention-log-"));
81+
const sink = createInterventionLog(dir, { role: "leaf" });
82+
sink({ id: "tool-failure-recovery", class: "nudge", count: 3 });
83+
await flush();
84+
85+
const [record] = await readRecords(dir);
86+
expect(record?.count).toBe(3);
87+
});
88+
7989
test("a write failure never throws into the caller", async () => {
8090
const sink = createInterventionLog(
8191
join(tmpdir(), "intervention-log-missing-dir-xyz"),

src/subagent/intervention-log.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,12 @@ export interface InterventionRecord {
9090
};
9191
/** Free-form specifics, kept short (a looped window, a refused fingerprint). */
9292
detail?: string;
93+
/**
94+
* How many consecutive same-trigger audits this record represents. Present when
95+
* the director coalesced a burst (e.g. several failed tool.done events before
96+
* the pending recovery nudge was consumed) into one flush. Absent means one.
97+
*/
98+
count?: number;
9399
}
94100

95101
/** Fields every record from one run shares, supplied once at construction. */

src/subagent/nudge-director.test.ts

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,36 @@ describe("SubAgentDirector tool failure recovery", () => {
179179
expect(texts?.[0]).toContain("report the blocker");
180180
});
181181

182+
test("coalesces consecutive failed tool audits into one counted intervention", async () => {
183+
const director = new SubAgentDirector("system", [], undefined, 30);
184+
const caps = capabilities();
185+
const records: { id: string; count?: number }[] = [];
186+
director.observeInterventions((event) => {
187+
records.push(
188+
event.count === undefined
189+
? { id: event.id }
190+
: { id: event.id, count: event.count },
191+
);
192+
});
193+
194+
await director.decide(
195+
inferenceDone(["fail-a", "fail-b", "ok-c"]),
196+
state,
197+
caps,
198+
);
199+
await director.decide(toolDone("fail-a", true), state, caps);
200+
expect(records).toEqual([]);
201+
await director.decide(toolDone("fail-b", true), state, caps);
202+
expect(records).toEqual([]);
203+
204+
const texts = ephemeralTexts(
205+
inferAction(await director.decide(toolDone("ok-c"), state, caps)),
206+
);
207+
expect(texts).toHaveLength(1);
208+
expect(texts?.[0]).toContain("A tool call failed");
209+
expect(records).toEqual([{ id: "tool-failure-recovery", count: 2 }]);
210+
});
211+
182212
test("successful tool result has no ephemeral recovery turn", async () => {
183213
const director = new SubAgentDirector("system", [], undefined, 30);
184214
const caps = capabilities();

src/subagent/nudge-director.ts

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,10 @@ export class SubAgentDirector extends DefaultDirector {
132132
// overflow compact (interceptOverflow re-arms from lastConsumedNudgeText if
133133
// the infer that consumed pending never completed).
134134
private pendingNudgeText: string | null = null;
135+
// How many consecutive failed tool.done audits are waiting to be flushed as
136+
// one tool-failure-recovery intervention when applyPendingNudge consumes the
137+
// pending recovery nudge. Coalesces the audit trail without changing nudge text.
138+
private pendingToolFailureRecoveryCount = 0;
135139
// The text applyPendingNudge last attached to a returned infer. Overflow of
136140
// that infer means the model never saw it, so interceptOverflow re-arms
137141
// pending from this when pending is still null. Cleared on a successful
@@ -402,13 +406,10 @@ export class SubAgentDirector extends DefaultDirector {
402406
this.lastActivityAt = this.now();
403407
this.consecutiveStalls = 0;
404408
if (event.result.isError === true) {
405-
// Failed-tool recovery guidance.
409+
// Failed-tool recovery guidance. Arm once; coalesce consecutive failure
410+
// audits until applyPendingNudge flushes a single counted record.
406411
this.pendingNudgeText = TOOL_FAILURE_RECOVERY_NUDGE;
407-
this.interventions({
408-
id: "tool-failure-recovery",
409-
class: "nudge",
410-
state: this.interventionState(),
411-
});
412+
this.pendingToolFailureRecoveryCount += 1;
412413
}
413414
}
414415
const base = await super.decide(event, state, capabilities);
@@ -506,6 +507,15 @@ export class SubAgentDirector extends DefaultDirector {
506507
const text = this.pendingNudgeText;
507508
this.pendingNudgeText = null;
508509
this.lastConsumedNudgeText = text;
510+
if (this.pendingToolFailureRecoveryCount > 0) {
511+
this.interventions({
512+
id: "tool-failure-recovery",
513+
class: "nudge",
514+
count: this.pendingToolFailureRecoveryCount,
515+
state: this.interventionState(),
516+
});
517+
this.pendingToolFailureRecoveryCount = 0;
518+
}
509519
const existing = actions[inferIndex] as Extract<
510520
ReactorAction,
511521
{ type: "infer" }

0 commit comments

Comments
 (0)