Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions scripts/intervention-forensics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -151,15 +151,19 @@ for (const file of files) {
bucket = emptyBucket();
buckets.set(key, bucket);
}
bucket.count++;
const occurrence = record.count ?? 1;
bucket.count += occurrence;
const family = record.family ?? record.model ?? "unknown";
bucket.byFamily.set(family, (bucket.byFamily.get(family) ?? 0) + 1);
bucket.byFamily.set(
family,
(bucket.byFamily.get(family) ?? 0) + occurrence,
);
const model = record.model ?? "unknown";
bucket.byModel.set(model, (bucket.byModel.get(model) ?? 0) + 1);
bucket.byModel.set(model, (bucket.byModel.get(model) ?? 0) + occurrence);
if (record.class === "stop" || record.class === "nudge") {
interventionsByModel.set(
model,
(interventionsByModel.get(model) ?? 0) + 1,
(interventionsByModel.get(model) ?? 0) + occurrence,
);
}
if (record.measurement !== undefined) {
Expand Down
10 changes: 10 additions & 0 deletions src/subagent/intervention-log.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,16 @@ describe("intervention log", () => {
expect(records.map((r) => r.id)).toEqual(["report-forced", "turn-budget"]);
});

test("preserves an optional coalesced count on the record", async () => {
const dir = await mkdtemp(join(tmpdir(), "intervention-log-"));
const sink = createInterventionLog(dir, { role: "leaf" });
sink({ id: "tool-failure-recovery", class: "nudge", count: 3 });
await flush();

const [record] = await readRecords(dir);
expect(record?.count).toBe(3);
});

test("a write failure never throws into the caller", async () => {
const sink = createInterventionLog(
join(tmpdir(), "intervention-log-missing-dir-xyz"),
Expand Down
6 changes: 6 additions & 0 deletions src/subagent/intervention-log.ts
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,12 @@ export interface InterventionRecord {
};
/** Free-form specifics, kept short (a looped window, a refused fingerprint). */
detail?: string;
/**
* How many consecutive same-trigger audits this record represents. Present when
* the director coalesced a burst (e.g. several failed tool.done events before
* the pending recovery nudge was consumed) into one flush. Absent means one.
*/
count?: number;
}

/** Fields every record from one run shares, supplied once at construction. */
Expand Down
77 changes: 77 additions & 0 deletions src/subagent/nudge-director.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,83 @@ describe("SubAgentDirector tool failure recovery", () => {
expect(texts?.[0]).toContain("report the blocker");
});

test("coalesces consecutive failed tool audits into one counted intervention", async () => {
const director = new SubAgentDirector("system", [], undefined, 30);
const caps = capabilities();
const records: { id: string; count?: number }[] = [];
director.observeInterventions((event) => {
records.push(
event.count === undefined
? { id: event.id }
: { id: event.id, count: event.count },
);
});

await director.decide(
inferenceDone(["fail-a", "fail-b", "ok-c"]),
state,
caps,
);
await director.decide(toolDone("fail-a", true), state, caps);
expect(records).toEqual([]);
await director.decide(toolDone("fail-b", true), state, caps);
expect(records).toEqual([]);

const texts = ephemeralTexts(
inferAction(await director.decide(toolDone("ok-c"), state, caps)),
);
expect(texts).toHaveLength(1);
expect(texts?.[0]).toContain("A tool call failed");
expect(records).toEqual([{ id: "tool-failure-recovery", count: 2 }]);
});

test("a single failed tool audit omits the count field", async () => {
const director = new SubAgentDirector("system", [], undefined, 30);
const caps = capabilities();
const records: { id: string; count: number | null }[] = [];
director.observeInterventions((event) => {
records.push({ id: event.id, count: event.count ?? null });
});

await director.decide(inferenceDone(["fail-a"]), state, caps);
await director.decide(toolDone("fail-a", true), state, caps);
await director.decide(inferenceDoneText(REPORT_ENVELOPE), state, caps);

expect(records).toEqual([{ id: "tool-failure-recovery", count: null }]);
});

test("flushes an undelivered recovery burst when the run goes terminal", async () => {
const director = new SubAgentDirector("system", [], undefined, 30);
const caps = capabilities();
const records: { id: string; count?: number }[] = [];
director.observeInterventions((event) => {
records.push(
event.count === undefined
? { id: event.id }
: { id: event.id, count: event.count },
);
});

// ok-c stays pending so the armed recovery nudge never reaches an infer.
await director.decide(
inferenceDone(["fail-a", "fail-b", "ok-c"]),
state,
caps,
);
await director.decide(toolDone("fail-a", true), state, caps);
await director.decide(toolDone("fail-b", true), state, caps);
expect(records).toEqual([]);

const result = actions(
await director.decide(inferenceDoneText(REPORT_ENVELOPE), state, caps),
);
expect(result).toContainEqual({
type: "checkpoint",
message: "subagent-complete",
});
expect(records).toEqual([{ id: "tool-failure-recovery", count: 2 }]);
});

test("successful tool result has no ephemeral recovery turn", async () => {
const director = new SubAgentDirector("system", [], undefined, 30);
const caps = capabilities();
Expand Down
36 changes: 30 additions & 6 deletions src/subagent/nudge-director.ts
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,10 @@ export class SubAgentDirector extends DefaultDirector {
// overflow compact (interceptOverflow re-arms from lastConsumedNudgeText if
// the infer that consumed pending never completed).
private pendingNudgeText: string | null = null;
// How many consecutive failed tool.done audits are waiting to be flushed as
// one tool-failure-recovery intervention when applyPendingNudge consumes the
// pending recovery nudge. Coalesces the audit trail without changing nudge text.
private pendingToolFailureRecoveryCount = 0;
// The text applyPendingNudge last attached to a returned infer. Overflow of
// that infer means the model never saw it, so interceptOverflow re-arms
// pending from this when pending is still null. Cleared on a successful
Expand Down Expand Up @@ -321,6 +325,7 @@ export class SubAgentDirector extends DefaultDirector {

if (stop === "complete") {
this.reportReplied = true;
this.flushToolFailureRecoveryAudit();
const terminal: ReactorAction[] = [
capabilities.checkpoint("subagent-complete"),
capabilities.reply(lastText(content)),
Expand Down Expand Up @@ -380,6 +385,7 @@ export class SubAgentDirector extends DefaultDirector {
});
this.onForcedStop("incomplete-report");
this.reportReplied = true;
this.flushToolFailureRecoveryAudit();
const terminal: ReactorAction[] = [
capabilities.checkpoint("subagent-incomplete-report"),
capabilities.reply(
Expand All @@ -402,13 +408,10 @@ export class SubAgentDirector extends DefaultDirector {
this.lastActivityAt = this.now();
this.consecutiveStalls = 0;
if (event.result.isError === true) {
// Failed-tool recovery guidance.
// Failed-tool recovery guidance. Arm once; coalesce consecutive failure
// audits until applyPendingNudge flushes a single counted record.
this.pendingNudgeText = TOOL_FAILURE_RECOVERY_NUDGE;
this.interventions({
id: "tool-failure-recovery",
class: "nudge",
state: this.interventionState(),
});
this.pendingToolFailureRecoveryCount += 1;
}
}
const base = await super.decide(event, state, capabilities);
Expand Down Expand Up @@ -479,6 +482,7 @@ export class SubAgentDirector extends DefaultDirector {
});
this.onForcedStop("stalled");
this.reportReplied = true;
this.flushToolFailureRecoveryAudit();
const terminal: ReactorAction[] = [
capabilities.checkpoint("subagent-stalled"),
capabilities.reply(
Expand All @@ -491,6 +495,25 @@ export class SubAgentDirector extends DefaultDirector {
return terminal;
}

/**
* Write the coalesced tool-failure-recovery audit once the burst ends —
* when the armed nudge lands on an infer, or when the run goes terminal
* (complete / forced stop / stalled) with the nudge still undelivered.
* Without the terminal-path flush a burst that is never followed by an
* infer would vanish from the audit trail entirely.
*/
private flushToolFailureRecoveryAudit(): void {
if (this.pendingToolFailureRecoveryCount === 0) return;
const count = this.pendingToolFailureRecoveryCount;
this.pendingToolFailureRecoveryCount = 0;
this.interventions({
id: "tool-failure-recovery",
class: "nudge",
...(count > 1 ? { count } : {}),
state: this.interventionState(),
});
}

/**
* Rewrite the infer action in a fall-through actions batch to carry the
* armed nudge, once — this matches the infer after report-forced or
Expand All @@ -506,6 +529,7 @@ export class SubAgentDirector extends DefaultDirector {
const text = this.pendingNudgeText;
this.pendingNudgeText = null;
this.lastConsumedNudgeText = text;
this.flushToolFailureRecoveryAudit();
const existing = actions[inferIndex] as Extract<
ReactorAction,
{ type: "infer" }
Expand Down
Loading