Skip to content

Commit 40da15d

Browse files
committed
Keep a second credential error from failing the session
A rebuilt agent reused seq 0 for the next same-category inference error, so commitErrors threw Duplicate error record and afterCheckpoint failed the run. Resume the durable error sequence on assembly and drop a colliding flush instead of failing the session.
1 parent 9f4de35 commit 40da15d

12 files changed

Lines changed: 317 additions & 2 deletions

File tree

src/session/assemble-runtime.test.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ function stubAuditStore(): AuditStore {
143143
commitAudit: async () => undefined,
144144
commitErrors: async () => undefined,
145145
loadAudit: async () => [],
146+
loadErrors: async () => [],
146147
};
147148
}
148149

src/session/optimized-context-store.test.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -800,6 +800,7 @@ describe("createSessionStores", () => {
800800
expect(typeof audit.commitAudit).toBe("function");
801801
expect(typeof audit.commitErrors).toBe("function");
802802
expect(typeof audit.loadAudit).toBe("function");
803+
expect(typeof audit.loadErrors).toBe("function");
803804
});
804805
});
805806

src/session/optimized-context-store.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -848,6 +848,7 @@ export async function createSessionStores(
848848
commitErrors: (records, signal) =>
849849
withResolvedDirLock(dir, () => base.commitErrors(records, signal)),
850850
loadAudit: (sessionId, signal) => base.loadAudit(sessionId, signal),
851+
loadErrors: (sessionId, signal) => base.loadErrors(sessionId, signal),
851852
};
852853

853854
return { storage: store, audit: store };

vendor/intx-agent/src/agent.ts

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -512,7 +512,12 @@ export async function createAgent<EnvReq extends BaseEnv>(
512512
// the next flush, so a fourth caller arriving after the follow-up
513513
// begins still observes a clean state and starts its own flush.
514514
const accumulatedErrors: ErrorRecord[] = [];
515+
// Resume from durable records so a rebuilt agent does not reuse seq 0
516+
// and collide with files the previous assembly already committed.
515517
let errorSeq = 0;
518+
for (const record of await auditStore.loadErrors(sessionId)) {
519+
if (record.seq >= errorSeq) errorSeq = record.seq + 1;
520+
}
516521
let flushInProgress: Promise<void> | undefined;
517522
let pendingFollowUp: Promise<void> | undefined;
518523

@@ -551,6 +556,16 @@ export async function createAgent<EnvReq extends BaseEnv>(
551556
try {
552557
await auditStore.commitErrors(batch);
553558
accumulatedErrors.splice(0, count);
559+
} catch (cause) {
560+
if (
561+
cause instanceof Error &&
562+
cause.message.startsWith("Duplicate error record:")
563+
) {
564+
logger.warn`duplicate error record already stored; dropping the colliding batch`;
565+
accumulatedErrors.splice(0, count);
566+
return;
567+
}
568+
throw cause;
554569
} finally {
555570
flushInProgress = undefined;
556571
}

vendor/intx-agent/src/audit-integration.test.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@ function makeRecordingAuditStore(): RecordingAuditStore {
7171
async loadAudit(_sessionId: string): Promise<AuditRecord[]> {
7272
return committedAudit.flat();
7373
},
74+
async loadErrors(_sessionId: string): Promise<ErrorRecord[]> {
75+
return committedErrors.flat();
76+
},
7477
getCommittedAudit() {
7578
return committedAudit;
7679
},

vendor/intx-agent/src/flush-errors.test.ts

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import { tmpdir } from "node:os";
1414
import { join } from "node:path";
1515
import { type } from "arktype";
1616

17+
import { createDefaultDependencies } from "@intx/inference/providers";
1718
import { createInboundMessage } from "@intx/mime";
1819
import { createIsogitStore } from "@intx/storage-isogit/node";
1920
import type { AuditRecord, ErrorRecord } from "@intx/types/audit";
@@ -59,6 +60,9 @@ function makeRecordingAuditStore(): RecordingAuditStore {
5960
async loadAudit(_sessionId: string): Promise<AuditRecord[]> {
6061
return [];
6162
},
63+
async loadErrors(_sessionId: string): Promise<ErrorRecord[]> {
64+
return committedErrors.flat();
65+
},
6266
getCommittedErrors() {
6367
return committedErrors;
6468
},
@@ -91,12 +95,37 @@ function makeFailFirstAuditStore(): FailingAuditStore {
9195
async loadAudit(_sessionId: string): Promise<AuditRecord[]> {
9296
return [];
9397
},
98+
async loadErrors(_sessionId: string): Promise<ErrorRecord[]> {
99+
return committedErrors.flat();
100+
},
94101
getCommittedErrors() {
95102
return committedErrors;
96103
},
97104
};
98105
}
99106

107+
function makeDuplicateErrorAuditStore(): FailingAuditStore {
108+
return {
109+
async commitAudit(_records: AuditRecord[]): Promise<void> {
110+
// No-op.
111+
},
112+
async commitErrors(records: ErrorRecord[]): Promise<void> {
113+
throw new Error(
114+
`Duplicate error record: ${records[0]?.sessionId ?? "session"}/00000000-credential_failure`,
115+
);
116+
},
117+
async loadAudit(_sessionId: string): Promise<AuditRecord[]> {
118+
return [];
119+
},
120+
async loadErrors(_sessionId: string): Promise<ErrorRecord[]> {
121+
return [];
122+
},
123+
getCommittedErrors() {
124+
return [];
125+
},
126+
};
127+
}
128+
100129
// Director factory that closes over a caller-supplied `decide` to drive
101130
// the reactor through targeted event shapes. The factory shape requires
102131
// a configSchema (arktype) and returns a ReactorDirector; this helper
@@ -153,6 +182,86 @@ async function waitForReactorDone(
153182
}
154183
}
155184

185+
const FORBIDDEN_DEPS = {
186+
...createDefaultDependencies(),
187+
fetch: async () =>
188+
new Response("Unauthorized", { status: 401, statusText: "Unauthorized" }),
189+
};
190+
191+
function credentialFailureDirectors(): BaseEnv["directors"] {
192+
return makeDirectorRegistry(
193+
async (
194+
event: ReactorInboundEvent,
195+
_state: ReactorState,
196+
caps: ReactorCapabilities,
197+
) => {
198+
if (event.type === "message.received") return caps.infer();
199+
if (event.type === "inference.error") {
200+
return [caps.checkpoint("after-error"), caps.done()];
201+
}
202+
return caps.done();
203+
},
204+
);
205+
}
206+
207+
function forbiddenAgentDef(id: string) {
208+
return defineAgent({
209+
id,
210+
systemPrompt: "test",
211+
tools: [],
212+
capabilities: [],
213+
inference: {
214+
sources: [
215+
{
216+
provider: UNREACHABLE_SOURCE.provider,
217+
model: UNREACHABLE_SOURCE.model,
218+
},
219+
],
220+
},
221+
});
222+
}
223+
224+
function duplicateFlushFailures(
225+
events: ReadonlyArray<{ type: string; data?: unknown }>,
226+
): ReadonlyArray<{ type: string; data?: unknown }> {
227+
return events.filter((event) => {
228+
if (event.type !== "reactor.error") return false;
229+
return JSON.stringify(event.data ?? {}).includes("Duplicate error record");
230+
});
231+
}
232+
233+
async function runForbiddenCycle(opts: {
234+
workdir: string;
235+
sessionId: string;
236+
agentId: string;
237+
}): Promise<{ events: Array<{ type: string; data?: unknown }> }> {
238+
const store = await createIsogitStore(opts.workdir);
239+
const env: BaseEnv = {
240+
sources: [UNREACHABLE_SOURCE],
241+
defaultSource: UNREACHABLE_SOURCE.id,
242+
storage: store,
243+
workdir: opts.workdir,
244+
audit: store,
245+
authorize: permissiveAuthorize(),
246+
directors: credentialFailureDirectors(),
247+
sessionId: opts.sessionId,
248+
deps: FORBIDDEN_DEPS,
249+
};
250+
const agent = await createAgent(forbiddenAgentDef(opts.agentId), env);
251+
const events: Array<{ type: string; data?: unknown }> = [];
252+
const stream = agent.stream();
253+
try {
254+
agent.deliver(inboundConversation());
255+
for await (const event of stream) {
256+
events.push(event);
257+
if (event.type === "reactor.done") break;
258+
}
259+
} finally {
260+
await agent.close();
261+
}
262+
return { events };
263+
}
264+
156265
describe("agent error flushing", () => {
157266
let workDir: string;
158267

@@ -470,4 +579,101 @@ describe("agent error flushing", () => {
470579
expect(batches.length).toBe(1);
471580
expect(batches[0]?.[0]?.source).toBe("reactor");
472581
});
582+
583+
test("two credential_failure errors in one session persist without failing the run", async () => {
584+
const sessionId = "session-credential-once";
585+
let inferenceErrors = 0;
586+
const store = await createIsogitStore(workDir);
587+
const env: BaseEnv = {
588+
sources: [UNREACHABLE_SOURCE],
589+
defaultSource: UNREACHABLE_SOURCE.id,
590+
storage: store,
591+
workdir: workDir,
592+
audit: store,
593+
authorize: permissiveAuthorize(),
594+
directors: makeDirectorRegistry(
595+
async (
596+
event: ReactorInboundEvent,
597+
_state: ReactorState,
598+
caps: ReactorCapabilities,
599+
) => {
600+
if (event.type === "message.received") return caps.infer();
601+
if (event.type === "inference.error") {
602+
inferenceErrors += 1;
603+
if (inferenceErrors === 1) {
604+
return [caps.checkpoint("after-first"), caps.infer()];
605+
}
606+
return [caps.checkpoint("after-second"), caps.done()];
607+
}
608+
return caps.done();
609+
},
610+
),
611+
sessionId,
612+
deps: FORBIDDEN_DEPS,
613+
};
614+
const agent = await createAgent(forbiddenAgentDef("cred-flush-once"), env);
615+
const events: Array<{ type: string; data?: unknown }> = [];
616+
const stream = agent.stream();
617+
try {
618+
agent.deliver(inboundConversation());
619+
for await (const event of stream) {
620+
events.push(event);
621+
if (event.type === "reactor.done") break;
622+
}
623+
} finally {
624+
await agent.close();
625+
}
626+
627+
expect(duplicateFlushFailures(events)).toEqual([]);
628+
const records = (await store.loadErrors(sessionId)).filter(
629+
(record) => record.category === "credential_failure",
630+
);
631+
expect(records).toHaveLength(2);
632+
expect(new Set(records.map((record) => record.seq)).size).toBe(2);
633+
});
634+
635+
test("two credential_failure errors persist across re-assembly without failing the session", async () => {
636+
const sessionId = "session-credential";
637+
const first = await runForbiddenCycle({
638+
workdir: workDir,
639+
sessionId,
640+
agentId: "cred-flush-1",
641+
});
642+
const second = await runForbiddenCycle({
643+
workdir: workDir,
644+
sessionId,
645+
agentId: "cred-flush-2",
646+
});
647+
648+
expect(duplicateFlushFailures(first.events)).toEqual([]);
649+
expect(duplicateFlushFailures(second.events)).toEqual([]);
650+
const store = await createIsogitStore(workDir);
651+
const records = (await store.loadErrors(sessionId)).filter(
652+
(record) => record.category === "credential_failure",
653+
);
654+
expect(records).toHaveLength(2);
655+
expect(new Set(records.map((record) => record.seq)).size).toBe(2);
656+
});
657+
658+
test("a duplicate error record from commitErrors does not fail the session", async () => {
659+
const audit = makeDuplicateErrorAuditStore();
660+
const directors = credentialFailureDirectors();
661+
const def = forbiddenAgentDef("cred-flush-duplicate");
662+
const env = await buildAgentEnv({ workdir: workDir, audit, directors });
663+
const agent = await createAgent(def, { ...env, deps: FORBIDDEN_DEPS });
664+
const events: Array<{ type: string; data?: unknown }> = [];
665+
const stream = agent.stream();
666+
try {
667+
agent.deliver(inboundConversation());
668+
for await (const event of stream) {
669+
events.push(event);
670+
if (event.type === "reactor.done") break;
671+
}
672+
} finally {
673+
await agent.close();
674+
}
675+
676+
expect(duplicateFlushFailures(events)).toEqual([]);
677+
expect(events.some((event) => event.type === "reactor.done")).toBe(true);
678+
});
473679
});

vendor/intx-agent/src/testing/audit-noop.test.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,11 @@ describe("noopAuditStore", () => {
1818
expect(await store.loadAudit("sess")).toEqual([]);
1919
});
2020

21+
test("loadErrors returns an empty array", async () => {
22+
const store = noopAuditStore();
23+
expect(await store.loadErrors("sess")).toEqual([]);
24+
});
25+
2126
test("each call returns a fresh object", () => {
2227
expect(noopAuditStore()).not.toBe(noopAuditStore());
2328
});

vendor/intx-agent/src/testing/audit-noop.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,5 +25,8 @@ export function noopAuditStore(): AuditStore {
2525
async loadAudit(_sessionId: string): Promise<AuditRecord[]> {
2626
return [];
2727
},
28+
async loadErrors(_sessionId: string): Promise<ErrorRecord[]> {
29+
return [];
30+
},
2831
};
2932
}

vendor/intx-inference/src/assembly.test.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,9 @@ function makeRecordingAuditStore(): AuditStore & {
198198
async commitErrors() {
199199
/* noop */
200200
},
201+
async loadErrors() {
202+
return [];
203+
},
201204
getCommitted() {
202205
return committed;
203206
},

vendor/intx-storage-isogit/src/store.test.ts

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -692,6 +692,38 @@ describe("error store", () => {
692692
),
693693
).toHaveLength(1);
694694
});
695+
696+
test("loadErrors round-trips records ordered by seq", async () => {
697+
const dir = await tempDir();
698+
const store = await createAuditStore(dir);
699+
const later = makeErrorRecord({ seq: 2, category: "retryable" });
700+
const earlier = makeErrorRecord({ seq: 1, category: "credential_failure" });
701+
702+
await store.commitErrors([later]);
703+
await store.commitErrors([earlier]);
704+
705+
expect(await store.loadErrors("session-1")).toEqual([earlier, later]);
706+
});
707+
708+
test("loadErrors returns empty array for nonexistent session", async () => {
709+
const dir = await tempDir();
710+
const store = await createAuditStore(dir);
711+
712+
expect(await store.loadErrors("no-such-session")).toEqual([]);
713+
});
714+
715+
test("rejects sessionId with path traversal on loadErrors", async () => {
716+
const dir = await tempDir();
717+
const store = await createAuditStore(dir);
718+
719+
let thrown: Error | undefined;
720+
try {
721+
await store.loadErrors("../escape");
722+
} catch (cause) {
723+
thrown = cause instanceof Error ? cause : new Error(String(cause));
724+
}
725+
expect(thrown?.message).toContain("unsafe characters");
726+
});
695727
});
696728

697729
describe("audit and error durability retries", () => {

0 commit comments

Comments
 (0)