Skip to content

Commit 1cfffe3

Browse files
TheGreatAxiosTheGreatAxios
authored
Merge pull request #711 from corbitsdev/cl-7119-hide-recovered-quota-and-session-expired-errors-from-the
Retract recovered inference errors on failover
2 parents 198afce + aa031e9 commit 1cfffe3

4 files changed

Lines changed: 439 additions & 23 deletions

File tree

‎src/tui/runtime-bridge.test.ts‎

Lines changed: 326 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -814,6 +814,332 @@ describe("committed inference retry", () => {
814814
});
815815
});
816816

817+
describe("same-turn failover after inference.error", () => {
818+
const errorRows = (shell: {
819+
streamLog: readonly { role: string; meta?: string; text: string }[];
820+
}) => shell.streamLog.filter((r) => r.meta === "error").map((r) => r.text);
821+
822+
test("a recovered quota error does not stay in the transcript", async () => {
823+
await withTestRenderer(
824+
async (h) => {
825+
const shell = createAppShell(h.renderer, {
826+
terminal: { columns: 80, rows: 24 },
827+
wireKeys: false,
828+
run: "idle",
829+
});
830+
const bridge = attachSessionBridge(shell, createRecordingPort());
831+
try {
832+
for (const event of [
833+
{ type: "inference.start", data: {} },
834+
{
835+
type: "inference.error",
836+
data: {
837+
error: {
838+
category: "quota_exhausted",
839+
message: "The usage limit has been reached",
840+
statusCode: 429,
841+
},
842+
},
843+
},
844+
{ type: "inference.start", data: {} },
845+
{ type: "inference.text.delta", data: { token: "recovered" } },
846+
{ type: "inference.done", data: {} },
847+
{ type: "reactor.done", data: {} },
848+
] as const) {
849+
bridge.handle(event);
850+
}
851+
852+
expect(errorRows(shell)).toEqual([]);
853+
expect(shell.streamLog.map((r) => r.text).join("\n")).toContain("recovered");
854+
} finally {
855+
bridge.dispose();
856+
shell.dispose();
857+
}
858+
},
859+
{ width: 80, height: 24 },
860+
);
861+
});
862+
863+
test("a recovered credential error does not stay in the transcript", async () => {
864+
await withTestRenderer(
865+
async (h) => {
866+
const shell = createAppShell(h.renderer, {
867+
terminal: { columns: 80, rows: 24 },
868+
wireKeys: false,
869+
run: "idle",
870+
});
871+
const bridge = attachSessionBridge(shell, createRecordingPort());
872+
try {
873+
for (const event of [
874+
{ type: "inference.start", data: {} },
875+
{
876+
type: "inference.error",
877+
data: {
878+
error: { category: "credential_failure", message: "Forbidden", statusCode: 403 },
879+
},
880+
},
881+
{ type: "inference.start", data: {} },
882+
{ type: "inference.text.delta", data: { token: "recovered" } },
883+
{ type: "inference.done", data: {} },
884+
{ type: "reactor.done", data: {} },
885+
] as const) {
886+
bridge.handle(event);
887+
}
888+
889+
expect(errorRows(shell)).toEqual([]);
890+
expect(shell.streamLog.map((r) => r.text).join("\n")).toContain("recovered");
891+
} finally {
892+
bridge.dispose();
893+
shell.dispose();
894+
}
895+
},
896+
{ width: 80, height: 24 },
897+
);
898+
});
899+
900+
test("an echoed auto-retry prompt does not expire recovery", async () => {
901+
await withTestRenderer(
902+
async (h) => {
903+
const shell = createAppShell(h.renderer, {
904+
terminal: { columns: 80, rows: 24 },
905+
wireKeys: false,
906+
run: "idle",
907+
});
908+
const port = createRecordingPort();
909+
const bridge = attachSessionBridge(shell, port);
910+
try {
911+
bridge.submit("retry this", "immediate");
912+
bridge.handle({ type: "message.received", data: { message: { content: "retry this" } } });
913+
bridge.handle({ type: "inference.start", data: {} });
914+
bridge.handle({
915+
type: "inference.error",
916+
data: {
917+
error: {
918+
category: "quota_exhausted",
919+
message: "The usage limit has been reached",
920+
statusCode: 429,
921+
},
922+
},
923+
});
924+
bridge.submit("retry this", "immediate");
925+
bridge.handle({ type: "message.received", data: { message: { content: "retry this" } } });
926+
bridge.handle({ type: "inference.start", data: {} });
927+
bridge.handle({ type: "inference.text.delta", data: { token: "recovered" } });
928+
bridge.handle({ type: "inference.done", data: {} });
929+
bridge.handle({ type: "reactor.done", data: {} });
930+
931+
expect(errorRows(shell)).toEqual([]);
932+
expect(shell.streamLog.map((r) => r.text).join("\n")).toContain("recovered");
933+
// The replay duplicates the operator's prompt; rollback drops the copy.
934+
expect(shell.streamLog.filter((r) => r.role === "user").map((r) => r.text)).toEqual([
935+
"retry this",
936+
]);
937+
// Same-turn failover, not an operator stop — recovery must not borrow interrupt.
938+
expect(port.calls.some((c) => c.op === "interrupt")).toBe(false);
939+
expect(shell.streamLog.some((r) => r.meta === "stop")).toBe(false);
940+
} finally {
941+
bridge.dispose();
942+
shell.dispose();
943+
}
944+
},
945+
{ width: 80, height: 24 },
946+
);
947+
});
948+
949+
test("a steer echo at a tool boundary opens a new thinking row", async () => {
950+
await withTestRenderer(
951+
async (h) => {
952+
const shell = createAppShell(h.renderer, {
953+
terminal: { columns: 80, rows: 24 },
954+
wireKeys: false,
955+
run: "idle",
956+
});
957+
const bridge = attachSessionBridge(shell, createRecordingPort());
958+
try {
959+
bridge.submit("first prompt", "immediate");
960+
bridge.handle({
961+
type: "message.received",
962+
data: { message: { content: "first prompt" } },
963+
});
964+
bridge.handle({ type: "inference.start", data: {} });
965+
bridge.handle({ type: "inference.thinking.delta", data: { token: "planning" } });
966+
bridge.handle({
967+
type: "inference.tool_call.end",
968+
data: { name: "run_shell", callId: "c1", arguments: "{}" },
969+
});
970+
bridge.handle({ type: "inference.done", data: {} });
971+
bridge.submit("steer this", "steer");
972+
bridge.handle({
973+
type: "tool.done",
974+
data: { result: { callId: "c1", content: "ok", isError: false } },
975+
});
976+
bridge.handle({ type: "message.received", data: { message: { content: "steer this" } } });
977+
bridge.handle({ type: "inference.start", data: {} });
978+
bridge.handle({ type: "inference.thinking.delta", data: { token: "after steer" } });
979+
bridge.handle({ type: "inference.text.delta", data: { token: "done" } });
980+
981+
const rows = shell.streamLog.map((r) => `${r.meta ?? r.role}:${r.text}`);
982+
expect(rows.indexOf("thinking:planning")).toBeGreaterThan(-1);
983+
expect(rows.indexOf("steering:steer this")).toBeGreaterThan(
984+
rows.indexOf("thinking:planning"),
985+
);
986+
expect(rows.indexOf("thinking:after steer")).toBeGreaterThan(
987+
rows.indexOf("steering:steer this"),
988+
);
989+
expect(shell.streamLog.filter((r) => r.meta === "thinking")).toHaveLength(2);
990+
} finally {
991+
bridge.dispose();
992+
shell.dispose();
993+
}
994+
},
995+
{ width: 80, height: 24 },
996+
);
997+
});
998+
999+
test("interrupt then a new prompt keeps the prompt and the classified error", async () => {
1000+
await withTestRenderer(
1001+
async (h) => {
1002+
const shell = createAppShell(h.renderer, {
1003+
terminal: { columns: 80, rows: 24 },
1004+
wireKeys: false,
1005+
run: "idle",
1006+
});
1007+
const port = createRecordingPort();
1008+
const bridge = attachSessionBridge(shell, port);
1009+
try {
1010+
bridge.handle({ type: "inference.start", data: {} });
1011+
bridge.handle({
1012+
type: "inference.error",
1013+
data: {
1014+
error: { category: "credential_failure", message: "Forbidden", statusCode: 403 },
1015+
},
1016+
});
1017+
bridge.interrupt();
1018+
bridge.submit("next prompt", "immediate");
1019+
bridge.handle({
1020+
type: "message.received",
1021+
data: { message: { content: "next prompt" } },
1022+
});
1023+
bridge.handle({ type: "inference.start", data: {} });
1024+
1025+
const text = shell.streamLog.map((r) => r.text).join("\n");
1026+
expect(text).toContain("next prompt");
1027+
expect(errorRows(shell)).toContain("Session expired — re-authenticating…");
1028+
} finally {
1029+
bridge.dispose();
1030+
shell.dispose();
1031+
}
1032+
},
1033+
{ width: 80, height: 24 },
1034+
);
1035+
});
1036+
1037+
test("a queued steer row survives failover rollback", async () => {
1038+
await withTestRenderer(
1039+
async (h) => {
1040+
const shell = createAppShell(h.renderer, {
1041+
terminal: { columns: 80, rows: 24 },
1042+
wireKeys: false,
1043+
run: "idle",
1044+
});
1045+
const bridge = attachSessionBridge(shell, createRecordingPort());
1046+
try {
1047+
bridge.handle({ type: "inference.start", data: {} });
1048+
bridge.handle({
1049+
type: "inference.error",
1050+
data: {
1051+
error: { category: "credential_failure", message: "Forbidden", statusCode: 403 },
1052+
},
1053+
});
1054+
bridge.submit("steer this", "steer");
1055+
bridge.handle({ type: "inference.start", data: {} });
1056+
bridge.handle({ type: "inference.text.delta", data: { token: "recovered" } });
1057+
bridge.handle({ type: "inference.done", data: {} });
1058+
bridge.handle({ type: "reactor.done", data: {} });
1059+
1060+
const text = shell.streamLog.map((r) => r.text).join("\n");
1061+
expect(errorRows(shell)).toEqual([]);
1062+
expect(text).toContain("recovered");
1063+
expect(text).toContain("steer this");
1064+
} finally {
1065+
bridge.dispose();
1066+
shell.dispose();
1067+
}
1068+
},
1069+
{ width: 80, height: 24 },
1070+
);
1071+
});
1072+
1073+
test("reinject interrupt keeps the prompt and the classified error", async () => {
1074+
await withTestRenderer(
1075+
async (h) => {
1076+
const shell = createAppShell(h.renderer, {
1077+
terminal: { columns: 80, rows: 24 },
1078+
wireKeys: false,
1079+
run: "idle",
1080+
});
1081+
const bridge = attachSessionBridge(shell, createRecordingPort());
1082+
try {
1083+
bridge.handle({ type: "inference.start", data: {} });
1084+
bridge.handle({
1085+
type: "inference.error",
1086+
data: {
1087+
error: { category: "credential_failure", message: "Forbidden", statusCode: 403 },
1088+
},
1089+
});
1090+
bridge.submit("restart from here", "reinject");
1091+
bridge.handle({ type: "inference.start", data: {} });
1092+
bridge.handle({ type: "inference.text.delta", data: { token: "recovered" } });
1093+
bridge.handle({ type: "inference.done", data: {} });
1094+
bridge.handle({ type: "reactor.done", data: {} });
1095+
1096+
const text = shell.streamLog.map((r) => r.text).join("\n");
1097+
expect(text).toContain("restart from here");
1098+
expect(text).toContain("stop — restarting from your message");
1099+
expect(errorRows(shell)).toContain("Session expired — re-authenticating…");
1100+
} finally {
1101+
bridge.dispose();
1102+
shell.dispose();
1103+
}
1104+
},
1105+
{ width: 80, height: 24 },
1106+
);
1107+
});
1108+
1109+
test("a terminal inference.error with no recovery still surfaces", async () => {
1110+
await withTestRenderer(
1111+
async (h) => {
1112+
const shell = createAppShell(h.renderer, {
1113+
terminal: { columns: 80, rows: 24 },
1114+
wireKeys: false,
1115+
run: "idle",
1116+
});
1117+
const bridge = attachSessionBridge(shell, createRecordingPort());
1118+
try {
1119+
for (const event of [
1120+
{ type: "inference.start", data: {} },
1121+
{
1122+
type: "inference.error",
1123+
data: {
1124+
error: { category: "credential_failure", message: "Forbidden", statusCode: 403 },
1125+
},
1126+
},
1127+
{ type: "reactor.error", data: { error: "failed" } },
1128+
] as const) {
1129+
bridge.handle(event);
1130+
}
1131+
1132+
expect(errorRows(shell)).toContain("Session expired — re-authenticating…");
1133+
} finally {
1134+
bridge.dispose();
1135+
shell.dispose();
1136+
}
1137+
},
1138+
{ width: 80, height: 24 },
1139+
);
1140+
});
1141+
});
1142+
8171143
describe("parallel sub-agent dispatch on the live session bridge", () => {
8181144
// The live main-session path tracks a call's row by callId in its own map
8191145
// (applyToolCall/applyToolResult), independent of tool-rows.ts's name-based

0 commit comments

Comments
 (0)