Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions packages/core/src/services/resume.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,17 +40,20 @@ const PLATFORMS = new Set(["slack", "telegram", "teams"]);
/** Claude Code's answer when Claude's API couldn't be reached or the connection dropped, after its own retries. */
const NO_CONNECTION =
/^API Error: [^\n]*(Can't reach the API server|Unable to connect|Connection error|Connection dropped|Request timed out|socket hang up|ENOTFOUND|EAI_AGAIN|ENETUNREACH|ECONNREFUSED|ECONNRESET|ECONNABORTED|EPIPE|ETIMEDOUT)/i;
/**
* Claude Code dropped the turn on its own, or Claude's API failed on its side: trying again usually helps. A 403 comes
* right after the computer wakes, before Claude Code has renewed its sign-in.
*/
/** Claude Code dropped the turn on its own, or Claude's API failed on its side: trying again usually helps. */
const ERRORED =
/^(?:Claude Code exited (?:without a result|with code \d+)$|API Error: (?:5\d\d\b|[^\n]*(?:overloaded|Internal server error|"type":"api_error"))|Failed to authenticate\. API Error: 403\b)/i;
/**
* Claude's API refused the request (right after the computer wakes, or while its network changes). Claude Code doesn't
* try again itself, and the refusal lasts a minute or two: a try right away fails the same way.
*/
const REFUSED = /^Failed to authenticate\. API Error: 403\b/i;
const OFFLINE_RESUMABLE: ReadonlySet<RunTrigger> = new Set(["chat", "manual", "api", "followup", "call", "routine", "delegation"]);
/** Minutes to wait before each try in a row: a connection that just dropped often drops again right away. */
const OFFLINE_BACKOFF_MIN = [0, 2, 5, 15, 30];
/** An error that comes back each time is no passing hiccup: fewer tries. */
const ERROR_BACKOFF_MIN = [0, 2, 5];
const REFUSED_BACKOFF_MIN = [2, 5, 15];
/** Broke off longer ago than this, the turn waits for someone to continue it. */
const OFFLINE_MAX_AGE_MS = 12 * 3_600_000;
const OFFLINE_CHECK_MS = 60_000;
Expand Down Expand Up @@ -333,7 +336,8 @@ function causeOf(error: string | null | undefined): "offline" | "error" | null {
}

function backoffOf(r: RunRow): number[] {
return causeOf(r.error) === "offline" ? OFFLINE_BACKOFF_MIN : ERROR_BACKOFF_MIN;
if (causeOf(r.error) === "offline") return OFFLINE_BACKOFF_MIN;
return REFUSED.test(r.error?.trim() ?? "") ? REFUSED_BACKOFF_MIN : ERROR_BACKOFF_MIN;
}

/** Turns that failed because Claude's API couldn't be reached or Claude Code dropped them, and nothing went on in their chat since. */
Expand Down
25 changes: 17 additions & 8 deletions packages/core/test/resume.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -388,15 +388,24 @@ describe("continuing once Claude can be reached again", () => {
expect(await continueAfterOutage(offline)).toBe(0);
}, 40_000);

test("a turn Claude's API refused with a 403 right after waking goes on", async () => {
test("a turn Claude's API refused with a 403 goes on after a pause, not right away, three times at most", async () => {
const refused = "Failed to authenticate. API Error: 403 Request not allowed";
const conv = chat(agent, "routine");
const last = cutOff(agent, conv, { status: "failed", error: "Failed to authenticate. API Error: 403 Request not allowed", trigger: "routine" });
expect(offlineRetryAt(last)).not.toBeNull();
expect(await continueAfterOutage(offline)).toBe(1);
const run = next(conv, last)!;
expect(marker(run.id)).toMatchObject({ type: "retry", runId: last, auto: true, cause: "error" });
await waitForRun(run.id, 10_000);
}, 20_000);
let last = cutOff(agent, conv, { status: "failed", error: refused, trigger: "routine" });
for (const waitMin of [2, 5, 15]) {
expect(await continueAfterOutage(offline)).toBe(0);
expect(Date.parse(offlineRetryAt(last)!)).toBeGreaterThan(Date.now() + (waitMin - 1) * 60_000);
sql("UPDATE runs SET finished_at = ? WHERE id = ?", ago(waitMin * 60_000 + 1000), last);
expect(await continueAfterOutage(offline)).toBe(1);
const run = next(conv, last)!;
expect(marker(run.id)).toMatchObject({ type: "retry", runId: last, auto: true, cause: "error" });
await waitForRun(run.id, 10_000);
sql("UPDATE runs SET status = 'failed', error = ?, finished_at = ? WHERE id = ?", refused, now(), run.id);
last = run.id;
}
expect(offlineRetryAt(last)).toBeNull();
expect(await continueAfterOutage(offline)).toBe(0);
}, 40_000);

test("work handed over without waiting continues; work someone waited for is left to them", async () => {
const parentChat = chat(agent);
Expand Down
Loading