diff --git a/src/agent/retry-policy.test.ts b/src/agent/retry-policy.test.ts index bbf0699b2..fe0dfac65 100644 --- a/src/agent/retry-policy.test.ts +++ b/src/agent/retry-policy.test.ts @@ -34,6 +34,39 @@ describe("createCorbitsRetryPolicy", () => { expect(decision).toEqual({ kind: "retry", delayMs: 500 }); }); + test("bounds attributable xAI capacity retries to three attempts", async () => { + const decide = policy({ providerId: "xai/default" }); + const situation = (attempt: number) => ({ + attempt, + elapsedMs: 0, + error: { + category: "protocol_mismatch" as const, + message: "The model is currently at capacity", + }, + }); + + expect(await decide(situation(1))).toEqual({ kind: "retry", delayMs: 500 }); + expect(await decide(situation(2))).toEqual({ + kind: "retry", + delayMs: 1000, + }); + expect(await decide(situation(3))).toEqual({ kind: "abort" }); + }); + + test("aborts attributable xAI quota exhaustion", async () => { + const decision = await policy({ providerId: "xai/default" })({ + attempt: 1, + elapsedMs: 0, + error: { + category: "quota_exhausted", + message: "Service temporarily unavailable: quota exhausted", + statusCode: 429, + retryAfterMs: 86_400_000, + }, + }); + expect(decision).toEqual({ kind: "abort" }); + }); + test("aborts an OpenCode Go malformed streamed SSE schema response", async () => { const decision = await policy({ providerId: "opencode-go/corbits" })({ attempt: 1, diff --git a/src/inference-gateway-error.test.ts b/src/inference-gateway-error.test.ts index 0222b1aa6..6c8f1a0e3 100644 --- a/src/inference-gateway-error.test.ts +++ b/src/inference-gateway-error.test.ts @@ -4,6 +4,7 @@ import { isGatewayOverloadInferenceError, looksLikeHtmlGatewayBody, normalizeInferenceErrorForRetry, + XAI_CAPACITY_USER_MESSAGE, } from "./inference-gateway-error.js"; const CLOUDFLARE_503_HTML = ` @@ -281,6 +282,117 @@ describe("normalizeInferenceErrorForRetry", () => { expect(normalized).toBe(error); }); + test("known-xAI message-only capacity protocol error becomes retryable", () => { + const normalized = normalizeInferenceErrorForRetry({ + category: "protocol_mismatch", + message: "The model is currently at capacity. Please try again later.", + providerId: "xai/default", + retryAfterMs: 2_500, + }); + expect(normalized.category).toBe("retryable"); + expect(normalized.retryAfterMs).toBe(2_500); + }); + + test("known-xAI JSON-bodied high-demand protocol error becomes retryable", () => { + const normalized = normalizeInferenceErrorForRetry({ + category: "protocol_mismatch", + message: "malformed JSON in SSE data payload", + providerId: "xai/default", + raw: { + error: { + message: "The service is unavailable due to high demand", + }, + }, + }); + expect(normalized.category).toBe("retryable"); + }); + + test("known-xAI exact temporary-unavailable phrase becomes retryable", () => { + const normalized = normalizeInferenceErrorForRetry({ + category: "protocol_mismatch", + message: "Service temporarily unavailable", + providerId: "xai/default", + }); + expect(normalized.category).toBe("retryable"); + }); + + test("known-xAI exact phrase carried on raw becomes retryable", () => { + const normalized = normalizeInferenceErrorForRetry({ + category: "protocol_mismatch", + message: "malformed JSON in SSE data payload", + providerId: "xai/default", + raw: "Service temporarily unavailable", + }); + expect(normalized.category).toBe("retryable"); + }); + + test("known-xAI exact phrase nested in JSON raw becomes retryable", () => { + const normalized = normalizeInferenceErrorForRetry({ + category: "protocol_mismatch", + message: "malformed JSON in SSE data payload", + providerId: "xai/default", + raw: { error: { message: "Service temporarily unavailable" } }, + }); + expect(normalized.category).toBe("retryable"); + }); + + test("remapped xAI capacity copy does not claim an ongoing retry", () => { + const normalized = normalizeInferenceErrorForRetry({ + category: "protocol_mismatch", + message: "The model is currently at capacity", + providerId: "xai/default", + }); + expect(normalized.message).toBe(XAI_CAPACITY_USER_MESSAGE); + expect(normalized.message).not.toContain("retrying"); + }); + + test("mixed xAI capacity and quota copy stays unchanged", () => { + const error = { + category: "protocol_mismatch" as const, + message: "The model is currently at capacity: quota exceeded", + providerId: "xai/default", + }; + expect(normalizeInferenceErrorForRetry(error)).toBe(error); + }); + + test("explicit Grok adapter overload protocol error becomes retryable", () => { + const normalized = normalizeInferenceErrorForRetry({ + category: "protocol_mismatch", + message: "The upstream service is overloaded", + providerId: "grok-responses", + }); + expect(normalized.category).toBe("retryable"); + }); + + test("unknown provider capacity protocol_mismatch stays unchanged", () => { + const error = { + category: "protocol_mismatch" as const, + message: "The model is currently at capacity", + providerId: "openai", + }; + expect(normalizeInferenceErrorForRetry(error)).toBe(error); + }); + + test("OpenCode Go capacity prose stays protocol_mismatch", () => { + const error = { + category: "protocol_mismatch" as const, + message: "The model is currently at capacity", + providerId: "opencode-go/default", + }; + expect(normalizeInferenceErrorForRetry(error)).toBe(error); + }); + + test("known-xAI quota exhaustion stays non-retryable despite capacity prose", () => { + const error = { + category: "quota_exhausted" as const, + message: "Service temporarily unavailable: quota exhausted", + statusCode: 429, + providerId: "xai/default", + retryAfterMs: 86_400_000, + }; + expect(normalizeInferenceErrorForRetry(error)).toBe(error); + }); + test("known-xAI bare 429 reclassifies as retryable", () => { const bare = { category: "quota_exhausted" as const, diff --git a/src/inference-gateway-error.ts b/src/inference-gateway-error.ts index 36d8e9e27..0c88a7f65 100644 --- a/src/inference-gateway-error.ts +++ b/src/inference-gateway-error.ts @@ -238,6 +238,88 @@ function textHasXaiQuotaMarkers(...parts: string[]): boolean { return XAI_QUOTA_BODY_MARKERS.some((marker) => combined.includes(marker)); } +/** + * User-visible line for an attributable xAI / Grok capacity error. Worded + * without "retrying" for the same reason as RATE_LIMIT_USER_MESSAGE — it also + * surfaces terminally once retries are exhausted. + */ +export const XAI_CAPACITY_USER_MESSAGE = "xAI at capacity"; + +/** + * xAI / Grok capacity and overload phrases that arrive as protocol_mismatch + * (message-only or JSON raw) when the stream is not valid SSE. Exact + * "Service temporarily unavailable" is intentional — do not widen to the + * gateway "service unavailable" substring, which would rematch quota copy. + */ +const XAI_CAPACITY_TEXT_MARKERS = [ + "currently at capacity", + "overloaded", + "high demand", +] as const; + +const XAI_CAPACITY_EXACT_MESSAGES = new Set([ + "service temporarily unavailable", +]); + +/** + * Exact-match only — never substring, so quota-suffixed copy stays out. Checks + * the part itself and common JSON message fields, since intx puts the server + * body on `raw` while `message` carries parser detail. + */ +function isXaiCapacityExactPhrase(part: string): boolean { + if (XAI_CAPACITY_EXACT_MESSAGES.has(part.trim().toLowerCase())) return true; + const parsed = tryParseJSON(part); + if (typeof parsed !== "object" || parsed === null) return false; + const record = parsed as Record; + const nested = record.error; + const candidates = [ + record.message, + typeof nested === "string" ? nested : undefined, + typeof nested === "object" && nested !== null + ? (nested as Record).message + : undefined, + ]; + return candidates.some( + (candidate) => + typeof candidate === "string" && + XAI_CAPACITY_EXACT_MESSAGES.has(candidate.trim().toLowerCase()), + ); +} + +function textSuggestsXaiCapacity(...parts: string[]): boolean { + const combined = parts.join("\n").toLowerCase(); + if (XAI_CAPACITY_TEXT_MARKERS.some((marker) => combined.includes(marker))) { + return true; + } + return parts.some(isXaiCapacityExactPhrase); +} + +/** + * Remap attributable xAI / Grok capacity protocol_mismatch errors to retryable. + * Unknown providers and OpenCode Go stay terminal. Quota markers anywhere in + * the copy veto the remap — mixed capacity+quota text stays a real quota error. + */ +export function normalizeXaiCapacityError( + error: InferenceErrorWithGoContext, +): InferenceError { + if (error.category !== "protocol_mismatch") return error; + if (!isKnownXaiProviderId(error.providerId)) return error; + const messageText = error.message ?? ""; + const rawText = stringFromRaw(error.raw); + if (textHasXaiQuotaMarkers(messageText, rawText)) return error; + if (!textSuggestsXaiCapacity(messageText, rawText)) return error; + + return { + category: "retryable", + message: XAI_CAPACITY_USER_MESSAGE, + statusCode: error.statusCode ?? 503, + ...(error.raw !== undefined ? { raw: error.raw } : {}), + ...(error.retryAfterMs !== undefined + ? { retryAfterMs: error.retryAfterMs } + : {}), + }; +} + /** * True when a known-xAI HTTP 429 looks like a short rate limit rather than a * usage/quota window. Used by both retry normalization and transcript copy — @@ -400,8 +482,9 @@ function normalizeCodexUsageLimitError( * Reclassify gateway overload errors so the default retry policy treats them as * transient instead of aborting on protocol_mismatch. Also normalizes OpenCode * Go quota/rate-limit shapes (including HTTP 400 mis-status), known-xAI short - * 429s, Codex usage limits (nested detail.error with resets_in_seconds), and - * known-Codex short 429s that are not usage_limit_reached. + * 429s, attributable xAI capacity protocol_mismatch, Codex usage limits + * (nested detail.error with resets_in_seconds), and known-Codex short 429s that + * are not usage_limit_reached. */ export function normalizeInferenceErrorForRetry( error: InferenceErrorWithGoContext, @@ -412,6 +495,9 @@ export function normalizeInferenceErrorForRetry( const xaiNormalized = normalizeXaiRateLimitError(error); if (xaiNormalized !== error) return xaiNormalized; + const xaiCapacity = normalizeXaiCapacityError(error); + if (xaiCapacity !== error) return xaiCapacity; + const codexNormalized = normalizeCodexUsageLimitError(error); if (codexNormalized !== error) return codexNormalized;