From 8e483d1706672dffc036183af806b961317cfd7e Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 10 Sep 2026 15:52:40 -0700 Subject: [PATCH 1/2] Retry attributable xAI capacity protocol mismatches xAI and Grok sometimes surface capacity or overload as protocol_mismatch instead of a retryable status. Remap those attributable phrases to retryable while leaving quota exhaustion, unknown providers, and OpenCode Go unchanged. --- src/agent/retry-policy.test.ts | 33 +++++++++++++ src/inference-gateway-error.test.ts | 72 +++++++++++++++++++++++++++++ src/inference-gateway-error.ts | 57 ++++++++++++++++++++++- 3 files changed, 160 insertions(+), 2 deletions(-) diff --git a/src/agent/retry-policy.test.ts b/src/agent/retry-policy.test.ts index bbf0699b2..fe0dfac65 100644 --- a/src/agent/retry-policy.test.ts +++ b/src/agent/retry-policy.test.ts @@ -34,6 +34,39 @@ describe("createCorbitsRetryPolicy", () => { expect(decision).toEqual({ kind: "retry", delayMs: 500 }); }); + test("bounds attributable xAI capacity retries to three attempts", async () => { + const decide = policy({ providerId: "xai/default" }); + const situation = (attempt: number) => ({ + attempt, + elapsedMs: 0, + error: { + category: "protocol_mismatch" as const, + message: "The model is currently at capacity", + }, + }); + + expect(await decide(situation(1))).toEqual({ kind: "retry", delayMs: 500 }); + expect(await decide(situation(2))).toEqual({ + kind: "retry", + delayMs: 1000, + }); + expect(await decide(situation(3))).toEqual({ kind: "abort" }); + }); + + test("aborts attributable xAI quota exhaustion", async () => { + const decision = await policy({ providerId: "xai/default" })({ + attempt: 1, + elapsedMs: 0, + error: { + category: "quota_exhausted", + message: "Service temporarily unavailable: quota exhausted", + statusCode: 429, + retryAfterMs: 86_400_000, + }, + }); + expect(decision).toEqual({ kind: "abort" }); + }); + test("aborts an OpenCode Go malformed streamed SSE schema response", async () => { const decision = await policy({ providerId: "opencode-go/corbits" })({ attempt: 1, diff --git a/src/inference-gateway-error.test.ts b/src/inference-gateway-error.test.ts index 0222b1aa6..df95a7ed4 100644 --- a/src/inference-gateway-error.test.ts +++ b/src/inference-gateway-error.test.ts @@ -281,6 +281,78 @@ describe("normalizeInferenceErrorForRetry", () => { expect(normalized).toBe(error); }); + test("known-xAI message-only capacity protocol error becomes retryable", () => { + const normalized = normalizeInferenceErrorForRetry({ + category: "protocol_mismatch", + message: "The model is currently at capacity. Please try again later.", + providerId: "xai/default", + retryAfterMs: 2_500, + }); + expect(normalized.category).toBe("retryable"); + expect(normalized.retryAfterMs).toBe(2_500); + }); + + test("known-xAI JSON-bodied high-demand protocol error becomes retryable", () => { + const normalized = normalizeInferenceErrorForRetry({ + category: "protocol_mismatch", + message: "malformed JSON in SSE data payload", + providerId: "xai/default", + raw: { + error: { + message: "The service is unavailable due to high demand", + }, + }, + }); + expect(normalized.category).toBe("retryable"); + }); + + test("known-xAI exact temporary-unavailable phrase becomes retryable", () => { + const normalized = normalizeInferenceErrorForRetry({ + category: "protocol_mismatch", + message: "Service temporarily unavailable", + providerId: "xai/default", + }); + expect(normalized.category).toBe("retryable"); + }); + + test("explicit Grok adapter overload protocol error becomes retryable", () => { + const normalized = normalizeInferenceErrorForRetry({ + category: "protocol_mismatch", + message: "The upstream service is overloaded", + providerId: "grok-responses", + }); + expect(normalized.category).toBe("retryable"); + }); + + test("unknown provider capacity protocol_mismatch stays unchanged", () => { + const error = { + category: "protocol_mismatch" as const, + message: "The model is currently at capacity", + providerId: "openai", + }; + expect(normalizeInferenceErrorForRetry(error)).toBe(error); + }); + + test("OpenCode Go capacity prose stays protocol_mismatch", () => { + const error = { + category: "protocol_mismatch" as const, + message: "The model is currently at capacity", + providerId: "opencode-go/default", + }; + expect(normalizeInferenceErrorForRetry(error)).toBe(error); + }); + + test("known-xAI quota exhaustion stays non-retryable despite capacity prose", () => { + const error = { + category: "quota_exhausted" as const, + message: "Service temporarily unavailable: quota exhausted", + statusCode: 429, + providerId: "xai/default", + retryAfterMs: 86_400_000, + }; + expect(normalizeInferenceErrorForRetry(error)).toBe(error); + }); + test("known-xAI bare 429 reclassifies as retryable", () => { const bare = { category: "quota_exhausted" as const, diff --git a/src/inference-gateway-error.ts b/src/inference-gateway-error.ts index 36d8e9e27..5d21c0c11 100644 --- a/src/inference-gateway-error.ts +++ b/src/inference-gateway-error.ts @@ -238,6 +238,55 @@ function textHasXaiQuotaMarkers(...parts: string[]): boolean { return XAI_QUOTA_BODY_MARKERS.some((marker) => combined.includes(marker)); } +/** + * xAI / Grok capacity and overload phrases that arrive as protocol_mismatch + * (message-only or JSON raw) when the stream is not valid SSE. Exact + * "Service temporarily unavailable" is intentional — do not widen to the + * gateway "service unavailable" substring, which would rematch quota copy. + */ +const XAI_CAPACITY_TEXT_MARKERS = [ + "currently at capacity", + "overloaded", + "high demand", +] as const; + +const XAI_CAPACITY_EXACT_MESSAGES = new Set([ + "service temporarily unavailable", +]); + +function textSuggestsXaiCapacity(...parts: string[]): boolean { + const combined = parts.join("\n").toLowerCase(); + if (XAI_CAPACITY_TEXT_MARKERS.some((marker) => combined.includes(marker))) { + return true; + } + // Exact phrase is message-only; do not substring-match so quota suffixes stay out. + return XAI_CAPACITY_EXACT_MESSAGES.has(parts[0]?.trim().toLowerCase() ?? ""); +} + +/** + * Remap attributable xAI / Grok capacity protocol_mismatch errors to retryable. + * Unknown providers and OpenCode Go stay terminal. Never remaps quota_exhausted. + */ +export function normalizeXaiCapacityError( + error: InferenceErrorWithGoContext, +): InferenceError { + if (error.category !== "protocol_mismatch") return error; + if (!isKnownXaiProviderId(error.providerId)) return error; + if (!textSuggestsXaiCapacity(error.message ?? "", stringFromRaw(error.raw))) { + return error; + } + + return { + category: "retryable", + message: GATEWAY_OVERLOAD_USER_MESSAGE, + statusCode: error.statusCode ?? 503, + ...(error.raw !== undefined ? { raw: error.raw } : {}), + ...(error.retryAfterMs !== undefined + ? { retryAfterMs: error.retryAfterMs } + : {}), + }; +} + /** * True when a known-xAI HTTP 429 looks like a short rate limit rather than a * usage/quota window. Used by both retry normalization and transcript copy — @@ -400,8 +449,9 @@ function normalizeCodexUsageLimitError( * Reclassify gateway overload errors so the default retry policy treats them as * transient instead of aborting on protocol_mismatch. Also normalizes OpenCode * Go quota/rate-limit shapes (including HTTP 400 mis-status), known-xAI short - * 429s, Codex usage limits (nested detail.error with resets_in_seconds), and - * known-Codex short 429s that are not usage_limit_reached. + * 429s, attributable xAI capacity protocol_mismatch, Codex usage limits + * (nested detail.error with resets_in_seconds), and known-Codex short 429s that + * are not usage_limit_reached. */ export function normalizeInferenceErrorForRetry( error: InferenceErrorWithGoContext, @@ -412,6 +462,9 @@ export function normalizeInferenceErrorForRetry( const xaiNormalized = normalizeXaiRateLimitError(error); if (xaiNormalized !== error) return xaiNormalized; + const xaiCapacity = normalizeXaiCapacityError(error); + if (xaiCapacity !== error) return xaiCapacity; + const codexNormalized = normalizeCodexUsageLimitError(error); if (codexNormalized !== error) return codexNormalized; From b1c3b7d2813aab2b2eb5ee379a58bbda39716cc5 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 10 Sep 2026 18:26:11 -0700 Subject: [PATCH 2/2] Match xAI capacity copy on raw, veto quota markers, and drop retrying wording --- src/inference-gateway-error.test.ts | 40 ++++++++++++++++++++++++ src/inference-gateway-error.ts | 47 ++++++++++++++++++++++++----- 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/src/inference-gateway-error.test.ts b/src/inference-gateway-error.test.ts index df95a7ed4..6c8f1a0e3 100644 --- a/src/inference-gateway-error.test.ts +++ b/src/inference-gateway-error.test.ts @@ -4,6 +4,7 @@ import { isGatewayOverloadInferenceError, looksLikeHtmlGatewayBody, normalizeInferenceErrorForRetry, + XAI_CAPACITY_USER_MESSAGE, } from "./inference-gateway-error.js"; const CLOUDFLARE_503_HTML = ` @@ -315,6 +316,45 @@ describe("normalizeInferenceErrorForRetry", () => { expect(normalized.category).toBe("retryable"); }); + test("known-xAI exact phrase carried on raw becomes retryable", () => { + const normalized = normalizeInferenceErrorForRetry({ + category: "protocol_mismatch", + message: "malformed JSON in SSE data payload", + providerId: "xai/default", + raw: "Service temporarily unavailable", + }); + expect(normalized.category).toBe("retryable"); + }); + + test("known-xAI exact phrase nested in JSON raw becomes retryable", () => { + const normalized = normalizeInferenceErrorForRetry({ + category: "protocol_mismatch", + message: "malformed JSON in SSE data payload", + providerId: "xai/default", + raw: { error: { message: "Service temporarily unavailable" } }, + }); + expect(normalized.category).toBe("retryable"); + }); + + test("remapped xAI capacity copy does not claim an ongoing retry", () => { + const normalized = normalizeInferenceErrorForRetry({ + category: "protocol_mismatch", + message: "The model is currently at capacity", + providerId: "xai/default", + }); + expect(normalized.message).toBe(XAI_CAPACITY_USER_MESSAGE); + expect(normalized.message).not.toContain("retrying"); + }); + + test("mixed xAI capacity and quota copy stays unchanged", () => { + const error = { + category: "protocol_mismatch" as const, + message: "The model is currently at capacity: quota exceeded", + providerId: "xai/default", + }; + expect(normalizeInferenceErrorForRetry(error)).toBe(error); + }); + test("explicit Grok adapter overload protocol error becomes retryable", () => { const normalized = normalizeInferenceErrorForRetry({ category: "protocol_mismatch", diff --git a/src/inference-gateway-error.ts b/src/inference-gateway-error.ts index 5d21c0c11..0c88a7f65 100644 --- a/src/inference-gateway-error.ts +++ b/src/inference-gateway-error.ts @@ -238,6 +238,13 @@ function textHasXaiQuotaMarkers(...parts: string[]): boolean { return XAI_QUOTA_BODY_MARKERS.some((marker) => combined.includes(marker)); } +/** + * User-visible line for an attributable xAI / Grok capacity error. Worded + * without "retrying" for the same reason as RATE_LIMIT_USER_MESSAGE — it also + * surfaces terminally once retries are exhausted. + */ +export const XAI_CAPACITY_USER_MESSAGE = "xAI at capacity"; + /** * xAI / Grok capacity and overload phrases that arrive as protocol_mismatch * (message-only or JSON raw) when the stream is not valid SSE. Exact @@ -254,31 +261,57 @@ const XAI_CAPACITY_EXACT_MESSAGES = new Set([ "service temporarily unavailable", ]); +/** + * Exact-match only — never substring, so quota-suffixed copy stays out. Checks + * the part itself and common JSON message fields, since intx puts the server + * body on `raw` while `message` carries parser detail. + */ +function isXaiCapacityExactPhrase(part: string): boolean { + if (XAI_CAPACITY_EXACT_MESSAGES.has(part.trim().toLowerCase())) return true; + const parsed = tryParseJSON(part); + if (typeof parsed !== "object" || parsed === null) return false; + const record = parsed as Record; + const nested = record.error; + const candidates = [ + record.message, + typeof nested === "string" ? nested : undefined, + typeof nested === "object" && nested !== null + ? (nested as Record).message + : undefined, + ]; + return candidates.some( + (candidate) => + typeof candidate === "string" && + XAI_CAPACITY_EXACT_MESSAGES.has(candidate.trim().toLowerCase()), + ); +} + function textSuggestsXaiCapacity(...parts: string[]): boolean { const combined = parts.join("\n").toLowerCase(); if (XAI_CAPACITY_TEXT_MARKERS.some((marker) => combined.includes(marker))) { return true; } - // Exact phrase is message-only; do not substring-match so quota suffixes stay out. - return XAI_CAPACITY_EXACT_MESSAGES.has(parts[0]?.trim().toLowerCase() ?? ""); + return parts.some(isXaiCapacityExactPhrase); } /** * Remap attributable xAI / Grok capacity protocol_mismatch errors to retryable. - * Unknown providers and OpenCode Go stay terminal. Never remaps quota_exhausted. + * Unknown providers and OpenCode Go stay terminal. Quota markers anywhere in + * the copy veto the remap — mixed capacity+quota text stays a real quota error. */ export function normalizeXaiCapacityError( error: InferenceErrorWithGoContext, ): InferenceError { if (error.category !== "protocol_mismatch") return error; if (!isKnownXaiProviderId(error.providerId)) return error; - if (!textSuggestsXaiCapacity(error.message ?? "", stringFromRaw(error.raw))) { - return error; - } + const messageText = error.message ?? ""; + const rawText = stringFromRaw(error.raw); + if (textHasXaiQuotaMarkers(messageText, rawText)) return error; + if (!textSuggestsXaiCapacity(messageText, rawText)) return error; return { category: "retryable", - message: GATEWAY_OVERLOAD_USER_MESSAGE, + message: XAI_CAPACITY_USER_MESSAGE, statusCode: error.statusCode ?? 503, ...(error.raw !== undefined ? { raw: error.raw } : {}), ...(error.retryAfterMs !== undefined