Remove automatic provider failover

Author: TheGreatAxios

docs/TELEMETRY.md

@@ -169,11 +169,11 @@ settles an unresolved turn once as an unsampled terminal failure. Attribution
 uses the latest `inference.usage` source, the first lifecycle payload carrying
 the runtime-resolved provider/model pair for an attempt. Each `inference.start`
 clears that authoritative source and records the newly attempted model. If a
-fallback fails before usage exposes its source, telemetry retains that actual
-model but uses the fixed `unknown` provider/source bucket rather than attributing
-it to the previously selected provider. When the attempted model still matches
-the selected source, that full source remains valid. Therefore a parent turn
-emits at most one terminal `$ai_generation`, including retry and failover paths.
+retry fails before usage exposes its source, telemetry retains that attempted
+model but uses the fixed `unknown` provider/source bucket. When the attempted
+model still matches the selected source, that full source remains valid.
+Therefore a parent turn emits at most one terminal `$ai_generation`, including
+retry paths.
 
 ## What's never collected
 

src/config/inference-sources.ts

@@ -13,46 +13,20 @@ import type { Settings } from "./settings.js";
 import { resolveSessionEffort, type ReasoningEffort } from "../provider/reasoning-effort.js";
 import { SOURCE_MAX_TOKENS } from "./index.js";
 import { isOpenCodeGoProvider } from "../../packages/opencode-go/src/index.js";
-import { resolveDefaultModel } from "./providers.js";
 
 export interface BuildSourceContext {
   sessionId: string;
   reasoningEffort?: ReasoningEffort;
   catalog: readonly ProviderCatalogEntry[];
 }
 
-// A resolved provider+model, with optional reasoningEffort — the unit both
-// the primary source and its backups are built from.
+// A resolved provider+model with optional reasoning effort.
 export interface ProviderRef {
   provider: string;
   model: string;
   reasoningEffort?: ReasoningEffort;
 }
 
-function refKey(ref: ProviderRef): string {
-  return `${ref.provider}\0${ref.model}`;
-}
-
-// Every other configured provider, one model each, so a primary source that
-// fails to build (bad credentials, missing baseURL) still has somewhere to
-// fall back to. Order follows settings.providers; providers already covered
-// by `existing` are skipped.
-function backupRefsFromSettings(
-  settings: Settings,
-  existing: readonly ProviderRef[],
-): ProviderRef[] {
-  const seenProviders = new Set(existing.map((r) => r.provider));
-  const tail: ProviderRef[] = [];
-  for (const [provider, p] of Object.entries(settings.providers)) {
-    if (seenProviders.has(provider)) continue;
-    const model = resolveDefaultModel(p);
-    if (model === undefined || model.length === 0) continue;
-    seenProviders.add(provider);
-    tail.push({ provider, model });
-  }
-  return tail;
-}
-
 function catalogEntry(
   catalog: readonly ProviderCatalogEntry[],
   provider: string,
@@ -170,31 +144,6 @@ export function buildInferenceSourceForRef(
   };
 }
 
-export function buildSourcesFromRefs(
-  refs: readonly ProviderRef[],
-  ctx: BuildSourceContext,
-  settings: Settings | undefined,
-): InferenceSource[] {
-  const out: InferenceSource[] = [];
-  const seenIds = new Set<string>();
-  for (const ref of refs) {
-    const src = buildInferenceSourceForRef(ref, ctx, settings);
-    if (src === null) continue;
-    if (seenIds.has(src.id)) continue;
-    seenIds.add(src.id);
-    out.push(src);
-  }
-  return out;
-}
-
-export function prependActiveRef(refs: readonly ProviderRef[], active: ProviderRef): ProviderRef[] {
-  const without = refs.filter((r) => refKey(r) !== refKey(active));
-  return [active, ...without];
-}
-
-// Builds the primary source for `head` plus one backup per other configured
-// provider, so a mid-run failure (bad credentials, dropped connection) has
-// somewhere else to go. `head` always wins as defaultSource when it builds.
 function buildSourceBundle(args: {
   settings: Settings | undefined;
   catalog: readonly ProviderCatalogEntry[];
@@ -208,25 +157,13 @@ function buildSourceBundle(args: {
     ...(args.reasoningEffort !== undefined ? { reasoningEffort: args.reasoningEffort } : {}),
   };
 
-  const refs =
-    args.settings !== undefined
-      ? prependActiveRef(backupRefsFromSettings(args.settings, [args.head]), args.head)
-      : [args.head];
-
-  const sources = buildSourcesFromRefs(refs, ctx, args.settings);
-  const defaultId = args.head.provider;
-  if (sources.length === 0) {
-    const fallback = buildInferenceSourceForRef(args.head, ctx, args.settings);
-    if (fallback === null) {
-      throw new Error(`No inference source for provider "${defaultId}"`);
-    }
-    return { sources: [fallback], defaultSource: fallback.id };
+  const source = buildInferenceSourceForRef(args.head, ctx, args.settings);
+  if (source === null) {
+    throw new Error(
+      `Unable to build inference source for selected provider "${args.head.provider}"`,
+    );
   }
-  const hasDefault = sources.some((s) => s.id === defaultId);
-  return {
-    sources,
-    defaultSource: hasDefault ? defaultId : (sources[0]?.id ?? defaultId),
-  };
+  return { sources: [source], defaultSource: source.id };
 }
 
 export function buildMainSessionSources(args: {

src/exec/runner.ts

@@ -63,6 +63,7 @@ import { createAgentToolset, type AgentToolset, type OperatorResult } from "../a
 import { createAgentWithLiveToolDispatch } from "../agent/live-tool-dispatch.js";
 import { liveTelemetry } from "../telemetry/singleton.js";
 import { createTurnObserver } from "../telemetry/ai-observability.js";
+import { terminalProviderFailureMessage } from "../inference-error-message.js";
 import { collectToolPlugins, resolveToolPlugins } from "../plugins/tool-plugins.js";
 import {
   expandExistingPluginMembers,
@@ -116,6 +117,32 @@ export function formatCaughtError(err: unknown): string {
   return err instanceof Error ? err.message : String(err);
 }
 
+const SELECTED_PROVIDER_FAILURE = "SelectedProviderFailure";
+
+export async function refreshSelectedProviderCredential<T>(refresh: () => Promise<T>): Promise<T> {
+  try {
+    return await refresh();
+  } catch (cause) {
+    const error = new Error(formatCaughtError(cause), { cause });
+    error.name = SELECTED_PROVIDER_FAILURE;
+    throw error;
+  }
+}
+
+export function execUserFailureMessage(
+  config: Config,
+  err: unknown,
+  inferenceStarted: boolean,
+): string {
+  if (inferenceStarted || (err instanceof Error && err.name === SELECTED_PROVIDER_FAILURE)) {
+    return terminalProviderFailureMessage(
+      config.providerName,
+      config.settings?.providers[config.providerName]?.name,
+    );
+  }
+  return formatCaughtError(err);
+}
+
 /**
  * Exec-primary director overlay. Omit / skywalker keep the product default
  * (`loadSessionChatPrompt` + advertised session tools). Any other closed-fleet
@@ -252,6 +279,7 @@ export async function runExec(config: Config): Promise<ExecResult> {
   let finalized = false;
   let turnsUsed = 0;
   let runSink: RunSink | null = null;
+  let inferenceStarted = false;
 
   const persist = async (
     status: "running" | "done" | "failed" | "cancelled",
@@ -605,15 +633,19 @@ export async function runExec(config: Config): Promise<ExecResult> {
 
     // Refresh OAuth tokens before first inference when starting on codex/xai.
     if (initialCodexProfile !== undefined) {
-      const { access } = await getValidCodexToken(initialCodexProfile);
+      const { access } = await refreshSelectedProviderCredential(() =>
+        getValidCodexToken(initialCodexProfile),
+      );
       liveSource = { ...liveSource, apiKey: access };
       liveSubAgentProvider.current = {
         ...liveSubAgentProvider.current,
         apiKey: access,
       };
     }
     if (initialXaiProfile !== undefined) {
-      const { access } = await getValidXaiToken(initialXaiProfile);
+      const { access } = await refreshSelectedProviderCredential(() =>
+        getValidXaiToken(initialXaiProfile),
+      );
       liveSource = { ...liveSource, apiKey: access };
       liveSubAgentProvider.current = {
         ...liveSubAgentProvider.current,
@@ -739,6 +771,7 @@ export async function runExec(config: Config): Promise<ExecResult> {
     let runError: string | undefined;
     let sinkStatus: ReturnType<typeof liveSink.getStatus> = "cancelled";
     try {
+      inferenceStarted = true;
       // Final OAuth refresh immediately before send (token may have aged during MCP).
       if (initialCodexProfile !== undefined) {
         const { access } = await getValidCodexToken(initialCodexProfile);
@@ -835,17 +868,24 @@ export async function runExec(config: Config): Promise<ExecResult> {
     });
 
     if (!sendCompleted || runError !== undefined || summaryStatus === "failed") {
-      const message =
+      const diagnosticMessage =
         runError ??
         (summaryStatus === "cancelled" ? "run cancelled before completion" : "run failed");
-      stderr.write(`Error: ${message}\n`);
+      const userMessage =
+        summaryStatus === "failed"
+          ? terminalProviderFailureMessage(
+              config.providerName,
+              config.settings?.providers[config.providerName]?.name,
+            )
+          : diagnosticMessage;
+      stderr.write(`Error: ${userMessage}\n`);
       const persistStatus = summaryStatus === "cancelled" ? "cancelled" : "failed";
-      await persist(persistStatus, { error: message });
+      await persist(persistStatus, { error: diagnosticMessage });
       return {
         exitCode: 1,
         sessionId,
         text: textOut,
-        error: message,
+        error: userMessage,
         status: summaryStatus,
         durationMs: finishedAt - startedAt,
         turnsUsed: runSink.getTurnCount(),
@@ -870,15 +910,16 @@ export async function runExec(config: Config): Promise<ExecResult> {
       model: config.model,
     };
   } catch (err) {
-    const message = err instanceof Error ? err.message : String(err);
-    logger.error("exec failed: {error}", { error: message });
-    stderr.write(`Error: ${message}\n`);
-    await persist("failed", { error: message });
+    const diagnosticMessage = formatCaughtError(err);
+    logger.error("exec failed: {error}", { error: diagnosticMessage });
+    const userMessage = execUserFailureMessage(config, err, inferenceStarted);
+    stderr.write(`Error: ${userMessage}\n`);
+    await persist("failed", { error: diagnosticMessage });
     return {
       exitCode: 1,
       sessionId,
       text: textOut,
-      error: message,
+      error: userMessage,
       status: "failed",
       durationMs: Date.now() - startedAt,
       turnsUsed: runSink?.getTurnCount() ?? turnsUsed,

src/inference-error-message.test.ts

@@ -1,6 +1,9 @@
 import { describe, expect, test } from "bun:test";
 
-import { inferenceErrorMessage } from "./inference-error-message.js";
+import {
+  inferenceErrorMessage,
+  terminalProviderFailureMessage,
+} from "./inference-error-message.js";
 
 const CODEX_BODY = {
   detail: {
@@ -89,3 +92,25 @@ describe("inferenceErrorMessage", () => {
     expect(line.toLowerCase()).toMatch(/log in again|sign in again/);
   });
 });
+
+describe("terminalProviderFailureMessage", () => {
+  test("uses the selected provider display label in the terminal guidance", () => {
+    expect(terminalProviderFailureMessage("openai", "OpenAI")).toBe(
+      'OpenAI Provider failed. Try again or switch with "/model" and select another.',
+    );
+  });
+
+  test("falls back to the selected provider id", () => {
+    expect(terminalProviderFailureMessage("custom-provider")).toBe(
+      'custom-provider Provider failed. Try again or switch with "/model" and select another.',
+    );
+  });
+
+  test("uses a safe label when the provider id contains only control sequences", () => {
+    const message = terminalProviderFailureMessage("\u001b[31m\u001b[0m");
+    expect(message).toBe(
+      'Unknown Provider failed. Try again or switch with "/model" and select another.',
+    );
+    expect(message).not.toContain("\u001b");
+  });
+});

src/inference-error-message.ts

@@ -11,6 +11,7 @@ import {
   parseCodexUsageLimitError,
 } from "./auth/codex/usage-limit-error.js";
 import { codexProfileFromProviderName, isCodexProviderName } from "./config/codex-providers.js";
+import { stripTerminalControlSequences } from "./util/control-char-strip.js";
 import {
   gatewayOverloadUserMessage,
   isCodexShortRateLimitInferenceError,
@@ -91,6 +92,40 @@ function codexUsageLimitLine(error: InferenceErrorLike): string | undefined {
   return undefined;
 }
 
+export function terminalProviderFailureMessage(providerId: string, displayLabel?: string): string {
+  const preferred = displayLabel?.trim() || providerId;
+  const sanitized = stripTerminalControlSequences(preferred).replace(/\s+/g, " ").trim();
+  const label = sanitized.length > 0 ? sanitized : "Unknown";
+  return `${label} Provider failed. Try again or switch with "/model" and select another.`;
+}
+
+export type ResolvedProviderFailureError = Error & {
+  readonly name: "ResolvedProviderFailureError";
+  readonly diagnosticMessage: string;
+};
+
+export function createResolvedProviderFailureError(
+  providerId: string,
+  diagnosticMessage: string,
+  displayLabel?: string,
+): ResolvedProviderFailureError {
+  return Object.assign(new Error(terminalProviderFailureMessage(providerId, displayLabel)), {
+    name: "ResolvedProviderFailureError" as const,
+    diagnosticMessage,
+  });
+}
+
+export function isResolvedProviderFailureError(
+  error: unknown,
+): error is ResolvedProviderFailureError {
+  return (
+    error instanceof Error &&
+    error.name === "ResolvedProviderFailureError" &&
+    "diagnosticMessage" in error &&
+    typeof error.diagnosticMessage === "string"
+  );
+}
+
 /** One line describing the failure, falling back to the provider's own message. */
 export function inferenceErrorMessage(error: InferenceErrorLike): string {
   if (isGatewayOverloadInferenceError(error)) return gatewayOverloadUserMessage(error);

src/session/run-sink.test.ts

@@ -174,7 +174,7 @@ describe("createRunSink", () => {
     expect(failures).toEqual([{ turnIndex: 0, error: "429 rate limit" }]);
   });
 
-  test("uses unknown attribution when a fallback model fails before usage", () => {
+  test("uses unknown attribution when a retry model fails before usage", () => {
     const { captured, runSink } = attributionHarness();
 
     runSink.sink(event("inference.start", { model: "model-b" }));
@@ -203,14 +203,14 @@ describe("createRunSink", () => {
     });
   });
 
-  test("uses authoritative usage attribution for a failed fallback", () => {
+  test("uses authoritative usage attribution for a failed retry attempt", () => {
     const { captured, runSink } = attributionHarness();
 
     runSink.sink(event("inference.start", { model: "model-b" }));
     runSink.sink(
       event("inference.usage", {
         usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, thinking: 0 },
-        source: { sourceId: "fallback", provider: "provider-b", model: "model-b" },
+        source: { sourceId: "retry", provider: "provider-b", model: "model-b" },
       }),
     );
     failMessageRun(runSink);

src/session/run-sink.ts

@@ -26,8 +26,7 @@ export interface RunSinkArgs {
   // while the turn index is the collector's current in-flight turn count.
   onTurnStarted?: (info: { turnIndex: number; model: string }) => void;
   // inference.usage is the first attempt event carrying the runtime-resolved
-  // provider/model pair. It remains authoritative even when the selected source
-  // outside the reactor has not changed during fallback.
+  // provider/model pair. It remains authoritative across retry attempts.
   onTurnSourceObserved?: (info: { turnIndex: number; source: LastCycleSource }) => void;
   // Continues a resumed session's persisted run.json turn count instead of
   // restarting the collector at zero.