Remove automatic provider failover (#747)

* Remove automatic provider failover

* Harden provider failure handling

* Keep credential failures on login copy

Auth and OAuth-refresh send failures were still suggesting /model.
Keep that copy on login, drop the fused task coverage after spawn/wait
became the only path, and document the no-failover behavior.

* Format the spawn auth-failure reason

Author: TheGreatAxios

CHANGELOG.md

@@ -31,6 +31,8 @@ parallel copies under `docs/` or `scripts/notes/`. At cut time: rename
 
 - Cancelling a `task` or `wait_agents` worker reports wait status `interrupted`,
   not `failed`.
+- Inference no longer fails over to a backup provider. A selected-provider
+  failure stays on that provider; switch with `/model`.
 
 ### Fixed
 
@@ -46,6 +48,8 @@ parallel copies under `docs/` or `scripts/notes/`. At cut time: rename
   when a followup is already in flight.
 - `interrupt_agent` flips the wait mailbox so soft interrupt unblocks
   `wait_agents` while the background run is still in flight.
+- Credential-refresh and auth send failures tell the user to log in again
+  instead of suggesting `/model`.
 
 ## [0.3.11] - 2026-08-31
 

docs/TELEMETRY.md

@@ -169,11 +169,11 @@ settles an unresolved turn once as an unsampled terminal failure. Attribution
 uses the latest `inference.usage` source, the first lifecycle payload carrying
 the runtime-resolved provider/model pair for an attempt. Each `inference.start`
 clears that authoritative source and records the newly attempted model. If a
-fallback fails before usage exposes its source, telemetry retains that actual
-model but uses the fixed `unknown` provider/source bucket rather than attributing
-it to the previously selected provider. When the attempted model still matches
-the selected source, that full source remains valid. Therefore a parent turn
-emits at most one terminal `$ai_generation`, including retry and failover paths.
+retry fails before usage exposes its source, telemetry retains that attempted
+model but uses the fixed `unknown` provider/source bucket. When the attempted
+model still matches the selected source, that full source remains valid.
+Therefore a parent turn emits at most one terminal `$ai_generation`, including
+retry paths.
 
 ## What's never collected
 

src/config/inference-sources.ts

@@ -13,46 +13,20 @@ import type { Settings } from "./settings.js";
 import { resolveSessionEffort, type ReasoningEffort } from "../provider/reasoning-effort.js";
 import { SOURCE_MAX_TOKENS } from "./index.js";
 import { isOpenCodeGoProvider } from "../../packages/opencode-go/src/index.js";
-import { resolveDefaultModel } from "./providers.js";
 
 export interface BuildSourceContext {
   sessionId: string;
   reasoningEffort?: ReasoningEffort;
   catalog: readonly ProviderCatalogEntry[];
 }
 
-// A resolved provider+model, with optional reasoningEffort — the unit both
-// the primary source and its backups are built from.
+// A resolved provider+model with optional reasoning effort.
 export interface ProviderRef {
   provider: string;
   model: string;
   reasoningEffort?: ReasoningEffort;
 }
 
-function refKey(ref: ProviderRef): string {
-  return `${ref.provider}\0${ref.model}`;
-}
-
-// Every other configured provider, one model each, so a primary source that
-// fails to build (bad credentials, missing baseURL) still has somewhere to
-// fall back to. Order follows settings.providers; providers already covered
-// by `existing` are skipped.
-function backupRefsFromSettings(
-  settings: Settings,
-  existing: readonly ProviderRef[],
-): ProviderRef[] {
-  const seenProviders = new Set(existing.map((r) => r.provider));
-  const tail: ProviderRef[] = [];
-  for (const [provider, p] of Object.entries(settings.providers)) {
-    if (seenProviders.has(provider)) continue;
-    const model = resolveDefaultModel(p);
-    if (model === undefined || model.length === 0) continue;
-    seenProviders.add(provider);
-    tail.push({ provider, model });
-  }
-  return tail;
-}
-
 function catalogEntry(
   catalog: readonly ProviderCatalogEntry[],
   provider: string,
@@ -170,38 +144,6 @@ export function buildInferenceSourceForRef(
   };
 }
 
-export function buildSourcesFromRefs(
-  refs: readonly ProviderRef[],
-  ctx: BuildSourceContext,
-  settings: Settings | undefined,
-): InferenceSource[] {
-  const out: InferenceSource[] = [];
-  const seenIds = new Set<string>();
-  for (const ref of refs) {
-    let src: InferenceSource | null;
-    try {
-      src = buildInferenceSourceForRef(ref, ctx, settings);
-    } catch {
-      // A leftover sibling URL (e.g. Custom `/api/tags`) must not take down the
-      // whole bundle. Head failure is re-checked in `buildSourceBundle`.
-      continue;
-    }
-    if (src === null) continue;
-    if (seenIds.has(src.id)) continue;
-    seenIds.add(src.id);
-    out.push(src);
-  }
-  return out;
-}
-
-export function prependActiveRef(refs: readonly ProviderRef[], active: ProviderRef): ProviderRef[] {
-  const without = refs.filter((r) => refKey(r) !== refKey(active));
-  return [active, ...without];
-}
-
-// Builds the primary source for `head` plus one backup per other configured
-// provider, so a mid-run failure (bad credentials, dropped connection) has
-// somewhere else to go. `head` always wins as defaultSource when it builds.
 function buildSourceBundle(args: {
   settings: Settings | undefined;
   catalog: readonly ProviderCatalogEntry[];
@@ -215,30 +157,13 @@ function buildSourceBundle(args: {
     ...(args.reasoningEffort !== undefined ? { reasoningEffort: args.reasoningEffort } : {}),
   };
 
-  const refs =
-    args.settings !== undefined
-      ? prependActiveRef(backupRefsFromSettings(args.settings, [args.head]), args.head)
-      : [args.head];
-
-  const sources = buildSourcesFromRefs(refs, ctx, args.settings);
-  const defaultId = args.head.provider;
-  const hasDefault = sources.some((s) => s.id === defaultId);
-  if (!hasDefault) {
-    let fallback: InferenceSource | null;
-    try {
-      fallback = buildInferenceSourceForRef(args.head, ctx, args.settings);
-    } catch (error) {
-      throw new Error(`No inference source for provider "${defaultId}"`, { cause: error });
-    }
-    if (fallback === null) {
-      throw new Error(`No inference source for provider "${defaultId}"`);
-    }
-    return { sources: [fallback, ...sources], defaultSource: fallback.id };
+  const source = buildInferenceSourceForRef(args.head, ctx, args.settings);
+  if (source === null) {
+    throw new Error(
+      `Unable to build inference source for selected provider "${args.head.provider}"`,
+    );
   }
-  return {
-    sources,
-    defaultSource: defaultId,
-  };
+  return { sources: [source], defaultSource: source.id };
 }
 
 export function buildMainSessionSources(args: {

src/exec/runner.ts

@@ -13,12 +13,7 @@ import { noopAuditStore, permissiveAuthorize } from "@intx/agent/testing";
 import { getLogger } from "@intx/log";
 import { createOptimizedContextStore } from "../session/optimized-context-store.js";
 import { type } from "arktype";
-import {
-  buildCodexSource,
-  buildOpenAISource,
-  buildXaiSource,
-  type Config,
-} from "../config/index.js";
+import { type Config } from "../config/index.js";
 import {
   loadLocalSettings,
   resolveLocalSettingsPath,
@@ -67,6 +62,11 @@ import { createAgentToolset, type AgentToolset, type OperatorResult } from "../a
 import { createAgentWithLiveToolDispatch } from "../agent/live-tool-dispatch.js";
 import { liveTelemetry } from "../telemetry/singleton.js";
 import { createTurnObserver } from "../telemetry/ai-observability.js";
+import {
+  CREDENTIAL_FAILURE_USER_MESSAGE,
+  isResolvedProviderFailureError,
+  terminalProviderFailureMessage,
+} from "../inference-error-message.js";
 import { collectToolPlugins, resolveToolPlugins } from "../plugins/tool-plugins.js";
 import {
   expandExistingPluginMembers,
@@ -120,6 +120,35 @@ export function formatCaughtError(err: unknown): string {
   return err instanceof Error ? err.message : String(err);
 }
 
+const SELECTED_PROVIDER_FAILURE = "SelectedProviderFailure";
+
+export async function refreshSelectedProviderCredential<T>(refresh: () => Promise<T>): Promise<T> {
+  try {
+    return await refresh();
+  } catch (cause) {
+    const error = new Error(formatCaughtError(cause), { cause });
+    error.name = SELECTED_PROVIDER_FAILURE;
+    throw error;
+  }
+}
+
+export function execUserFailureMessage(
+  config: Config,
+  err: unknown,
+  providerFailureObserved: boolean,
+): string {
+  if (err instanceof Error && err.name === SELECTED_PROVIDER_FAILURE) {
+    return CREDENTIAL_FAILURE_USER_MESSAGE;
+  }
+  if (providerFailureObserved || isResolvedProviderFailureError(err)) {
+    return terminalProviderFailureMessage(
+      config.providerName,
+      config.settings?.providers[config.providerName]?.name,
+    );
+  }
+  return formatCaughtError(err);
+}
+
 /**
  * Headless analogue of TUI `runtime-shutdown`: abort live workers, then close
  * the primary agent and dispose the toolset. `cancelAll` is fire-and-forget —
@@ -297,6 +326,7 @@ export async function runExec(config: Config): Promise<ExecResult> {
   let finalized = false;
   let turnsUsed = 0;
   let runSink: RunSink | null = null;
+  let providerFailureObserved = false;
 
   const persist = async (
     status: "running" | "done" | "failed" | "cancelled",
@@ -586,56 +616,14 @@ export async function runExec(config: Config): Promise<ExecResult> {
 
     const initialCodexProfile = codexProfileFromProviderName(config.providerName);
     const initialXaiProfile = xaiProfileFromProviderName(config.providerName);
-    const initialCodexAccountId = config.providers.find(
-      (p) => p.name === config.providerName,
-    )?.codexAccountId;
-
-    const buildOpenAICompatibleInitialSource = (): InferenceSource =>
-      buildOpenAISource({
-        id: config.providerName,
-        baseURL: config.baseURL,
-        apiKey: config.apiKey,
-        model: config.model,
-        ...(config.reasoningEffort !== undefined
-          ? { reasoningEffort: config.reasoningEffort }
-          : {}),
-      });
-
-    const buildSessionSources = (): { sources: InferenceSource[]; defaultSource: string } =>
-      buildSessionSourcesFromConfig(config, sessionId);
-
-    const initialBundle = buildSessionSources();
+    const initialBundle = buildSessionSourcesFromConfig(config, sessionId);
     const liveSources = initialBundle.sources;
     const liveDefaultSource = initialBundle.defaultSource;
-
-    const buildInitialSourceFallback = (): InferenceSource =>
-      initialCodexProfile !== undefined
-        ? buildCodexSource({
-            id: config.providerName,
-            apiKey: config.apiKey,
-            model: config.model,
-            sessionId,
-            ...(initialCodexAccountId !== undefined ? { accountId: initialCodexAccountId } : {}),
-            ...(config.reasoningEffort !== undefined
-              ? { reasoningEffort: config.reasoningEffort }
-              : {}),
-          })
-        : initialXaiProfile !== undefined
-          ? buildXaiSource({
-              id: config.providerName,
-              apiKey: config.apiKey,
-              model: config.model,
-              sessionId,
-              ...(config.reasoningEffort !== undefined
-                ? { reasoningEffort: config.reasoningEffort }
-                : {}),
-            })
-          : buildOpenAICompatibleInitialSource();
-
-    let liveSource: InferenceSource =
-      liveSources.find((s) => s.id === liveDefaultSource) ??
-      liveSources[0] ??
-      buildInitialSourceFallback();
+    const selectedSource = liveSources[0];
+    if (selectedSource === undefined) {
+      throw new Error("Selected inference source was not assembled");
+    }
+    let liveSource: InferenceSource = selectedSource;
 
     // Refresh pinned Codex instructions before first inference, same as the
     // TUI path. Best-effort: a network failure falls back to the disk cache
@@ -651,15 +639,19 @@ export async function runExec(config: Config): Promise<ExecResult> {
 
     // Refresh OAuth tokens before first inference when starting on codex/xai.
     if (initialCodexProfile !== undefined) {
-      const { access } = await getValidCodexToken(initialCodexProfile);
+      const { access } = await refreshSelectedProviderCredential(() =>
+        getValidCodexToken(initialCodexProfile),
+      );
       liveSource = { ...liveSource, apiKey: access };
       liveSubAgentProvider.current = {
         ...liveSubAgentProvider.current,
         apiKey: access,
       };
     }
     if (initialXaiProfile !== undefined) {
-      const { access } = await getValidXaiToken(initialXaiProfile);
+      const { access } = await refreshSelectedProviderCredential(() =>
+        getValidXaiToken(initialXaiProfile),
+      );
       liveSource = { ...liveSource, apiKey: access };
       liveSubAgentProvider.current = {
         ...liveSubAgentProvider.current,
@@ -762,6 +754,11 @@ export async function runExec(config: Config): Promise<ExecResult> {
     // its partial output in partial.jsonl instead of vanishing.
     const cycleRecorder = createCycleTextRecorder(() => workdir);
     const sink = (event: ReactorEmittedEvent): void => {
+      if (event.type === "inference.start" || event.type === "inference.done") {
+        providerFailureObserved = false;
+      } else if (event.type === "inference.error") {
+        providerFailureObserved = true;
+      }
       liveSink.sink(event);
       cycleRecorder.handleEvent(event);
       if (event.type === "inference.text.delta") {
@@ -881,17 +878,24 @@ export async function runExec(config: Config): Promise<ExecResult> {
     });
 
     if (!sendCompleted || runError !== undefined || summaryStatus === "failed") {
-      const message =
+      const diagnosticMessage =
         runError ??
         (summaryStatus === "cancelled" ? "run cancelled before completion" : "run failed");
-      stderr.write(`Error: ${message}\n`);
+      const userMessage =
+        summaryStatus === "failed"
+          ? terminalProviderFailureMessage(
+              config.providerName,
+              config.settings?.providers[config.providerName]?.name,
+            )
+          : diagnosticMessage;
+      stderr.write(`Error: ${userMessage}\n`);
       const persistStatus = summaryStatus === "cancelled" ? "cancelled" : "failed";
-      await persist(persistStatus, { error: message });
+      await persist(persistStatus, { error: diagnosticMessage });
       return {
         exitCode: 1,
         sessionId,
         text: textOut,
-        error: message,
+        error: userMessage,
         status: summaryStatus,
         durationMs: finishedAt - startedAt,
         turnsUsed: runSink.getTurnCount(),
@@ -916,15 +920,16 @@ export async function runExec(config: Config): Promise<ExecResult> {
       model: config.model,
     };
   } catch (err) {
-    const message = err instanceof Error ? err.message : String(err);
-    logger.error("exec failed: {error}", { error: message });
-    stderr.write(`Error: ${message}\n`);
-    await persist("failed", { error: message });
+    const diagnosticMessage = formatCaughtError(err);
+    logger.error("exec failed: {error}", { error: diagnosticMessage });
+    const userMessage = execUserFailureMessage(config, err, providerFailureObserved);
+    stderr.write(`Error: ${userMessage}\n`);
+    await persist("failed", { error: diagnosticMessage });
     return {
       exitCode: 1,
       sessionId,
       text: textOut,
-      error: message,
+      error: userMessage,
       status: "failed",
       durationMs: Date.now() - startedAt,
       turnsUsed: runSink?.getTurnCount() ?? turnsUsed,