🤖 feat: validated make update-models refresh + full catalog in Treat as

.github/workflows/update-models.yml

@@ -0,0 +1,103 @@
+# Weekly refresh of the vendored LiteLLM model catalog
+# (src/common/utils/tokens/models.json) via scripts/update_models.ts, which
+# validates pricing shape and curated-model coverage before writing (#3727).
+# Opens or updates a bot PR only when the data changed.
+# SETUP: reuses the auto-cleanup GitHub App credentials (XUM_APP_ID and
+# XUM_APP_PRIVATE_KEY) so the PR triggers CI.
+
+name: Update Models
+
+on:
+  schedule:
+    - cron: "0 6 * * 1" # Weekly, Mondays 6 AM UTC
+  workflow_dispatch: {}
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: update-models
+  cancel-in-progress: true
+
+jobs:
+  update-models:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check required secrets
+        id: precheck
+        env:
+          XUM_APP_ID: ${{ secrets.XUM_APP_ID }}
+          XUM_APP_PRIVATE_KEY: ${{ secrets.XUM_APP_PRIVATE_KEY }}
+        run: |
+          if [ -z "$XUM_APP_ID" ] || [ -z "$XUM_APP_PRIVATE_KEY" ]; then
+            echo "Skipping (missing XUM_APP_ID / XUM_APP_PRIVATE_KEY)."
+            echo "enabled=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "enabled=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0
+        id: app-token
+        if: ${{ steps.precheck.outputs.enabled == 'true' }}
+        with:
+          # Use a GitHub App token so PR events trigger CI. PRs opened via
+          # GITHUB_TOKEN intentionally do not trigger other workflows.
+          app-id: ${{ secrets.XUM_APP_ID }}
+          private-key: ${{ secrets.XUM_APP_PRIVATE_KEY }}
+
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        if: ${{ steps.precheck.outputs.enabled == 'true' }}
+        with:
+          # Pin to main so a workflow_dispatch from a feature branch cannot leak
+          # unrelated commits into the shared bot/update-models branch.
+          ref: refs/heads/main
+          token: ${{ steps.app-token.outputs.token }}
+
+      - uses: oven-sh/setup-bun@b7a1c7ccf290d58743029c4f6903da283811b979 # v2.1.0
+        if: ${{ steps.precheck.outputs.enabled == 'true' }}
+        with:
+          bun-version: 1.3.5
+
+      - name: Update models.json
+        if: ${{ steps.precheck.outputs.enabled == 'true' }}
+        # Go through the Makefile target (which installs dependencies first) so
+        # automation and local refreshes share one entry point.
+        run: make update-models
+
+      - name: Push branch and open PR if changed
+        if: ${{ steps.precheck.outputs.enabled == 'true' }}
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+        run: |
+          if git diff --quiet -- src/common/utils/tokens/models.json; then
+            echo "models.json already up to date; no PR needed."
+            exit 0
+          fi
+
+          git config user.name "mux-bot[bot]"
+          git config user.email "264182336+mux-bot[bot]@users.noreply.github.com"
+          git checkout -B bot/update-models
+          git add src/common/utils/tokens/models.json
+          # Attribution footer per .xum/skills/pull-requests: automation
+          # commits use the workflow marker instead of model/thinking/cost.
+          footer="---
+
+          _Automated by the \`update-models\` workflow (GitHub Actions)._
+
+          <!-- xum-attribution: workflow=update-models -->"
+          git commit -m "🤖 chore: refresh models.json from LiteLLM" -m "$footer"
+          git push --force origin bot/update-models
+
+          open_prs=$(gh pr list --head bot/update-models --state open --json number --jq length)
+          if [ "$open_prs" = "0" ]; then
+            gh pr create \
+              --title "🤖 chore: refresh models.json from LiteLLM" \
+              --body "Automated weekly refresh of the vendored LiteLLM model catalog via \`make update-models\` (#3727).
+
+          $footer" \
+              --base main \
+              --head bot/update-models
+          else
+            echo "Existing open PR for bot/update-models updated by push."
+          fi

.xum/skills/pull-requests/SKILL.md

@@ -19,6 +19,16 @@ _Generated with `xum` • Model: `<modelString>` • Thinking: `<thinkingLevel>`
 
 Always check `$XUM_MODEL_STRING`, `$XUM_THINKING_LEVEL`, and `$XUM_COSTS_USD` via bash before creating or updating PRs—include them in the footer if set.
 
+Scheduled automation (non-AI) commits and PRs from GitHub Actions workflows use this footer instead, since model/thinking/cost do not apply:
+
+```md
+---
+
+_Automated by the `<workflow-name>` workflow (GitHub Actions)._
+
+<!-- xum-attribution: workflow=<workflow-name> -->
+```
+
 ## Lifecycle Rules
 
 - Before submitting a PR, ensure the branch name reflects the work and the base branch is correct.

Makefile

@@ -219,6 +219,20 @@ start: node_modules/.installed build-main build-preload build-static ## Build an
 ## Build targets (can run in parallel)
 build: node_modules/.installed src/version.ts build-renderer build-main build-preload build-icons build-static ## Build all targets
 
+.PHONY: update-models
+update-models: node_modules/.installed ## Fetch latest LiteLLM model data, validate it, update models.json if changed
+	@bun scripts/update_models.ts
+
+# #3727: `make build UPDATE_MODELS=1` refreshes the vendored models.json before
+# building; plain `make build` stays network-free and reproducible. The refresh
+# must be a prerequisite of every catalog-consuming bundle (not just `build`) so
+# parallel make cannot bundle the old catalog, and the phony prerequisite forces
+# those bundles stale because models.json is not part of $(TS_SOURCES).
+ifeq ($(UPDATE_MODELS),1)
+build: update-models
+build-renderer dist/cli/index.js dist/cli/api.mjs: update-models
+endif
+
 build-main: node_modules/.installed dist/cli/index.js dist/cli/api.mjs ## Build main process
 
 BUILTIN_AGENTS_GENERATED := src/node/services/agentDefinitions/builtInAgentContent.generated.ts
@@ -235,7 +249,9 @@ $(BUILTIN_SKILLS_GENERATED): $(BUILTIN_SKILL_SOURCES) $(DOCS_SOURCES) scripts/ge
 $(WORKFLOW_RUNTIME_SOURCES_GENERATED): $(WORKFLOW_RUNTIME_SOURCES) scripts/gen_workflow_runtime_sources.ts
 	@bun scripts/gen_workflow_runtime_sources.ts
 
-dist/cli/index.js: src/cli/index.ts src/desktop/main.ts src/cli/server.ts src/version.ts tsconfig.main.json tsconfig.json $(TS_SOURCES) $(BUILTIN_AGENTS_GENERATED) $(BUILTIN_SKILLS_GENERATED) $(BUILTIN_WORKFLOWS_GENERATED) $(WORKFLOW_RUNTIME_SOURCES_GENERATED)
+# models.json is bundled but not in $(TS_SOURCES); without this prerequisite a
+# catalog-only refresh would leave a stale main bundle (#3727).
+dist/cli/index.js: src/cli/index.ts src/desktop/main.ts src/cli/server.ts src/version.ts tsconfig.main.json tsconfig.json $(TS_SOURCES) src/common/utils/tokens/models.json $(BUILTIN_AGENTS_GENERATED) $(BUILTIN_SKILLS_GENERATED) $(BUILTIN_WORKFLOWS_GENERATED) $(WORKFLOW_RUNTIME_SOURCES_GENERATED)
 	@echo "Building main process..."
 	@NODE_ENV=production $(TSGO) -p tsconfig.main.json
 	@NODE_ENV=production bun x tsc-alias -p tsconfig.main.json

scripts/update_models.ts

@@ -1,62 +1,25 @@
 #!/usr/bin/env bun
 
 /**
- * Downloads the latest model prices and context window data from LiteLLM
- * and saves the subset Xum consumes to src/common/utils/tokens/models.json.
+ * Refreshes src/common/utils/tokens/models.json from LiteLLM. The transform and
+ * validation logic lives in src/common/utils/tokens/updateModelsData.ts so unit
+ * tests cover it. Writes only when the validated content changed, keeping
+ * `make update-models`, `make build UPDATE_MODELS=1`, and the scheduled
+ * update-models workflow idempotent (#3727).
  */
 
+import {
+  pruneModelData,
+  sanitizePricing,
+  serializeModelData,
+  validateModelData,
+  type ModelCatalogData,
+} from "../src/common/utils/tokens/updateModelsData";
+
 const LITELLM_URL =
   "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json";
 const OUTPUT_PATH = "src/common/utils/tokens/models.json";
 
-const RETAINED_FIELDS = [
-  "max_input_tokens",
-  "max_output_tokens",
-  "input_cost_per_token",
-  "output_cost_per_token",
-  "output_cost_per_image_token",
-  "input_cost_per_token_above_200k_tokens",
-  "output_cost_per_token_above_200k_tokens",
-  "cache_creation_input_token_cost",
-  "cache_creation_input_token_cost_above_200k_tokens",
-  "cache_read_input_token_cost",
-  "cache_read_input_token_cost_above_200k_tokens",
-  "tiered_pricing_threshold_tokens",
-  "mode",
-  "litellm_provider",
-  "supports_pdf_input",
-  "supports_vision",
-  "supports_audio_input",
-  "supports_video_input",
-  "max_pdf_size_mb",
-] as const;
-
-function pruneModelData(data: unknown): Record<string, Record<string, unknown>> {
-  if (!data || typeof data !== "object" || Array.isArray(data)) {
-    throw new Error("Expected LiteLLM model metadata object");
-  }
-
-  const pruned: Record<string, Record<string, unknown>> = {};
-  for (const [modelId, rawMetadata] of Object.entries(data)) {
-    if (!rawMetadata || typeof rawMetadata !== "object" || Array.isArray(rawMetadata)) {
-      continue;
-    }
-
-    const metadata = rawMetadata as Record<string, unknown>;
-    const retained: Record<string, unknown> = {};
-    // Keep models.json small: Xum only reads pricing, token limits, provider, mode, and media
-    // capability fields, while upstream LiteLLM ships many provider-specific fields we never use.
-    for (const field of RETAINED_FIELDS) {
-      if (metadata[field] !== undefined) {
-        retained[field] = metadata[field];
-      }
-    }
-    pruned[modelId] = retained;
-  }
-
-  return pruned;
-}
-
 async function updateModels() {
   console.log(`Fetching model data from ${LITELLM_URL}...`);
 
@@ -66,12 +29,37 @@ async function updateModels() {
     throw new Error(`Failed to fetch model data: ${response.status} ${response.statusText}`);
   }
 
-  const data = pruneModelData(await response.json());
+  const sanitized = sanitizePricing(pruneModelData(await response.json()));
+  if (sanitized.droppedModelIds.length > 0) {
+    console.warn(
+      `Dropped ${sanitized.droppedModelIds.length} entries with invalid pricing: ` +
+        sanitized.droppedModelIds.slice(0, 10).join(", ")
+    );
+  }
+
+  const existing = await Bun.file(OUTPUT_PATH)
+    .text()
+    .catch(() => null);
+  // Validate against the vendored catalog so a truncated upstream response, a
+  // field rename, or a targeted repricing cannot silently degrade known models.
+  let baseline: ModelCatalogData | undefined;
+  if (existing !== null) {
+    try {
+      baseline = JSON.parse(existing) as ModelCatalogData;
+    } catch {
+      console.warn(`Could not parse existing ${OUTPUT_PATH}; skipping baseline checks`);
+    }
+  }
+  validateModelData(sanitized, baseline);
 
-  console.log(`Writing model data to ${OUTPUT_PATH}...`);
-  await Bun.write(OUTPUT_PATH, `${JSON.stringify(data, null, 2)}\n`);
+  const serialized = serializeModelData(sanitized.catalog);
+  if (existing === serialized) {
+    console.log("✓ models.json already up to date");
+    return;
+  }
 
-  console.log("✓ Model data updated successfully");
+  await Bun.write(OUTPUT_PATH, serialized);
+  console.log(`✓ Updated ${OUTPUT_PATH} (${Object.keys(sanitized.catalog).length} models)`);
 }
 
 updateModels().catch((error) => {

src/browser/features/Settings/Components/SearchableModelSelect.tsx

@@ -6,6 +6,10 @@ import { stopKeyboardPropagation } from "@/browser/utils/events";
 import { cn } from "@/common/lib/utils";
 import { getModelName, getModelProvider } from "@/common/utils/ai/models";
 
+// The full model catalog is ~2k entries; rendering every row makes the popover
+// janky, so cap the list and prompt the user to narrow the search instead.
+const MAX_RENDERED_MODELS = 200;
+
 /** Searchable model dropdown with keyboard navigation */
 export function SearchableModelSelect(props: {
   value: string;
@@ -34,6 +38,9 @@ export function SearchableModelSelect(props: {
       model.toLowerCase().includes(searchLower) ||
       (getModelName(model)?.toLowerCase().includes(searchLower) ?? false)
   );
+  const hiddenModelCount = Math.max(0, filteredModels.length - MAX_RENDERED_MODELS);
+  const visibleModels =
+    hiddenModelCount > 0 ? filteredModels.slice(0, MAX_RENDERED_MODELS) : filteredModels;
 
   // Build list of all selectable items (empty option + filtered models)
   const items: Array<{ value: string; label: string; provider?: string; isMuted?: boolean }> = [];
@@ -44,7 +51,7 @@ export function SearchableModelSelect(props: {
       isMuted: true,
     });
   }
-  for (const model of filteredModels) {
+  for (const model of visibleModels) {
     items.push({
       value: model,
       label: getModelName(model) ?? model,
@@ -180,6 +187,11 @@ export function SearchableModelSelect(props: {
               </button>
             ))
           )}
+          {hiddenModelCount > 0 && (
+            <div className="text-muted px-2 py-1 text-center text-[10px]">
+              +{hiddenModelCount} more, keep typing to filter
+            </div>
+          )}
         </div>
       </PopoverContent>
     </Popover>

src/browser/features/Settings/Sections/ModelsSection.tsx

@@ -19,6 +19,7 @@ import { useMinThinkingLevels } from "@/browser/hooks/useMinThinkingLevels";
 import { usePersistedState } from "@/browser/hooks/usePersistedState";
 import { useProvidersConfig } from "@/browser/hooks/useProvidersConfig";
 import { KNOWN_MODELS } from "@/common/constants/knownModels";
+import { listModelCatalogIds } from "@/common/utils/tokens/modelCatalog";
 import { isCodexOauthRequiredModelId } from "@/common/constants/codexOAuth";
 import { usePolicy } from "@/browser/contexts/PolicyContext";
 import {
@@ -160,12 +161,10 @@ export function ModelsSection() {
   // cross-hook timing mismatches while settings are loading/refetching.
   const codexOauthConfigured = config?.openai?.codexOauthSet === true;
 
-  // "Treat as" dropdown should only list known models — custom models don't have
-  // the metadata (pricing, context window, tokenizer) that mapping inherits.
-  // Static list — React Compiler handles memoization; no manual useMemo needed.
-  const knownModelIds = Object.values(KNOWN_MODELS)
-    .map((model) => model.id)
-    .sort();
+  // "Treat as" targets must carry the metadata (pricing, context window) that
+  // mapping inherits: any model in the token catalog qualifies, not just the
+  // curated KNOWN_MODELS list (#3727).
+  const treatAsModelIds = listModelCatalogIds();
 
   // Check if a model already exists (for duplicate prevention)
   const modelExists = useCallback(
@@ -473,7 +472,7 @@ export function ModelsSection() {
                       editMappedToModel={isModelEditing ? editing.mappedToModel : undefined}
                       editAutofocus={isModelEditing ? editing.focus : undefined}
                       customContextWindowTokens={model.contextWindowTokens}
-                      allModels={knownModelIds}
+                      allModels={treatAsModelIds}
                       editError={isModelEditing ? error : undefined}
                       saving={false}
                       hasActiveEdit={editing !== null}

src/common/constants/knownModels.test.ts

@@ -5,29 +5,16 @@
 import { describe, test, expect } from "@jest/globals";
 import { KNOWN_MODELS, MODEL_ABBREVIATIONS } from "@/common/constants/knownModels";
 import modelsJson from "@/common/utils/tokens/models.json";
-import { modelsExtra } from "@/common/utils/tokens/models-extra";
+import { findMissingKnownModels } from "@/common/utils/tokens/updateModelsData";
 
 describe("Known Models Integration", () => {
   test("all known models exist in token metadata", () => {
-    const missingModels: string[] = [];
-
-    for (const [key, model] of Object.entries(KNOWN_MODELS)) {
-      const modelId = model.providerModelId;
-
-      // xAI and Moonshot models are provider-prefixed in token metadata.
-      const lookupKey =
-        model.provider === "xai" || model.provider === "moonshotai"
-          ? `${model.provider}/${modelId}`
-          : modelId;
-      if (!(lookupKey in modelsJson) && !(lookupKey in modelsExtra) && !(modelId in modelsExtra)) {
-        missingModels.push(`${key}: ${model.provider}:${modelId}`);
-      }
-    }
+    const missingModels = findMissingKnownModels(modelsJson);
 
     if (missingModels.length > 0) {
       throw new Error(
         `The following known models are missing from token metadata:\n${missingModels.join("\n")}\n\n` +
-          `Run 'bun scripts/update_models.ts' to refresh models.json from LiteLLM.`
+          `Run 'make update-models' to refresh models.json from LiteLLM.`
       );
     }
   });