From 64c86ec3d2d5ababc349ac561cb070332edbd546 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Fri, 11 Sep 2026 21:57:21 -0700 Subject: [PATCH] Stop sending contextWindow as the request max_tokens --- docs/IMPLEMENTATION.md | 2 +- src/config/inference-sources.test.ts | 87 ++++++++++++++++++++++++++++ src/config/inference-sources.ts | 24 +------- 3 files changed, 90 insertions(+), 23 deletions(-) create mode 100644 src/config/inference-sources.test.ts diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index 3a01b98f2..9852857a1 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -253,7 +253,7 @@ Provider and model configuration lives in JSON settings files. The global file h `models` is always an array (single- and multi-model providers are uniform). `defaultModel` (or the first entry) is used when no model is selected. With exactly one provider configured, `defaultProvider` may be omitted. - Optional `contextWindow` (positive number, tokens) overrides the models.dev / heuristic window for that provider. `loadConfig` applies it after `resolveProvider` via `setProviderContextWindowOverrides`, keyed as `:` for every model on a provider that sets the field, plus the bare model id for the resolved provider so occupancy lookups that only have `source.model` still hit. It takes precedence over models.dev metadata and family heuristics. OAuth-projected Codex/xAI providers still drop the field: the synthetic `ProviderSettings` written by the projection overwrites the settings entry and does not copy `contextWindow`, so a hand-edited value on `codex/...` or `xai/...` is ignored. API-key providers are unaffected. + Optional `contextWindow` (positive number, tokens) overrides the models.dev / heuristic window for that provider. It sizes compaction and the status-bar meter only — it never becomes the request's `max_tokens` output budget, which stays at the shared source default on every provider branch (Codex, xAI, Go, Anthropic, Bifrost, OpenAI-compatible). `loadConfig` applies it after `resolveProvider` via `setProviderContextWindowOverrides`, keyed as `:` for every model on a provider that sets the field, plus the bare model id for the resolved provider so occupancy lookups that only have `source.model` still hit. It takes precedence over models.dev metadata and family heuristics. OAuth-projected Codex/xAI providers still drop the field: the synthetic `ProviderSettings` written by the projection overwrites the settings entry and does not copy `contextWindow`, so a hand-edited value on `codex/...` or `xai/...` is ignored. API-key providers are unaffected. Optional `tools` block to arm the outer per-tool wall-clock budget (unset leaves the watchdog unarmed): diff --git a/src/config/inference-sources.test.ts b/src/config/inference-sources.test.ts new file mode 100644 index 000000000..f2145ea4c --- /dev/null +++ b/src/config/inference-sources.test.ts @@ -0,0 +1,87 @@ +import { describe, test, expect, afterEach } from "bun:test"; +import type { ConversationTurn, InferenceOptions } from "@intx/types/runtime"; +import { SOURCE_MAX_TOKENS, type ProviderCatalogEntry } from "./index.js"; +import { + buildInferenceSourceForRef, + type BuildSourceContext, +} from "./inference-sources.js"; +import type { Settings } from "./settings.js"; +import { + buildProviderContextWindowOverrides, + contextWindowFor, + setProviderContextWindowOverrides, +} from "../provider/context-window.js"; +import { createOpenAICompatibleAdapter } from "../provider/openai-compatible-adapter.js"; + +const WINDOW = 400_000; + +function catalog(): ProviderCatalogEntry[] { + return [ + { + name: "fp", + baseURL: "https://fp.example/v1", + apiKey: "fp-key", + models: ["fp-large"], + }, + ]; +} + +function ctx(): BuildSourceContext { + return { sessionId: "sess-1", catalog: catalog() }; +} + +function settingsWithWindow(): Settings { + return { + providers: { + fp: { + baseURL: "https://fp.example/v1", + apiKey: "fp-key", + models: ["fp-large"], + contextWindow: WINDOW, + }, + }, + }; +} + +afterEach(() => { + setProviderContextWindowOverrides(undefined); +}); + +describe("contextWindow / maxTokens split (CL-7784)", () => { + test("setting contextWindow does not change the source output budget", () => { + const source = buildInferenceSourceForRef( + { provider: "fp", model: "fp-large" }, + ctx(), + settingsWithWindow(), + ); + expect(source?.defaults?.maxTokens).toBe(SOURCE_MAX_TOKENS); + }); + + test("contextWindow 400000 does not reach the wire as max_tokens 400000", () => { + const source = buildInferenceSourceForRef( + { provider: "fp", model: "fp-large" }, + ctx(), + settingsWithWindow(), + ); + const adapter = createOpenAICompatibleAdapter( + source as unknown as Parameters[0], + ); + const messages = [ + { role: "user", content: [{ type: "text", text: "hi" }] }, + ] as unknown as ConversationTurn[]; + const built = adapter.buildRequest(messages, "fp-large", { + maxTokens: source?.defaults?.maxTokens, + } as InferenceOptions); + const body = JSON.parse(built.body) as Record; + expect(body["max_tokens"]).toBe(SOURCE_MAX_TOKENS); + expect(body["max_tokens"]).not.toBe(WINDOW); + }); + + test("setting contextWindow still changes contextWindowFor", () => { + const settings = settingsWithWindow(); + setProviderContextWindowOverrides( + buildProviderContextWindowOverrides(settings.providers, "fp", "fp-large"), + ); + expect(contextWindowFor("fp:fp-large")).toBe(WINDOW); + }); +}); diff --git a/src/config/inference-sources.ts b/src/config/inference-sources.ts index 51a8f3ed0..fa034ae68 100644 --- a/src/config/inference-sources.ts +++ b/src/config/inference-sources.ts @@ -14,7 +14,6 @@ import { resolveSessionEffort, type ReasoningEffort, } from "../provider/reasoning-effort.js"; -import { SOURCE_MAX_TOKENS } from "./index.js"; import { isOpenCodeGoProvider } from "../../packages/opencode-go/src/index.js"; export interface BuildSourceContext { @@ -37,16 +36,6 @@ function catalogEntry( return catalog.find((e) => e.name === provider); } -function maxTokensFor( - settings: Settings | undefined, - provider: string, - _model: string, -): number { - const cw = settings?.providers[provider]?.contextWindow; - if (typeof cw === "number" && cw > 0) return cw; - return SOURCE_MAX_TOKENS; -} - export function buildInferenceSourceForRef( ref: ProviderRef, ctx: BuildSourceContext, @@ -57,7 +46,6 @@ export function buildInferenceSourceForRef( const baseURL = entry?.baseURL ?? providerSettings?.baseURL; if (baseURL === undefined) return null; - const maxTokens = maxTokensFor(settings, ref.provider, ref.model); const configured = ref.reasoningEffort ?? ctx.reasoningEffort; const effort = configured !== undefined @@ -123,7 +111,7 @@ export function buildInferenceSourceForRef( }); } if (entry?.bifrostVirtualKey === true) { - const src = buildBifrostSource({ + return buildBifrostSource({ id: ref.provider, baseURL, ...(entry?.apiKey !== undefined @@ -134,13 +122,9 @@ export function buildInferenceSourceForRef( model: ref.model, ...(effort !== undefined ? { reasoningEffort: effort } : {}), }); - return { - ...src, - defaults: { ...src.defaults, maxTokens }, - }; } - const src = buildOpenAISource({ + return buildOpenAISource({ id: ref.provider, baseURL, ...(entry?.apiKey !== undefined @@ -151,10 +135,6 @@ export function buildInferenceSourceForRef( model: ref.model, ...(effort !== undefined ? { reasoningEffort: effort } : {}), }); - return { - ...src, - defaults: { ...src.defaults, maxTokens }, - }; } function buildSourceBundle(args: {