From 2a0def10fded08cc9389cdf2f6bc6a88a05ccb71 Mon Sep 17 00:00:00 2001 From: Lubomir Jacisko Date: Fri, 23 Oct 2026 14:24:28 +0200 Subject: [PATCH] feat(models): support responses custom endpoints Require explicit API selection when multiple routes answer: route availability does not establish model compatibility. --- .../manage/tabs/settings/ModelsSettings.tsx | 32 +-- src/server/ipc/guard.ts | 2 +- src/server/pi/models-config.ts | 99 +++---- src/server/workspace/settings.ts | 15 +- src/shared/types.ts | 8 +- tests/unit/models-config.test.ts | 101 +++++-- tests/unit/responses-integration.test.ts | 248 ++++++++++++++++++ tests/unit/settings.test.ts | 19 +- 8 files changed, 433 insertions(+), 91 deletions(-) create mode 100644 tests/unit/responses-integration.test.ts diff --git a/src/renderer/manage/tabs/settings/ModelsSettings.tsx b/src/renderer/manage/tabs/settings/ModelsSettings.tsx index 500d092f..d4df03ca 100644 --- a/src/renderer/manage/tabs/settings/ModelsSettings.tsx +++ b/src/renderer/manage/tabs/settings/ModelsSettings.tsx @@ -1735,11 +1735,8 @@ function LocalServerAddForm({ const [apiKey, setApiKey] = useState(''); const [models, setModels] = useState(''); const [customName, setCustomName] = useState(''); - // API flavor for the Custom endpoint. Ollama/LM Studio ignore it — always - // openai-completions. `null` = auto-detect (default): the Test probe - // classifies which chat route the endpoint exposes and snaps the dropdown to - // that value. Enable requires a concrete flavor; the button stays disabled - // until Test succeeds or the user picks one explicitly. + // Custom endpoints need a concrete flavor; ambiguous detection leaves Enable disabled. + // Ollama/LM Studio always use openai-completions. const [api, setApi] = useState(null); const [testing, setTesting] = useState(false); const [test, setTest] = useState(null); @@ -1809,10 +1806,7 @@ function LocalServerAddForm({ async function runTest() { const request = testGateRef.current.begin(); - // Auto-detect on Custom sends `undefined` so main runs the OPTIONS probe on - // both chat routes; a concrete pick from the dropdown short-circuits that. - // Ollama/LM Studio always name their flavor: they can't be anything but - // OpenAI-compatible, so sending it spares them two pointless round-trips. + // Explicit flavors bypass classification, including for Ollama/LM Studio. const probeApi: LocalProviderApi | undefined = server === 'custom' ? (api ?? undefined) : 'openai-completions'; const tested = localProbeTarget(server, baseUrl, apiKey, probeApi ?? 'openai-completions'); @@ -1885,7 +1879,11 @@ function LocalServerAddForm({ } const flavorLabel = (a: LocalProviderApi): string => - a === 'anthropic-messages' ? 'Anthropic Messages' : 'OpenAI Chat Completions'; + a === 'anthropic-messages' + ? 'Anthropic Messages' + : a === 'openai-responses' + ? 'OpenAI Responses' + : 'OpenAI Chat Completions'; const testLabel = test ? test.ok ? `${test.models?.length ?? 0} model${(test.models?.length ?? 0) === 1 ? '' : 's'} found` + @@ -1905,11 +1903,14 @@ function LocalServerAddForm({ knows. Ollama and LM Studio run on this machine and report their own models — LM Studio loads one on first use, so its first reply can take a while. Custom endpoint is any other URL: a gateway, a proxy, a server elsewhere on your network. Leave the flavor on{' '} - Auto-detect and Test connection classifies which chat route the endpoint exposes - (/v1/chat/completions vs /v1/messages) before listing its models — or pick a - flavor by hand if the server is picky about that OPTIONS probe. Stem strips a trailing /v1 - from your URL and lets the client add the versioned path itself. The key goes on the wire the way the - target API expects (Authorization: Bearer for OpenAI-flavored servers, X-Api-Key + Auto-detect and Test connection checks which routes answer ( + /v1/chat/completions, /v1/responses, or /v1/messages) before listing + models. If multiple routes answer, choose the API flavor your model supports explicitly; route + availability does not prove model compatibility. You can also pick a flavor by hand if the server + refuses the OPTIONS probe. A successful test verifies model listing, not a real model turn. Stem strips a + trailing /v1 from your URL and lets the client add the versioned path itself. The key goes on + the wire the way the target API expects (Authorization: Bearer for OpenAI-flavored servers,{' '} + X-Api-Key for Anthropic). Leave the model IDs empty and Stem uses whatever the endpoint lists, re-checked every half minute like Ollama and LM Studio — the way to add a second Ollama running on another machine. Type IDs to pin a fixed set instead; endpoints that serve no listing need that. @@ -1981,6 +1982,7 @@ function LocalServerAddForm({ > + = { a.string, a.string, a.optional(a.nullish(a.string)), - a.optional(a.nullish(a.oneOf(['openai-completions', 'anthropic-messages']))), + a.optional(a.nullish(a.oneOf(['openai-completions', 'openai-responses', 'anthropic-messages']))), // A paired computer to probe from; null/absent = from the server. a.optional(a.nullish(a.string)) ], diff --git a/src/server/pi/models-config.ts b/src/server/pi/models-config.ts index 191b5eca..3418184a 100644 --- a/src/server/pi/models-config.ts +++ b/src/server/pi/models-config.ts @@ -54,10 +54,10 @@ const DEVICE_PROBE_TIMEOUT_MS = 10_000; * trailing /v1 (users paste both forms; the API-specific client appends the * versioned path itself). * - * Both flavors strip trailing /v1: for openai-completions Stem appends /v1 - * before writing (pi then hits /v1/chat/completions), and for anthropic-messages - * pi hands the URL to @anthropic-ai/sdk which appends /v1/messages verbatim. - * Leaving /v1 in either case would double-version the request. + * All flavors strip trailing /v1. Stem re-appends it for OpenAI Chat + * Completions and Responses; for anthropic-messages pi hands the root to + * @anthropic-ai/sdk, which appends /v1/messages verbatim. Leaving /v1 in the + * Anthropic case would double-version the request. */ export function normalizeLocalBaseUrl(raw: string): string { let url = raw.trim().replace(/\/+$/, ''); @@ -110,11 +110,10 @@ async function isToolCapable(base: string, id: string, timeoutMs: number): Promi } /** - * Which chat/messages routes this endpoint exposes. Called by auto-detect to - * pick between openai-completions and anthropic-messages: both flavors serve - * `GET /v1/models` under the same route, so the listing can't tell them apart - * — the real signal is which of `/v1/chat/completions` (OpenAI) and - * `/v1/messages` (Anthropic) the proxy actually implements. + * Which completion/response/messages routes this endpoint exposes. Called by + * auto-detect to pick an API flavor: all flavors serve `GET /v1/models` under + * the same route, so the listing can't tell them apart — the real signal is + * which request route the proxy implements. * * Uses OPTIONS to keep the probe cost-free (no completion tokens spent). A * 404 = route absent; anything else (200/400/401/405/…) = route present and @@ -125,29 +124,30 @@ async function isToolCapable(base: string, id: string, timeoutMs: number): Promi * very thing being classified, and a header the server doesn't recognize is * ignored rather than rejected. */ -async function classifyChatRoutes( +async function classifyApiRoutes( base: string, apiKey: string | undefined, timeoutMs: number -): Promise<{ oc: boolean; am: boolean }> { +): Promise<{ oc: boolean; or: boolean; am: boolean }> { const key = apiKey?.trim(); const headers = { ...probeHeaders('openai-completions', key), ...probeHeaders('anthropic-messages', key) }; const options = { method: 'OPTIONS', headers, signal: AbortSignal.timeout(timeoutMs) }; - const [oc, am] = await Promise.allSettled([ + const [oc, or, am] = await Promise.allSettled([ fetch(`${base}/v1/chat/completions`, options), + fetch(`${base}/v1/responses`, options), fetch(`${base}/v1/messages`, options) ]); return { oc: oc.status === 'fulfilled' && oc.value.status !== 404, + or: or.status === 'fulfilled' && or.value.status !== 404, am: am.status === 'fulfilled' && am.value.status !== 404 }; } /** * Single-flavor probe: hits GET {base}/v1/models under the flavor's protocol - * expectations (Ollama capability filter for openai-completions, no filter for - * anthropic-messages). Success carries the `api` back so an auto-detect caller - * can attribute the answer. + * expectations (Ollama capability filter only for openai-completions). Success + * carries the `api` back so an auto-detect caller can attribute the answer. */ async function probeOneFlavor( baseUrl: string, @@ -170,10 +170,10 @@ async function probeOneFlavor( .map((m) => (typeof m.id === 'string' ? m.id.trim() : '')) .filter(Boolean) .sort((a, b) => a.localeCompare(b)); - // Ollama is keyless and OpenAI-flavored, so any of {bearer key, anthropic - // flavor} means this isn't Ollama — skip the per-model capability probe. + // Ollama is keyless and Chat Completions-flavored, so a bearer key or any + // other API flavor means this isn't Ollama — skip its capability probe. const capable = - key || api === 'anthropic-messages' + key || api !== 'openai-completions' ? all.map(() => true) : await Promise.all(all.map((id) => isToolCapable(base, id, timeoutMs))); const models = all.filter((_, i) => capable[i]); @@ -194,15 +194,13 @@ async function probeOneFlavor( * returned as-is; on success the result carries `api` back so a caller can * confirm. * - * `api` omitted (auto-detect): first classify which chat route the endpoint - * exposes (OPTIONS on /v1/chat/completions and /v1/messages — non-404 wins), - * then probe GET /v1/models under that flavor's downstream expectations. - * openai-completions wins ties (a proxy that serves both routes is a matter of - * downstream preference; the current default matches Stem's pre-existing - * behavior). If neither chat route resolves, fall back to a plain - * openai-completions probe of /v1/models — some servers only expose the - * listing and refuse OPTIONS entirely; the user can still enable and test in - * a real turn. + * `api` omitted (auto-detect): first classify which API route the endpoint + * exposes (OPTIONS on /v1/chat/completions, /v1/responses, and /v1/messages — + * non-404 counts as present), then probe GET /v1/models if exactly one flavor + * is detected. Multiple routes require an explicit flavor selection: route + * presence cannot establish which protocol a particular model supports. + * If no route resolves, retain the pre-existing openai-completions listing + * fallback; the user can select another flavor and test in a real turn. * * Never throws — failures come back as `{ ok:false, error }`. */ @@ -215,17 +213,19 @@ export async function probeLocalProvider( if (api) return probeOneFlavor(baseUrl, apiKey, api, timeoutMs); const base = normalizeLocalBaseUrl(baseUrl); if (!/^https?:\/\//i.test(base)) return { ok: false, error: 'Enter a URL like http://localhost:11434.' }; - const routes = await classifyChatRoutes(base, apiKey, timeoutMs); - // openai-completions wins ties: pre-existing default and more common. - const detected: LocalProviderApi | null = routes.oc - ? 'openai-completions' - : routes.am - ? 'anthropic-messages' - : null; - if (detected) return probeOneFlavor(baseUrl, apiKey, detected, timeoutMs); - // Neither OPTIONS resolved (server rejects OPTIONS, or both 404). Fall back - // to a plain openai-completions listing probe — same behavior as pre-flavor. - return probeOneFlavor(baseUrl, apiKey, 'openai-completions', timeoutMs); + const routes = await classifyApiRoutes(base, apiKey, timeoutMs); + const detected: LocalProviderApi[] = []; + if (routes.oc) detected.push('openai-completions'); + if (routes.or) detected.push('openai-responses'); + if (routes.am) detected.push('anthropic-messages'); + if (detected.length > 1) { + return { + ok: false, + error: `The endpoint exposes multiple API routes (${detected.join(', ')}). Select an API flavor explicitly and test again.` + }; + } + // Keep the legacy listing fallback when no route can be classified. + return probeOneFlavor(baseUrl, apiKey, detected[0] ?? 'openai-completions', timeoutMs); } /** @@ -278,18 +278,17 @@ function localProviderBlock( modelOverrides: Record ): PiProviderConfig { const normalized = normalizeLocalBaseUrl(baseUrl); - // openai-completions: append /v1 so pi hits /v1/chat/completions (matches the - // Ollama/LM Studio flow that always did it). + // OpenAI flavors: append /v1 so pi hits /v1/chat/completions or /v1/responses + // (matches the Ollama/LM Studio flow that always did it). // anthropic-messages: leave the normalized root as-is — pi's Anthropic client // (@anthropic-ai/sdk) appends /v1/messages verbatim to whatever baseURL it // receives. Appending /v1 here would produce /v1/v1/messages at request time. const finalBaseUrl = api === 'anthropic-messages' ? normalized : `${normalized}/v1`; - // Per-flavor compat: OpenAI-compat servers get the local-server defaults; - // anthropic-messages providers get an empty block so pi's Anthropic defaults - // (eager tool streaming, cache retention, …) apply. Users needing proxy- - // specific overrides can hand-edit models.json. + // Chat Completions servers get conservative local-server defaults. Responses + // and Anthropic providers use pi's native protocol defaults; proxy-specific + // changes can still be supplied through per-model compat overrides. const compat: Record = - api === 'anthropic-messages' ? {} : { supportsDeveloperRole: false, supportsReasoningEffort: false }; + api === 'openai-completions' ? { supportsDeveloperRole: false, supportsReasoningEffort: false } : {}; return { baseUrl: finalBaseUrl, api, @@ -484,9 +483,13 @@ export function syncModelsConfig(): Promise { const parsed = parseModelOverrides(settings[id].modelOverrides); if (!parsed.ok) log('pi', `ignoring invalid model overrides for ${id}`, { errors: parsed.errors }); const overrides = parsed.ok ? parsed.value : {}; - // Only custom endpoints may speak anthropic-messages; the coercion enforces this too, - // this is defense in depth against a hand-edited settings.json. - const api: LocalProviderApi = isCustomProviderId(id) && rawApi === 'anthropic-messages' ? 'anthropic-messages' : 'openai-completions'; + // Only custom endpoints may select Responses or Anthropic Messages; the + // coercion enforces this too, and this is defense in depth against a + // hand-edited settings.json. + const api: LocalProviderApi = + isCustomProviderId(id) && (rawApi === 'openai-responses' || rawApi === 'anthropic-messages') + ? rawApi + : 'openai-completions'; // Hand-entered ids are authoritative: an endpoint that names its models has // opted out of discovery, so don't probe it (and don't let a listing endpoint // it happens to serve override the user's choice). diff --git a/src/server/workspace/settings.ts b/src/server/workspace/settings.ts index 4e083099..d72f178b 100644 --- a/src/server/workspace/settings.ts +++ b/src/server/workspace/settings.ts @@ -627,14 +627,17 @@ function coerce(parsed: Partial | null): ServerSettings { const models = Array.isArray(r.models) ? r.models.filter((m): m is string => typeof m === 'string' && !!m.trim()).map((m) => m.trim()) : []; - // API flavor: only custom endpoints may opt into anthropic-messages. Ollama/LM Studio - // are always openai-completions — a hand-edited settings.json cannot switch - // them; the field would be ignored downstream anyway. Persist the flavor - // verbatim (both `openai-completions` and `anthropic-messages`) so a saved - // settings.json is self-describing and future debugging can tell an explicit + // API flavor: only custom endpoints may opt into Responses or Anthropic + // Messages. Ollama/LM Studio are always openai-completions — a hand-edited + // settings.json cannot switch them; the field would be ignored downstream + // anyway. Persist the flavor verbatim so a saved settings.json is + // self-describing and future debugging can tell an explicit // openai-completions pick from an absent field (= not yet configured). const api: LocalProviderApi | undefined = - custom && (r.api === 'anthropic-messages' || r.api === 'openai-completions') ? r.api : undefined; + custom && + (r.api === 'anthropic-messages' || r.api === 'openai-responses' || r.api === 'openai-completions') + ? r.api + : undefined; // Per-model overrides: custom endpoints only, coerced only as far as their shape — // an entry that fails the deeper guard (pi/model-overrides.ts) is KEPT here // and dropped at sync time instead. Losing the text on read is the exact diff --git a/src/shared/types.ts b/src/shared/types.ts index de6e0164..645e9467 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -265,11 +265,11 @@ export type LocalProviderId = BuiltInLocalProviderId | CustomProviderId; /** * Which API flavor the custom endpoint speaks. Ollama and LM Studio are always - * OpenAI-compatible; only custom endpoints may set this to `anthropic-messages` (for - * Anthropic's Messages API or a proxy that speaks it). Default (absent) is - * `openai-completions` — matches the pre-existing single-flavor behavior. + * OpenAI-compatible; custom endpoints may instead select OpenAI's Responses API + * or Anthropic's Messages API (or a proxy speaking either protocol). Default + * (absent) is `openai-completions` — the pre-existing single-flavor behavior. */ -export type LocalProviderApi = 'openai-completions' | 'anthropic-messages'; +export type LocalProviderApi = 'openai-completions' | 'openai-responses' | 'anthropic-messages'; /** * What one model of a custom endpoint needs said about it that its `/v1/models` diff --git a/tests/unit/models-config.test.ts b/tests/unit/models-config.test.ts index f94adfff..734b904c 100644 --- a/tests/unit/models-config.test.ts +++ b/tests/unit/models-config.test.ts @@ -175,6 +175,17 @@ describe('probeLocalProvider', () => { expect(init.headers).toEqual({ Authorization: 'Bearer sk-secret' }); }); + it('probes models for openai-responses with bearer authentication', async () => { + const fetchMock = vi.fn(async () => new Response(JSON.stringify({ data: [{ id: 'gpt-6' }] }), { status: 200 })); + vi.stubGlobal('fetch', fetchMock); + const res = await probeLocalProvider('https://gw.example.com/v1', ' sk-secret ', 'openai-responses'); + expect(res).toEqual({ ok: true, api: 'openai-responses', models: ['gpt-6'] }); + expect(fetchMock).toHaveBeenCalledTimes(1); + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, { headers?: Record }]; + expect(url).toBe('https://gw.example.com/v1/models'); + expect(init.headers).toEqual({ Authorization: 'Bearer sk-secret' }); + }); + describe('auto-detect (api omitted)', () => { it('classifies via OPTIONS and probes openai-completions when /v1/chat/completions exists', async () => { // OPTIONS: /v1/chat/completions -> 200 (route present), /v1/messages -> 404. @@ -191,12 +202,26 @@ describe('probeLocalProvider', () => { expect(res).toEqual({ ok: true, api: 'openai-completions', models: ['gpt-x'] }); }); - it('classifies as anthropic-messages when /v1/messages exists and /v1/chat/completions is 404', async () => { - // Anthropic-only proxy shape: /v1/chat/completions -> 404, /v1/messages -> 405 (route exists, wrong method). + it('classifies as openai-responses when only /v1/responses exists', async () => { + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + const path = new URL(String(url)).pathname; + if (init?.method === 'OPTIONS') { + return new Response(null, { status: path.endsWith('/v1/responses') ? 200 : 404 }); + } + return new Response(JSON.stringify({ data: [{ id: 'gpt-6' }] }), { status: 200 }); + }); + vi.stubGlobal('fetch', fetchMock); + const res = await probeLocalProvider('http://localhost:8080', 'sk-secret'); + expect(res).toEqual({ ok: true, api: 'openai-responses', models: ['gpt-6'] }); + }); + + it('classifies as anthropic-messages when only /v1/messages exists', async () => { + // Anthropic-only proxy shape: other routes -> 404, /v1/messages -> 405 (route exists, wrong method). const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { const path = new URL(String(url)).pathname; if (init?.method === 'OPTIONS') { if (path.endsWith('/v1/chat/completions')) return new Response(null, { status: 404 }); + if (path.endsWith('/v1/responses')) return new Response(null, { status: 404 }); if (path.endsWith('/v1/messages')) return new Response(null, { status: 405 }); } return new Response(JSON.stringify({ data: [{ id: 'anthropic--claude-4.8-opus' }] }), { status: 200 }); @@ -206,23 +231,47 @@ describe('probeLocalProvider', () => { expect(res).toEqual({ ok: true, api: 'anthropic-messages', models: ['anthropic--claude-4.8-opus'] }); }); - it('prefers openai-completions when both chat routes exist (tie-break to pre-existing default)', async () => { - // Both 200 on OPTIONS — the ambiguous case. Auto-detect must not silently - // pick anthropic when the more common flavor also works. - const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => { - if (init?.method === 'OPTIONS') return new Response(null, { status: 200 }); - return new Response(JSON.stringify({ data: [{ id: 'x' }] }), { status: 200 }); + it.each([ + { routes: ['/v1/chat/completions', '/v1/responses'] }, + { routes: ['/v1/chat/completions', '/v1/messages'] }, + { routes: ['/v1/responses', '/v1/messages'] }, + { routes: ['/v1/chat/completions', '/v1/responses', '/v1/messages'] } + ])('requires an explicit flavor when routes $routes exist', async ({ routes }) => { + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (init?.method === 'OPTIONS') { + return new Response(null, { status: routes.includes(new URL(String(url)).pathname) ? 200 : 404 }); + } + return new Response(JSON.stringify({ data: [{ id: 'gpt6.1sol' }] }), { status: 200 }); }); vi.stubGlobal('fetch', fetchMock); const res = await probeLocalProvider('http://localhost:8080', 'sk-secret'); - expect(res.ok).toBe(true); - expect(res.api).toBe('openai-completions'); + expect(res.ok).toBe(false); + expect(res.error).toMatch(/multiple API routes.*select an API flavor explicitly/i); + expect(res.api).toBeUndefined(); + expect(res.models).toBeUndefined(); + expect(fetchMock.mock.calls.some(([url]) => new URL(String(url)).pathname === '/v1/models')).toBe(false); }); - it('falls back to an openai-completions listing when both OPTIONS return 404', async () => { - // Some servers refuse OPTIONS entirely (405) or 404 every unknown path; - // the fallback keeps the pre-existing default so Ollama/LM Studio still - // probe as before. + it.each(['openai-completions', 'openai-responses', 'anthropic-messages'] as const)( + 'allows an explicit %s selection after ambiguous detection', + async (api) => { + const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => { + if (init?.method === 'OPTIONS') return new Response(null, { status: 200 }); + return new Response(JSON.stringify({ data: [{ id: 'gpt6.1sol' }] }), { status: 200 }); + }); + vi.stubGlobal('fetch', fetchMock); + expect((await probeLocalProvider('http://localhost:8080', 'sk-secret')).ok).toBe(false); + fetchMock.mockClear(); + const res = await probeLocalProvider('http://localhost:8080', 'sk-secret', api); + expect(res).toEqual({ ok: true, api, models: ['gpt6.1sol'] }); + expect(fetchMock.mock.calls.map(([url, init]) => [url, init?.method ?? 'GET'])).toEqual([ + ['http://localhost:8080/v1/models', 'GET'] + ]); + } + ); + + it('falls back to an openai-completions listing when all OPTIONS return 404', async () => { + // Preserve the legacy listing fallback when route classification is unavailable. const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => { if (init?.method === 'OPTIONS') return new Response(null, { status: 404 }); return new Response(JSON.stringify({ data: [{ id: 'llama' }] }), { status: 200 }); @@ -478,6 +527,26 @@ describe('syncModelsConfig', () => { }); }); + it('writes an openai-responses block with /v1 and native protocol defaults', async () => { + use({ + custom: { + enabled: true, + baseUrl: 'http://proxy.example:9999/openai/v1', + api: 'openai-responses', + apiKey: 'sk-secret', + models: ['gpt-6'] + } + }); + expect(await syncModelsConfig()).toBe(true); + expect(readConfig().providers.custom).toEqual({ + baseUrl: 'http://proxy.example:9999/openai/v1', + api: 'openai-responses', + apiKey: 'sk-secret', + compat: {}, + models: [{ id: 'gpt-6' }] + }); + }); + // The flavor and issue #1's empty-block guard meet on the same line. A merge // that keeps the flavor but drops the guard writes `custom` with no models — // and every pi spawn dies on "Unknown provider" again, this time for the @@ -496,7 +565,7 @@ describe('syncModelsConfig', () => { expect(existsSync(configPath)).toBe(false); }); - it('rejects anthropic-messages on ollama/lmstudio and falls back to openai-completions', async () => { + it('rejects non-completions APIs on ollama/lmstudio and falls back to openai-completions', async () => { // Defense in depth: settings coercion also strips this, so getting here means // a hand-edited settings.json tried to bypass. models.json must not honor it. stubModels(['llama']); @@ -506,7 +575,7 @@ describe('syncModelsConfig', () => { ollama: { enabled: true, baseUrl: 'http://localhost:11434', - api: 'anthropic-messages' + api: 'openai-responses' } as unknown as LocalProvidersSettings['ollama'] }); await syncModelsConfig(); diff --git a/tests/unit/responses-integration.test.ts b/tests/unit/responses-integration.test.ts new file mode 100644 index 00000000..7721fc76 --- /dev/null +++ b/tests/unit/responses-integration.test.ts @@ -0,0 +1,248 @@ +import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'; +import { createServer, type IncomingHttpHeaders } from 'node:http'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { createInterface } from 'node:readline'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { probeLocalProvider, syncModelsConfig } from '../../src/server/pi/models-config'; +import { updateLocalProvider } from '../../src/server/workspace/settings'; +import { settingsStorePath } from '../../src/server/workspace/paths'; + +const piCli = join(__dirname, '../../node_modules/@earendil-works/pi-coding-agent/dist/cli.js'); +const modelId = 'gpt6.1sol'; +const apiKey = 'sk-responses-test'; +const cleanups: Array<() => Promise> = []; + +interface RequestRecord { + method: string | undefined; + path: string | undefined; + headers: IncomingHttpHeaders; + body: Record; +} + +interface RpcEvent { + type: string; + id?: string; + success?: boolean; + error?: string; + message?: { role?: string; stopReason?: string; errorMessage?: string; content?: Array> }; + assistantMessageEvent?: { type?: string; delta?: string }; +} + +const probeExtension = ` +export default function responsesProbe(pi) { + pi.registerTool({ + name: 'responses_probe', + label: 'Responses probe', + description: 'Return the supplied marker for a protocol test.', + parameters: { type: 'object', properties: { marker: { type: 'string' } }, required: ['marker'] }, + async execute(_id, params) { + return { content: [{ type: 'text', text: 'probe:' + params.marker }], details: {} }; + } + }); +} +`; + +async function responsesServer(rejectResponses = false): Promise<{ + baseUrl: string; + requests: RequestRecord[]; +}> { + const requests: RequestRecord[] = []; + let turns = 0; + const server = createServer((req, res) => { + let raw = ''; + req.setEncoding('utf8'); + req.on('data', (chunk: string) => { raw += chunk; }); + req.on('end', () => { + let body: Record; + try { + body = raw ? JSON.parse(raw) : {}; + } catch { + res.writeHead(400).end('invalid JSON'); + return; + } + requests.push({ method: req.method, path: req.url, headers: req.headers, body }); + if (req.headers.authorization !== `Bearer ${apiKey}`) { + res.writeHead(401).end('wrong bearer token'); + return; + } + if (req.method === 'GET' && req.url === '/openai/v1/models') { + res.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify({ data: [{ id: modelId }] })); + return; + } + if (req.method !== 'POST' || req.url !== '/openai/v1/responses' || rejectResponses) { + res.writeHead(404, { 'content-type': 'application/json' }).end(JSON.stringify({ error: { message: 'Responses unavailable' } })); + return; + } + if (++turns > 2) { + res.writeHead(400).end('unexpected extra turn'); + return; + } + res.writeHead(200, { 'content-type': 'text/event-stream' }); + let sequence = 0; + const send = (type: string, data: Record) => { + res.write(`event: ${type}\ndata: ${JSON.stringify({ type, sequence_number: sequence++, ...data })}\n\n`); + }; + const response = { id: `resp_${turns}`, object: 'response', model: modelId, status: 'in_progress', output: [] }; + send('response.created', { response }); + let item: Record; + if (turns === 1) { + item = { type: 'function_call', id: 'fc_probe_1', call_id: 'call_probe_1', name: 'responses_probe', arguments: '', status: 'in_progress' }; + send('response.output_item.added', { output_index: 0, item }); + for (const delta of ['{"marker":', '"round-trip"}']) { + send('response.function_call_arguments.delta', { output_index: 0, item_id: item.id, delta }); + } + item = { ...item, arguments: '{"marker":"round-trip"}', status: 'completed' }; + send('response.function_call_arguments.done', { output_index: 0, item_id: item.id, arguments: item.arguments }); + } else { + item = { type: 'message', id: 'msg_final', role: 'assistant', status: 'in_progress', content: [] }; + send('response.output_item.added', { output_index: 0, item }); + send('response.content_part.added', { output_index: 0, item_id: item.id, content_index: 0, part: { type: 'output_text', text: '', annotations: [] } }); + for (const delta of ['Responses ', 'round-trip complete']) { + send('response.output_text.delta', { output_index: 0, item_id: item.id, content_index: 0, delta }); + } + const text = 'Responses round-trip complete'; + send('response.output_text.done', { output_index: 0, item_id: item.id, content_index: 0, text }); + item = { ...item, status: 'completed', content: [{ type: 'output_text', text, annotations: [] }] }; + } + send('response.output_item.done', { output_index: 0, item }); + send('response.completed', { + response: { ...response, status: 'completed', output: [item], usage: { input_tokens: 20, output_tokens: 5, total_tokens: 25 } } + }); + res.end(); + }); + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + cleanups.push(() => new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()); + server.closeAllConnections(); + })); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('missing loopback port'); + return { baseUrl: `http://127.0.0.1:${address.port}/openai/v1`, requests }; +} + +async function configuredHome(baseUrl: string): Promise { + const home = await mkdtemp(join(tmpdir(), 'stem-responses-')); + cleanups.push(() => rm(home, { recursive: true, force: true })); + vi.stubEnv('STEM_STATE_DIR', join(home, 'state')); + vi.stubEnv('STEM_PI_MODELS_CONFIG', join(home, 'agent/models.json')); + await mkdir(dirname(settingsStorePath()), { recursive: true }); + await updateLocalProvider('custom', { enabled: true, baseUrl, api: 'openai-responses', apiKey, models: [modelId] }); + expect(await syncModelsConfig()).toBe(true); + return home; +} + +async function runPiTurn(home: string): Promise { + const extensionPath = join(home, 'responses-probe.mjs'); + await writeFile(extensionPath, probeExtension); + const child: ChildProcessWithoutNullStreams = spawn(process.execPath, [ + piCli, '--mode', 'rpc', '--no-extensions', '-e', extensionPath, + '--no-skills', '--no-prompt-templates', '--tools', 'responses_probe', + '--provider', 'custom', '--model', modelId + ], { + cwd: home, + env: { PATH: process.env.PATH, HOME: home, USERPROFILE: home, PI_CODING_AGENT_DIR: join(home, 'agent'), PI_OFFLINE: '1' }, + stdio: ['pipe', 'pipe', 'pipe'] + }); + cleanups.push(async () => { + if (!child.pid || child.exitCode !== null || child.signalCode !== null) return; + const exited = new Promise((resolve) => child.once('exit', () => resolve())); + child.kill('SIGKILL'); + await exited; + }); + let stderr = ''; + child.stderr.setEncoding('utf8'); + child.stderr.on('data', (chunk: string) => { stderr += chunk; }); + const lines = createInterface({ input: child.stdout }); + const events: RpcEvent[] = []; + try { + return await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`pi turn timed out: ${stderr}`)), 30_000); + const fail = (error: Error) => { clearTimeout(timer); reject(error); }; + child.once('error', fail); + child.once('exit', (code, signal) => fail(new Error(`pi exited (${code}, ${signal}): ${stderr}`))); + child.stdin.on('error', fail); + lines.on('line', (line) => { + let event: RpcEvent; + try { event = JSON.parse(line); } catch { return; } + events.push(event); + if (event.type === 'response' && event.success === false) { + fail(new Error(`RPC failed: ${event.error}`)); + } else if (event.type === 'response' && event.id === 'ready') { + child.stdin.write(`${JSON.stringify({ type: 'prompt', id: 'turn', message: 'Call responses_probe with marker round-trip, then report completion.' })}\n`); + } else if (event.type === 'agent_end') { + clearTimeout(timer); + resolve(events); + } + }); + // Wait for an RPC readiness response rather than guessing startup duration. + child.stdin.write(`${JSON.stringify({ type: 'get_state', id: 'ready' })}\n`); + }); + } finally { + lines.close(); + } +} + +afterEach(async () => { + const errors: unknown[] = []; + for (const cleanup of cleanups.splice(0).reverse()) { + try { await cleanup(); } catch (error) { errors.push(error); } + } + vi.unstubAllEnvs(); + if (errors.length) throw new AggregateError(errors, 'Responses integration cleanup failed'); +}); + +describe('generated Responses config against real pi', () => { + it('streams a tool call, executes it, and replays its result through authenticated Responses requests', async () => { + const { baseUrl, requests } = await responsesServer(); + const home = await configuredHome(baseUrl); + const events = await runPiTurn(home); + + expect(requests.map(({ method, path }) => [method, path])).toEqual([ + ['POST', '/openai/v1/responses'], + ['POST', '/openai/v1/responses'] + ]); + for (const request of requests) { + expect(request.headers.authorization).toBe(`Bearer ${apiKey}`); + expect(request.body).toMatchObject({ model: modelId, stream: true, store: false }); + } + expect(requests[0].body.tools).toEqual([ + expect.objectContaining({ type: 'function', name: 'responses_probe', parameters: expect.objectContaining({ required: ['marker'] }) }) + ]); + const replay = requests[1].body.input as Array>; + const call = replay.find((item) => item.type === 'function_call'); + expect(call).toMatchObject({ name: 'responses_probe', arguments: '{"marker":"round-trip"}', call_id: expect.any(String) }); + expect(replay).toContainEqual({ type: 'function_call_output', call_id: call!.call_id, output: 'probe:round-trip' }); + expect(events).toContainEqual(expect.objectContaining({ + type: 'message_update', assistantMessageEvent: expect.objectContaining({ type: 'toolcall_delta' }) + })); + expect(events).toContainEqual(expect.objectContaining({ + type: 'message_end', message: expect.objectContaining({ role: 'toolResult', content: [{ type: 'text', text: 'probe:round-trip' }] }) + })); + expect(events.filter((e) => e.assistantMessageEvent?.type === 'text_delta').map((e) => e.assistantMessageEvent?.delta).join('')) + .toBe('Responses round-trip complete'); + expect(events.filter((e) => e.type === 'message_end' && e.message?.role === 'assistant').at(-1)?.message) + .toMatchObject({ stopReason: 'stop', content: [expect.objectContaining({ type: 'text', text: 'Responses round-trip complete' })] }); + }, 60_000); + + it('surfaces a rejected Responses turn even when Test connection lists the model', async () => { + const { baseUrl, requests } = await responsesServer(true); + const home = await configuredHome(baseUrl); + expect(await probeLocalProvider(baseUrl, apiKey, 'openai-responses')) + .toEqual({ ok: true, api: 'openai-responses', models: [modelId] }); + + const events = await runPiTurn(home); + expect(requests.map(({ method, path }) => [method, path])).toEqual([ + ['GET', '/openai/v1/models'], + ['POST', '/openai/v1/responses'] + ]); + expect(events.filter((e) => e.type === 'message_end' && e.message?.role === 'assistant').at(-1)?.message) + .toMatchObject({ stopReason: 'error', errorMessage: expect.stringContaining('Responses unavailable') }); + expect(events.some((e) => e.type === 'tool_execution_start')).toBe(false); + }, 60_000); +}); diff --git a/tests/unit/settings.test.ts b/tests/unit/settings.test.ts index 6d870cc0..fe47fbb0 100644 --- a/tests/unit/settings.test.ts +++ b/tests/unit/settings.test.ts @@ -427,8 +427,25 @@ describe('local provider settings', () => { }); }); + it('round-trips the openai-responses flavor on a custom endpoint', async () => { + await updateLocalProvider('custom', { + enabled: true, + baseUrl: 'http://proxy.example:9999/openai/v1', + api: 'openai-responses', + apiKey: 'sk-secret', + models: ['gpt-6'] + }); + expect((await readSettings()).localProviders.custom).toEqual({ + enabled: true, + baseUrl: 'http://proxy.example:9999/openai/v1', + api: 'openai-responses', + apiKey: 'sk-secret', + models: ['gpt-6'] + }); + }); + it('strips the api field for ollama/lmstudio (hand-edited settings.json cannot force it)', async () => { - // Only `custom` may opt into anthropic-messages; a hand-edited settings.json + // Only `custom` may opt into another protocol; a hand-edited settings.json // trying to set it on ollama must round-trip without the field. await updateLocalProvider('ollama', { enabled: true,