Skip to content

Commit ff2bcfa

Browse files
Merge pull request #543 from corbitsdev/cl-6904-grokopenai-responses-adapters-omit-prompt_cache_key-xai
Set prompt_cache_key on the Grok and OpenAI Responses adapters
2 parents 7df3956 + ecd0380 commit ff2bcfa

9 files changed

Lines changed: 164 additions & 6 deletions

src/config.test.ts

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -927,7 +927,12 @@ describe("buildBifrostSource", () => {
927927

928928
describe("buildXaiSource", () => {
929929
test("omits reasoning_effort when effort is absent", () => {
930-
const source = buildXaiSource({ id: "xai/work", apiKey: "tok", model: "grok-4.6" });
930+
const source = buildXaiSource({
931+
id: "xai/work",
932+
apiKey: "tok",
933+
model: "grok-4.6",
934+
sessionId: "sess-1",
935+
});
931936
expect(source.provider).toBe("grok-responses");
932937
expect(source.defaults?.providerOptions).not.toHaveProperty("reasoning_effort");
933938
});
@@ -937,15 +942,31 @@ describe("buildXaiSource", () => {
937942
id: "xai/work",
938943
apiKey: "tok",
939944
model: "grok-4.6",
945+
sessionId: "sess-1",
940946
reasoningEffort: "low",
941947
});
942948
expect(source.defaults?.providerOptions).toMatchObject({ reasoning_effort: "low" });
943949
});
944950

945951
test("does not invent high when effort is absent", () => {
946-
const source = buildXaiSource({ id: "xai/work", apiKey: "tok", model: "grok-4.6" });
952+
const source = buildXaiSource({
953+
id: "xai/work",
954+
apiKey: "tok",
955+
model: "grok-4.6",
956+
sessionId: "sess-1",
957+
});
947958
expect(source.defaults?.providerOptions?.["reasoning_effort"]).toBeUndefined();
948959
});
960+
961+
test("stashes the session id for the adapter's prompt_cache_key", () => {
962+
const source = buildXaiSource({
963+
id: "xai/work",
964+
apiKey: "tok",
965+
model: "grok-4.6",
966+
sessionId: "sess-1",
967+
});
968+
expect(source.defaults?.providerOptions).toMatchObject({ grokSessionId: "sess-1" });
969+
});
949970
});
950971

951972
describe("buildProviderCatalog", () => {

src/config/index.ts

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,14 @@ import {
3333
} from "../provider/codex-responses-adapter.js";
3434
import {
3535
GROK_RESPONSES_PROVIDER,
36+
GROK_SESSION_ID_OPTION,
3637
GROK_USER_ID_OPTION,
3738
} from "../provider/grok-responses-adapter.js";
3839
import { BIFROST_PROVIDER } from "../provider/bifrost-adapter.js";
39-
import { OPENAI_RESPONSES_PROVIDER } from "../provider/openai-responses-adapter.js";
40+
import {
41+
OPENAI_RESPONSES_PROVIDER,
42+
OPENAI_SESSION_ID_OPTION,
43+
} from "../provider/openai-responses-adapter.js";
4044
import { xaiUserIdFromAccessToken } from "../auth/xai/session.js";
4145
import {
4246
OPENCODE_GO_BASE_URL,
@@ -169,14 +173,19 @@ export function buildCodexSource(fields: {
169173
// "grok-responses" adapter (the grok-cli proxy speaks the Responses API, not
170174
// Chat Completions). The access token is the apiKey; the caller's user id is
171175
// decoded from it and lifted into the x-grok-user-id header by the adapter.
176+
// The session id becomes the request's prompt_cache_key so every call in the
177+
// thread routes to the same cache shard (store:false has no other signal).
172178
export function buildXaiSource(fields: {
173179
id: string;
174180
apiKey: string;
175181
model: string;
182+
sessionId: string;
176183
reasoningEffort?: ReasoningEffort;
177184
}): InferenceSource {
178185
const userId = xaiUserIdFromAccessToken(fields.apiKey);
179-
const providerOptions: Record<string, unknown> = {};
186+
const providerOptions: Record<string, unknown> = {
187+
[GROK_SESSION_ID_OPTION]: fields.sessionId,
188+
};
180189
if (userId !== undefined) providerOptions[GROK_USER_ID_OPTION] = userId;
181190
if (fields.reasoningEffort !== undefined)
182191
providerOptions["reasoning_effort"] = fields.reasoningEffort;
@@ -234,10 +243,12 @@ export function buildAnthropicSource(fields: {
234243
}
235244

236245
// OpenCode Go: per-model protocol routing (chat completions / responses / messages).
246+
// sessionId feeds the Responses-protocol prompt_cache_key (see buildXaiSource).
237247
export function buildGoSource(fields: {
238248
id: string;
239249
apiKey?: string;
240250
model: string;
251+
sessionId?: string;
241252
reasoningEffort?: ReasoningEffort;
242253
}): InferenceSource {
243254
const endpoint = resolveGoEndpoint(fields.model);
@@ -258,7 +269,12 @@ export function buildGoSource(fields: {
258269
baseURL: endpoint.baseURL,
259270
apiKey,
260271
model: fields.model,
261-
defaults: { maxTokens: SOURCE_MAX_TOKENS },
272+
defaults: {
273+
maxTokens: SOURCE_MAX_TOKENS,
274+
...(fields.sessionId !== undefined
275+
? { providerOptions: { [OPENAI_SESSION_ID_OPTION]: fields.sessionId } }
276+
: {}),
277+
},
262278
};
263279
}
264280
// chat-completions (default)

src/config/inference-sources.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ export function buildInferenceSourceForRef(
9898
id: ref.provider,
9999
apiKey: entry.apiKey ?? "",
100100
model: ref.model,
101+
sessionId: ctx.sessionId,
101102
...(effort !== undefined ? { reasoningEffort: effort } : {}),
102103
});
103104
}
@@ -118,6 +119,7 @@ export function buildInferenceSourceForRef(
118119
? { apiKey: providerSettings.apiKey }
119120
: {}),
120121
model: ref.model,
122+
sessionId: ctx.sessionId,
121123
...(effort !== undefined ? { reasoningEffort: effort } : {}),
122124
});
123125
}

src/exec/runner.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -556,6 +556,7 @@ export async function runExec(config: Config): Promise<ExecResult> {
556556
id: config.providerName,
557557
apiKey: config.apiKey,
558558
model: config.model,
559+
sessionId,
559560
...(config.reasoningEffort !== undefined
560561
? { reasoningEffort: config.reasoningEffort }
561562
: {}),

src/provider/grok-responses-adapter.ts

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,9 @@ import {
3232

3333
export const GROK_RESPONSES_PROVIDER = "grok-responses";
3434

35-
// Key the source stashes in defaults.providerOptions for this adapter.
35+
// Keys the source stashes in defaults.providerOptions for this adapter.
3636
export const GROK_USER_ID_OPTION = "grokUserId";
37+
export const GROK_SESSION_ID_OPTION = "grokSessionId";
3738

3839
type ResponsesInputContentPart =
3940
{ type: "input_text"; text: string } | { type: "input_image"; image_url: string };
@@ -202,6 +203,10 @@ function buildRequest(
202203
body["tools"] = tools;
203204
body["tool_choice"] = "auto";
204205
}
206+
// With store:false this is the only cache-routing signal; keying it to the
207+
// inference thread's session id keeps every request on the same cache shard.
208+
const sessionId = optionString(options, GROK_SESSION_ID_OPTION);
209+
if (sessionId !== undefined) body["prompt_cache_key"] = sessionId;
205210

206211
const headers: Record<string, string> = {
207212
"content-type": "application/json",

src/provider/openai-responses-adapter.ts

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@ import {
2424

2525
export const OPENAI_RESPONSES_PROVIDER = "openai-responses";
2626

27+
// Key the source stashes in defaults.providerOptions for this adapter.
28+
export const OPENAI_SESSION_ID_OPTION = "openaiSessionId";
29+
2730
type ResponsesInputContentPart =
2831
{ type: "input_text"; text: string } | { type: "input_image"; image_url: string };
2932

@@ -134,6 +137,11 @@ function toResponsesTools(options: InferenceOptions): unknown[] | undefined {
134137
}));
135138
}
136139

140+
function optionString(options: InferenceOptions, key: string): string | undefined {
141+
const value = options.providerOptions?.[key];
142+
return typeof value === "string" && value.length > 0 ? value : undefined;
143+
}
144+
137145
function dedupeToolOutputs(items: ResponsesInputItem[]): ResponsesInputItem[] {
138146
const seen = new Set<string>();
139147
const deduped: ResponsesInputItem[] = [];
@@ -177,6 +185,10 @@ function buildRequest(
177185
}
178186
if (options.maxTokens !== undefined) body["max_output_tokens"] = options.maxTokens;
179187
if (options.temperature !== undefined) body["temperature"] = options.temperature;
188+
// With store:false this is the only cache-routing signal; keying it to the
189+
// inference thread's session id keeps every request on the same cache shard.
190+
const sessionId = optionString(options, OPENAI_SESSION_ID_OPTION);
191+
if (sessionId !== undefined) body["prompt_cache_key"] = sessionId;
180192

181193
return {
182194
url: "/responses",

src/tui/runner.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1446,6 +1446,7 @@ export async function runTUI(initialConfig: Config): Promise<number> {
14461446
id: config.providerName,
14471447
apiKey: config.apiKey,
14481448
model: config.model,
1449+
sessionId,
14491450
...(config.reasoningEffort !== undefined
14501451
? { reasoningEffort: config.reasoningEffort }
14511452
: {}),

tests/unit/grok-responses-adapter.test.ts

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { test, expect, describe } from "bun:test";
22
import {
33
createGrokResponsesAdapter,
4+
GROK_SESSION_ID_OPTION,
45
GROK_USER_ID_OPTION,
56
} from "../../src/provider/grok-responses-adapter.js";
67
import { BEARER_CREDENTIAL_SENTINEL } from "@intx/inference";
@@ -99,6 +100,39 @@ describe("grok-responses buildRequest", () => {
99100
expect(body["tool_choice"]).toBe("auto");
100101
});
101102

103+
test("sets prompt_cache_key from the session id, stable across builds", () => {
104+
const options: InferenceOptions = {
105+
...baseOptions,
106+
providerOptions: { ...baseOptions.providerOptions, [GROK_SESSION_ID_OPTION]: "sess-1" },
107+
};
108+
const first = JSON.parse(
109+
adapter().buildRequest([userTurn("a")], "grok-4.5", options).body,
110+
) as Record<string, unknown>;
111+
const second = JSON.parse(
112+
adapter().buildRequest([userTurn("b")], "grok-4.5", options).body,
113+
) as Record<string, unknown>;
114+
expect(first["prompt_cache_key"]).toBe("sess-1");
115+
expect(second["prompt_cache_key"]).toBe("sess-1");
116+
});
117+
118+
test("distinct session ids yield distinct prompt_cache_keys", () => {
119+
const bodyFor = (sessionId: string): Record<string, unknown> =>
120+
JSON.parse(
121+
adapter().buildRequest([userTurn("hi")], "grok-4.5", {
122+
providerOptions: { [GROK_SESSION_ID_OPTION]: sessionId },
123+
}).body,
124+
) as Record<string, unknown>;
125+
expect(bodyFor("sess-1")["prompt_cache_key"]).toBe("sess-1");
126+
expect(bodyFor("sess-2")["prompt_cache_key"]).toBe("sess-2");
127+
});
128+
129+
test("omits prompt_cache_key when no session id is present", () => {
130+
const body = JSON.parse(
131+
adapter().buildRequest([userTurn("hi")], "grok-4.5", baseOptions).body,
132+
) as Record<string, unknown>;
133+
expect(body).not.toHaveProperty("prompt_cache_key");
134+
});
135+
102136
test("drops duplicate tool results for a call id", () => {
103137
const turns: ConversationTurn[] = [
104138
{
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
import { test, expect, describe } from "bun:test";
2+
import {
3+
createOpenAIResponsesAdapter,
4+
OPENAI_SESSION_ID_OPTION,
5+
} from "../../src/provider/openai-responses-adapter.js";
6+
import { BEARER_CREDENTIAL_SENTINEL } from "@intx/inference";
7+
import type { ConversationTurn, InferenceOptions, LastCycleSource } from "@intx/types/runtime";
8+
9+
const SOURCE: LastCycleSource = {
10+
sourceId: "go/default",
11+
provider: "openai-responses",
12+
model: "gpt-5.6-luna",
13+
};
14+
15+
function adapter() {
16+
return createOpenAIResponsesAdapter(SOURCE);
17+
}
18+
19+
function userTurn(text: string): ConversationTurn {
20+
return { role: "user", content: [{ type: "text", text }], timestamp: 0 };
21+
}
22+
23+
describe("openai-responses buildRequest", () => {
24+
test("targets the Responses path with store off and streaming on", () => {
25+
const req = adapter().buildRequest([userTurn("hi")], "gpt-5.6-luna", {});
26+
expect(req.url).toBe("/responses");
27+
expect(req.headers["authorization"]).toBe(BEARER_CREDENTIAL_SENTINEL);
28+
expect(req.headers["accept"]).toBe("text/event-stream");
29+
const body = JSON.parse(req.body) as Record<string, unknown>;
30+
expect(body["model"]).toBe("gpt-5.6-luna");
31+
expect(body["stream"]).toBe(true);
32+
expect(body["store"]).toBe(false);
33+
});
34+
35+
test("sets prompt_cache_key from the session id, stable across builds", () => {
36+
const options: InferenceOptions = {
37+
providerOptions: { [OPENAI_SESSION_ID_OPTION]: "sess-1" },
38+
};
39+
const first = JSON.parse(
40+
adapter().buildRequest([userTurn("a")], "gpt-5.6-luna", options).body,
41+
) as Record<string, unknown>;
42+
const second = JSON.parse(
43+
adapter().buildRequest([userTurn("b")], "gpt-5.6-luna", options).body,
44+
) as Record<string, unknown>;
45+
expect(first["prompt_cache_key"]).toBe("sess-1");
46+
expect(second["prompt_cache_key"]).toBe("sess-1");
47+
});
48+
49+
test("distinct session ids yield distinct prompt_cache_keys", () => {
50+
const bodyFor = (sessionId: string): Record<string, unknown> =>
51+
JSON.parse(
52+
adapter().buildRequest([userTurn("hi")], "gpt-5.6-luna", {
53+
providerOptions: { [OPENAI_SESSION_ID_OPTION]: sessionId },
54+
}).body,
55+
) as Record<string, unknown>;
56+
expect(bodyFor("sess-1")["prompt_cache_key"]).toBe("sess-1");
57+
expect(bodyFor("sess-2")["prompt_cache_key"]).toBe("sess-2");
58+
});
59+
60+
test("omits prompt_cache_key when no session id is present", () => {
61+
const body = JSON.parse(
62+
adapter().buildRequest([userTurn("hi")], "gpt-5.6-luna", {}).body,
63+
) as Record<string, unknown>;
64+
expect(body).not.toHaveProperty("prompt_cache_key");
65+
});
66+
});

0 commit comments

Comments
 (0)