diff --git a/packages/ai/src/protocols/open-responses.ts b/packages/ai/src/protocols/open-responses.ts index 79fec2d2b8bd..fbdce21e1e5c 100644 --- a/packages/ai/src/protocols/open-responses.ts +++ b/packages/ai/src/protocols/open-responses.ts @@ -72,6 +72,11 @@ const OpenResponsesReasoningSummaryText = Schema.Struct({ text: Schema.String, }) +const OpenResponsesReasoningText = Schema.Struct({ + type: Schema.tag("reasoning_text"), + text: Schema.String, +}) + const OpenResponsesReasoningItem = Schema.Struct({ type: Schema.tag("reasoning"), id: Schema.optionalKey(Schema.String), @@ -79,6 +84,18 @@ const OpenResponsesReasoningItem = Schema.Struct({ encrypted_content: optionalNull(Schema.String), }) +export const ReasoningItem = Schema.StructWithRest( + Schema.Struct({ + type: Schema.tag("reasoning"), + id: Schema.String, + summary: Schema.Array(OpenResponsesReasoningSummaryText), + content: Schema.optional(Schema.Array(OpenResponsesReasoningText)), + encrypted_content: optionalNull(Schema.String), + }), + [Schema.Record(Schema.String, Schema.Unknown)], +) +export type ReasoningItem = Schema.Schema.Type + const OpenResponsesWebSearchCall = Schema.StructWithRest( Schema.Struct({ type: Schema.tag("web_search_call"), @@ -181,9 +198,16 @@ export type ExtendedHostedToolItem = { readonly id: string readonly [key: string]: unknown } +export type ExtendedReasoningItem = { + readonly type: "reasoning" + readonly id: string + readonly summary: ReadonlyArray<{ readonly type: "summary_text"; readonly text: string }> + readonly [key: string]: unknown +} type LoweredInputItem = | OpenResponsesInputItem | ExtendedHostedToolItem + | ExtendedReasoningItem | { readonly type: "message" readonly id?: string @@ -345,6 +369,7 @@ export const Event = Schema.StructWithRest( text: Schema.optional(Schema.String), item_id: Schema.optional(Schema.String), output_index: Schema.optional(Schema.Number), + content_index: Schema.optional(Schema.Number), summary_index: Schema.optional(Schema.Number), // OutputItemAdded/Done permit a null item in the Open Responses OpenAPI schema. item: optionalNull(StreamItem), @@ -382,6 +407,7 @@ export interface Extension { readonly request: LLMRequest }) => MediaInput | undefined readonly lowerHostedToolItem?: (item: unknown) => ExtendedHostedToolItem | undefined + readonly lowerReasoningItem?: (item: unknown) => ExtendedReasoningItem | undefined } const BASE: Extension = { id: ADAPTER, name: NAME } @@ -400,19 +426,12 @@ export interface ParserState { readonly reasoningItems: Readonly> } -type ReasoningSummaryStatus = "active" | "can-conclude" | "concluded" - interface ReasoningStreamItem { readonly open: boolean readonly encryptedContent: string | null | undefined - // Keyed by the wire protocol's numeric `summary_index`. JS object keys coerce to - // strings, but typing the map as `Record` documents intent - // and matches the wire field. - readonly summaryParts: Readonly> - // Summary indexes that received at least one streamed delta. The `:0` block - // is started eagerly when the item opens, so block existence cannot tell - // whether a `.done` final would duplicate streamed text. - readonly deltaIndexes: ReadonlySet + readonly text: string + readonly rawIndexes: ReadonlySet + readonly summaryIndexes: ReadonlySet } // ============================================================================= @@ -463,9 +482,22 @@ const lowerToolCall = (part: ToolCallPart, providerMetadataKey: string): OpenRes } } -const lowerReasoning = (part: ReasoningPart, providerMetadataKey: string): OpenResponsesReasoningInput | undefined => { +const lowerReasoning = ( + part: ReasoningPart, + providerMetadataKey: string, + extension: Extension, +): OpenResponsesReasoningInput | ExtendedReasoningItem | undefined => { const metadata = part.providerMetadata?.[providerMetadataKey] if (!ProviderShared.isRecord(metadata)) return undefined + const extended = extension.lowerReasoningItem?.(metadata.reasoningItem) + if (extended) return extended + if (!extension.lowerReasoningItem && Schema.is(ReasoningItem)(metadata.reasoningItem)) + return { + type: "reasoning", + id: metadata.reasoningItem.id, + summary: [...metadata.reasoningItem.summary], + encrypted_content: metadata.reasoningItem.encrypted_content, + } const id = itemID(part.providerMetadata, providerMetadataKey) const encryptedContent = typeof metadata.reasoningEncryptedContent === "string" || metadata.reasoningEncryptedContent === null @@ -619,17 +651,18 @@ const lowerMessages = Effect.fn("OpenResponses.lowerMessages")(function* (reques } if (part.type === "reasoning") { flushText() - const reasoning = lowerReasoning(part, providerMetadataKey) + const reasoning = lowerReasoning(part, providerMetadataKey, extension) if (!reasoning) continue const existing = reasoning.id === undefined ? undefined : reasoningItems[reasoning.id] if (existing) { existing.summary.push(...reasoning.summary) - if (typeof reasoning.encrypted_content === "string") + if (reasoning.encrypted_content === null || typeof reasoning.encrypted_content === "string") existing.encrypted_content = reasoning.encrypted_content continue } - if (reasoning.id !== undefined) reasoningItems[reasoning.id] = reasoning - input.push(reasoning) + const lowered = { ...reasoning, summary: [...reasoning.summary] } + if (reasoning.id !== undefined) reasoningItems[reasoning.id] = lowered + input.push(lowered) continue } if (part.type === "tool-call") { @@ -863,39 +896,27 @@ const joinReasoningText = (parts: ReadonlyArray) => { export const outputItemID = (state: ParserState, event: Event) => event.output_index === undefined ? event.item_id : (state.outputItems[event.output_index] ?? event.item_id) -const startReasoningSummaryPart = (state: ParserState, itemID: string, index: number): StepResult => { - const item = state.reasoningItems[itemID] - if (!item?.open || index === 0 || item.summaryParts[index] !== undefined) return [state, NO_EVENTS] +type ReasoningRawDeltaEvent = Pick +type ReasoningRawDoneEvent = Pick +type ReasoningSummaryDeltaEvent = Pick +type ReasoningSummaryDoneEvent = Pick +const appendReasoningText = ( + state: ParserState, + itemID: string, + item: ReasoningStreamItem, + text: string, +): StepResult => { const events: LLMEvent[] = [] - const lifecycle = Object.entries(item.summaryParts) - .filter((entry) => entry[1] !== "concluded") - .reduce( - (lifecycle, entry) => - Lifecycle.reasoningEnd(lifecycle, events, `${itemID}:${entry[0]}`, providerMetadata(state, { itemId: itemID })), - state.lifecycle, - ) return [ { ...state, - lifecycle: Lifecycle.reasoningStart( - lifecycle, - events, - `${itemID}:${index}`, - providerMetadata(state, { itemId: itemID, reasoningEncryptedContent: item.encryptedContent ?? null }), - ), + lifecycle: Lifecycle.reasoningDelta(state.lifecycle, events, itemID, text), reasoningItems: { ...state.reasoningItems, [itemID]: { ...item, - summaryParts: { - ...Object.fromEntries( - Object.entries(item.summaryParts).map((entry) => - entry[1] === "concluded" ? entry : [entry[0], "concluded" as const], - ), - ), - [index]: "active", - }, + text: item.text + text, }, }, }, @@ -903,41 +924,70 @@ const startReasoningSummaryPart = (state: ParserState, itemID: string, index: nu ] } -export const onReasoningDelta = (state: ParserState, event: Event, itemID: string): StepResult => { +const onReasoningRawDelta = (state: ParserState, event: ReasoningRawDeltaEvent, itemID: string): StepResult => { const item = state.reasoningItems[itemID] if (!event.delta || !item?.open) return [state, NO_EVENTS] - const index = event.summary_index ?? 0 - if (item.summaryParts[index] === "concluded") return [state, NO_EVENTS] - const [started, emitted] = startReasoningSummaryPart(state, itemID, index) - const current = started.reasoningItems[itemID] - if (!current) return [started, emitted] - const events: LLMEvent[] = [...emitted] - return [ - { - ...started, - lifecycle: Lifecycle.reasoningDelta(started.lifecycle, events, `${itemID}:${index}`, event.delta), - reasoningItems: { - ...started.reasoningItems, - [itemID]: { ...current, deltaIndexes: new Set([...current.deltaIndexes, index]) }, - }, - }, - events, - ] + const index = event.content_index ?? 0 + return appendReasoningText(state, itemID, { ...item, rawIndexes: new Set([...item.rawIndexes, index]) }, event.delta) } -// Some compatible gateways emit a reasoning final without streaming any -// deltas, mirroring `response.output_text.done`. Reconcile the complete text -// as a single delta unless that summary index already streamed one. -export const onReasoningDone = (state: ParserState, event: Event, itemID: string): StepResult => { +const onReasoningRawDone = (state: ParserState, event: ReasoningRawDoneEvent, itemID: string): StepResult => { const item = state.reasoningItems[itemID] - if (!item?.open || typeof event.text !== "string") return [state, NO_EVENTS] + if (!event.text || !item?.open) return [state, NO_EVENTS] + const index = event.content_index ?? 0 + if (item.rawIndexes.has(index)) return [state, NO_EVENTS] + return appendReasoningText(state, itemID, { ...item, rawIndexes: new Set([...item.rawIndexes, index]) }, event.text) +} + +const appendReasoningSummary = ( + state: ParserState, + itemID: string, + item: ReasoningStreamItem, + index: number, + text: string, +) => + appendReasoningText( + state, + itemID, + { ...item, summaryIndexes: new Set([...item.summaryIndexes, index]) }, + (index > 0 && !item.summaryIndexes.has(index) && item.text.length > 0 ? "\n\n" : "") + text, + ) + +const onReasoningSummaryDelta = (state: ParserState, event: ReasoningSummaryDeltaEvent, itemID: string): StepResult => { + const item = state.reasoningItems[itemID] + if (!event.delta || !item?.open) return [state, NO_EVENTS] + return appendReasoningSummary(state, itemID, item, event.summary_index ?? 0, event.delta) +} + +const onReasoningSummaryDone = (state: ParserState, event: ReasoningSummaryDoneEvent, itemID: string): StepResult => { + const item = state.reasoningItems[itemID] + if (!event.text || !item?.open) return [state, NO_EVENTS] const index = event.summary_index ?? 0 - if (item.deltaIndexes.has(index)) return [state, NO_EVENTS] - return onReasoningDelta(state, { ...event, delta: event.text }, itemID) + if (item.summaryIndexes.has(index)) return [state, NO_EVENTS] + return appendReasoningSummary(state, itemID, item, index, event.text) +} + +const completedReasoningItem = ( + item: StreamItem & { type: "reasoning"; id: string }, + tracked?: ReasoningStreamItem, +) => { + const completed = + item.encrypted_content === undefined && tracked?.encryptedContent !== undefined + ? { ...item, encrypted_content: tracked.encryptedContent } + : item + return Schema.is(ReasoningItem)(completed) ? completed : undefined } -const reasoningMetadata = (state: ParserState, item: StreamItem & { id: string }) => - providerMetadata(state, { itemId: item.id, reasoningEncryptedContent: item.encrypted_content ?? null }) +const reasoningMetadata = ( + state: ParserState, + item: StreamItem & { type: "reasoning"; id: string }, + completed: ReasoningItem | undefined, +) => + providerMetadata(state, { + itemId: item.id, + reasoningEncryptedContent: completed?.encrypted_content ?? item.encrypted_content ?? null, + ...(completed ? { reasoningItem: completed } : {}), + }) // Responses APIs normally stream reasoning items in this order: // `output_item.added` (reasoning) → @@ -946,9 +996,8 @@ const reasoningMetadata = (state: ParserState, item: StreamItem & { id: string } // `reasoning_summary_part.done` (index=0) → // (repeat for index>0) → // `output_item.done` (reasoning). -// `onOutputItemAdded` seeds the per-item entry, while each later part start is -// also an implicit boundary for the previous part. This keeps the common event -// lifecycle ordered when a compatible provider omits or delays a part-done event. +// `onOutputItemAdded` seeds the per-item entry. Summary boundaries remain +// readable separators inside that one item-scoped lifecycle. const onOutputItemAdded = (state: ParserState, event: Event): StepResult => { const item = event.item if (item?.type === "message" && item.id !== undefined) { @@ -985,14 +1034,15 @@ const onOutputItemAdded = (state: ParserState, event: Event): StepResult => { return [ { ...state, - lifecycle: Lifecycle.reasoningStart(state.lifecycle, events, `${item.id}:0`, reasoningMetadata(state, item)), + lifecycle: Lifecycle.reasoningStart(state.lifecycle, events, item.id), reasoningItems: { ...state.reasoningItems, [item.id]: { open: true, encryptedContent: item.encrypted_content, - summaryParts: { 0: "active" }, - deltaIndexes: new Set(), + text: "", + rawIndexes: new Set(), + summaryIndexes: new Set(), }, }, }, @@ -1021,34 +1071,6 @@ const onOutputItemAdded = (state: ParserState, event: Event): StepResult => { ] } -const onReasoningSummaryPartAdded = (state: ParserState, event: Event): StepResult => { - if (event.item_id === undefined || event.summary_index === undefined) return [state, NO_EVENTS] - return startReasoningSummaryPart(state, event.item_id, event.summary_index) -} - -const onReasoningSummaryPartDone = (state: ParserState, event: Event): StepResult => { - if (event.item_id === undefined || event.summary_index === undefined) return [state, NO_EVENTS] - const item = state.reasoningItems[event.item_id] - if (!item?.open) return [state, NO_EVENTS] - if (item.summaryParts[event.summary_index] !== "active") return [state, NO_EVENTS] - return [ - { - ...state, - reasoningItems: { - ...state.reasoningItems, - [event.item_id]: { - ...item, - summaryParts: { - ...item.summaryParts, - [event.summary_index]: "can-conclude", - }, - }, - }, - }, - NO_EVENTS, - ] -} - const onFunctionCallArgumentsDelta = Effect.fn("OpenResponses.onFunctionCallArgumentsDelta")(function* ( state: ParserState, event: Event, @@ -1162,7 +1184,11 @@ const onOutputItemDone = Effect.fn("OpenResponses.onOutputItemDone")(function* ( if (isReasoningItem(item)) { if (state.reasoningItems[item.id]?.open === false) return [state, NO_EVENTS] satisfies StepResult - const metadata = reasoningMetadata(state, item) + const tracked = state.reasoningItems[item.id] + if (!tracked && state.lifecycle.reasoning.size > 0) + return yield* ProviderShared.eventError(state.id, "reasoning completed before the previous item ended") + const completed = completedReasoningItem(item, tracked) + const metadata = reasoningMetadata(state, item, completed) const summaryParts: ReadonlyArray = Array.isArray(item.summary) ? item.summary : [] const summary: Array = [] for (const part of summaryParts) { @@ -1178,61 +1204,40 @@ const onOutputItemDone = Effect.fn("OpenResponses.onOutputItemDone")(function* ( } const itemText = joinReasoningText(summary) ?? joinReasoningText(content) const events: LLMEvent[] = [] - const reasoningItem = state.reasoningItems[item.id] - if (reasoningItem) { - const fragments = Object.entries(reasoningItem.summaryParts) - let lifecycle = state.lifecycle - for (const [index, status] of fragments) { - if (status === "concluded") continue - // Do not repeat earlier summaries that were already emitted as separate fragments. - const finalText = fragments.length === 1 ? itemText : summary[Number(index)] - lifecycle = Lifecycle.reasoningEnd(lifecycle, events, `${item.id}:${index}`, metadata, finalText || undefined) - } + const text = itemText ?? tracked?.text ?? "" + if (tracked) { return [ { ...state, - lifecycle, + lifecycle: Lifecycle.reasoningEnd(state.lifecycle, events, item.id, metadata, text), reasoningItems: { ...state.reasoningItems, [item.id]: { - ...reasoningItem, + ...tracked, open: false, - encryptedContent: item.encrypted_content ?? reasoningItem.encryptedContent, + encryptedContent: completed?.encrypted_content ?? tracked.encryptedContent, }, }, }, events, ] satisfies StepResult } - if (!state.lifecycle.reasoning.has(item.id)) { - const lifecycle = Lifecycle.stepStart(state.lifecycle, events) - events.push(LLMEvent.reasoningStart({ id: item.id, providerMetadata: metadata })) - events.push( - LLMEvent.reasoningEnd({ - id: item.id, - providerMetadata: metadata, - text: itemText, - }), - ) - return [ - { - ...state, - lifecycle, - reasoningItems: { - ...state.reasoningItems, - [item.id]: { - open: false, - encryptedContent: item.encrypted_content, - summaryParts: { 0: "concluded" }, - deltaIndexes: new Set(), - }, + const started = Lifecycle.reasoningStart(state.lifecycle, events, item.id) + return [ + { + ...state, + lifecycle: Lifecycle.reasoningEnd(started, events, item.id, metadata, text), + reasoningItems: { + ...state.reasoningItems, + [item.id]: { + open: false, + encryptedContent: completed?.encrypted_content, + text, + rawIndexes: new Set(), + summaryIndexes: new Set(), }, }, - events, - ] satisfies StepResult - } - return [ - { ...state, lifecycle: Lifecycle.reasoningEnd(state.lifecycle, events, item.id, metadata) }, + }, events, ] satisfies StepResult } @@ -1337,25 +1342,29 @@ export const step = (state: ParserState, input: Event) => { : onOutputTextDone(state, { ...event, text: value }, event.item_id), ) } - if (event.type === "response.reasoning.delta" || event.type === "response.reasoning_summary_text.delta") { + if (event.type === "response.reasoning.delta" || event.type === "response.reasoning_text.delta") { + if (event.item_id === undefined) return ProviderShared.eventError(state.id, `${event.type} is missing item_id`) + return Effect.succeed(onReasoningRawDelta(state, event, event.item_id)) + } + if (event.type === "response.reasoning.done" || event.type === "response.reasoning_text.done") { + if (event.item_id === undefined) return ProviderShared.eventError(state.id, `${event.type} is missing item_id`) + return Effect.succeed(onReasoningRawDone(state, event, event.item_id)) + } + if (event.type === "response.reasoning_summary_text.delta") { if (event.item_id === undefined) return ProviderShared.eventError(state.id, `${event.type} is missing item_id`) - return Effect.succeed(onReasoningDelta(state, event, event.item_id)) + return Effect.succeed(onReasoningSummaryDelta(state, event, event.item_id)) } - if ( - event.type === "response.reasoning.done" || - event.type === "response.reasoning_summary_text.done" || - event.type === "response.reasoning_text.done" - ) { + if (event.type === "response.reasoning_summary_text.done") { if (event.item_id === undefined) return ProviderShared.eventError(state.id, `${event.type} is missing item_id`) - return Effect.succeed(onReasoningDone(state, event, event.item_id)) + return Effect.succeed(onReasoningSummaryDone(state, event, event.item_id)) } if (event.type === "response.reasoning_summary_part.added") return event.item_id !== undefined - ? Effect.succeed(onReasoningSummaryPartAdded(state, event)) + ? Effect.succeed([state, NO_EVENTS]) : ProviderShared.eventError(state.id, `${event.type} is missing item_id`) if (event.type === "response.reasoning_summary_part.done") return event.item_id !== undefined - ? Effect.succeed(onReasoningSummaryPartDone(state, event)) + ? Effect.succeed([state, NO_EVENTS]) : ProviderShared.eventError(state.id, `${event.type} is missing item_id`) if (event.type === "response.output_item.added") { if (event.item?.type === "message" && event.item.id === undefined) diff --git a/packages/ai/src/protocols/openai-responses.ts b/packages/ai/src/protocols/openai-responses.ts index f88b1edb7f46..07bb2b7611f3 100644 --- a/packages/ai/src/protocols/openai-responses.ts +++ b/packages/ai/src/protocols/openai-responses.ts @@ -32,6 +32,18 @@ const OpenAIResponsesImageGenerationTool = Schema.Struct({ size: Schema.optional(OpenAIImage.Size), }) +const OpenAIResponsesReasoningItem = Schema.StructWithRest( + Schema.Struct({ + type: Schema.tag("reasoning"), + id: Schema.String, + summary: Schema.Array(Schema.Struct({ type: Schema.tag("summary_text"), text: Schema.String })), + content: Schema.optional(Schema.Array(Schema.Struct({ type: Schema.tag("reasoning_text"), text: Schema.String }))), + encrypted_content: optionalNull(Schema.String), + status: Schema.optional(Schema.Literals(["in_progress", "completed", "incomplete"])), + }), + [Schema.Record(Schema.String, Schema.Unknown)], +) + const OpenAIResponsesHostedToolItem = Schema.Union([ Schema.StructWithRest( Schema.Struct({ @@ -75,7 +87,9 @@ const OpenAIResponsesToolChoice = Schema.Union([ const OpenAIResponsesCoreFields = { ...OpenResponses.coreFields, - input: Schema.Array(Schema.Union([OpenResponses.InputItem, OpenAIResponsesHostedToolItem])), + input: Schema.Array( + Schema.Union([OpenAIResponsesReasoningItem, OpenResponses.InputItem, OpenAIResponsesHostedToolItem]), + ), tools: optionalArray(OpenAIResponsesTools), tool_choice: Schema.optional(OpenAIResponsesToolChoice), } @@ -90,6 +104,7 @@ const extension = { id: ADAPTER, name: NAME, lowerHostedToolItem: (item: unknown) => (Schema.is(OpenAIResponsesHostedToolItem)(item) ? item : undefined), + lowerReasoningItem: (item: unknown) => (Schema.is(OpenAIResponsesReasoningItem)(item) ? item : undefined), } satisfies OpenResponses.Extension const nativeImageToolInput = (tool: ToolDefinition) => { @@ -185,12 +200,6 @@ const HOSTED_TOOLS = { } as const satisfies ResponsesHostedTools.Definitions const step = (state: OpenResponses.ParserState, event: OpenResponses.Event) => { - if (event.type === "response.reasoning_text.delta") - return event.item_id !== undefined - ? Effect.succeed( - OpenResponses.onReasoningDelta(state, event, OpenResponses.outputItemID(state, event) ?? event.item_id), - ) - : ProviderShared.eventError(ADAPTER, `${event.type} is missing item_id`) if (event.type === "response.output_item.done" && event.item && ResponsesHostedTools.isItem(event.item, HOSTED_TOOLS)) return ResponsesHostedTools.onDone(state, event.item, HOSTED_TOOLS) return OpenResponses.step(state, event) diff --git a/packages/ai/test/provider/golden.recorded.test.ts b/packages/ai/test/provider/golden.recorded.test.ts index 5e49fe4cff3f..10c54c4e8c91 100644 --- a/packages/ai/test/provider/golden.recorded.test.ts +++ b/packages/ai/test/provider/golden.recorded.test.ts @@ -8,6 +8,7 @@ import * as OpenAICompatible from "../../src/providers/openai-compatible.js" import * as OpenRouter from "../../src/providers/openrouter.js" import * as XAI from "../../src/providers/xai.js" import { describeRecordedGoldenScenarios } from "../recorded-golden.js" +import { matchOptionalEmptyReasoningContent } from "../recorded-test.js" const openAI = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY ?? "fixture", @@ -88,6 +89,7 @@ describeRecordedGoldenScenarios([ model: openAIResponses, requires: ["OPENAI_API_KEY"], tags: ["flagship"], + options: { match: matchOptionalEmptyReasoningContent }, scenarios: [ { id: "text", temperature: false }, { id: "reasoning", temperature: false }, diff --git a/packages/ai/test/provider/open-responses-finals.test.ts b/packages/ai/test/provider/open-responses-finals.test.ts index d46a335ed7a4..78bf0ad42a09 100644 --- a/packages/ai/test/provider/open-responses-finals.test.ts +++ b/packages/ai/test/provider/open-responses-finals.test.ts @@ -138,13 +138,23 @@ describe("Open Responses completed item reasoning", () => { expect(response.reasoning).toBe(fixture.text) expect(response.events.filter(LLMEvent.is.reasoningEnd)).toHaveLength(1) expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({ - "openai-compatible": { itemId: "rs_1", reasoningEncryptedContent: "encrypted" }, + "openai-compatible": { + itemId: "rs_1", + reasoningEncryptedContent: "encrypted", + reasoningItem: { + type: "reasoning", + id: "rs_1", + summary: fixture.summary, + content: fixture.content, + encrypted_content: "encrypted", + }, + }, }) }), ) }) - it.effect("replaces only the still-open summary without repeating earlier text", () => + it.effect("replaces all streamed summary text with the completed item", () => Effect.gen(function* () { const response = yield* generate( { type: "response.output_item.added", item: { type: "reasoning", id: "rs_1" } }, @@ -164,8 +174,8 @@ describe("Open Responses completed item reasoning", () => { }, completed, ) - expect(response.reasoning).toBe("First final") - expect(response.events.filter(LLMEvent.is.reasoningEnd).map((event) => event.text)).toEqual([undefined, "final"]) + expect(response.reasoning).toBe("First \n\nfinal") + expect(response.events.filter(LLMEvent.is.reasoningEnd).map((event) => event.text)).toEqual(["First \n\nfinal"]) }), ) }) diff --git a/packages/ai/test/provider/open-responses-lifecycle.test.ts b/packages/ai/test/provider/open-responses-lifecycle.test.ts index dfd3c8e4a4ab..c0189e054390 100644 --- a/packages/ai/test/provider/open-responses-lifecycle.test.ts +++ b/packages/ai/test/provider/open-responses-lifecycle.test.ts @@ -71,7 +71,7 @@ function expectLifecycle(events: ReadonlyArray, completed: boolean) { } describe("Open Responses basic-item lifecycles", () => { - it.effect("closes implicit summary boundaries and ignores late events for completed reasoning", () => + it.effect("keeps summary boundaries in one block and ignores late events for completed reasoning", () => Effect.gen(function* () { const item = { type: "reasoning", id: "rs_1", encrypted_content: "encrypted-state" } const events = yield* collect( @@ -102,27 +102,16 @@ describe("Open Responses basic-item lifecycles", () => { expect(events.filter((event) => event.type.startsWith("reasoning-"))).toEqual([ { type: "reasoning-start", - id: "rs_1:0", - providerMetadata: { "openai-compatible": { itemId: "rs_1", reasoningEncryptedContent: null } }, - }, - { type: "reasoning-delta", id: "rs_1:0", text: "First" }, - { type: "reasoning-end", id: "rs_1:0", providerMetadata: { "openai-compatible": { itemId: "rs_1" } } }, - { - type: "reasoning-start", - id: "rs_1:1", - providerMetadata: { "openai-compatible": { itemId: "rs_1", reasoningEncryptedContent: null } }, - }, - { type: "reasoning-delta", id: "rs_1:1", text: "Second" }, - { type: "reasoning-end", id: "rs_1:1", providerMetadata: { "openai-compatible": { itemId: "rs_1" } } }, - { - type: "reasoning-start", - id: "rs_1:2", - providerMetadata: { "openai-compatible": { itemId: "rs_1", reasoningEncryptedContent: null } }, + id: "rs_1", + providerMetadata: undefined, }, - { type: "reasoning-delta", id: "rs_1:2", text: "Third" }, + { type: "reasoning-delta", id: "rs_1", text: "First" }, + { type: "reasoning-delta", id: "rs_1", text: "\n\nSecond" }, + { type: "reasoning-delta", id: "rs_1", text: "\n\nThird" }, { type: "reasoning-end", - id: "rs_1:2", + id: "rs_1", + text: "First\n\nSecond\n\nThird", providerMetadata: { "openai-compatible": { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } }, }, ]) @@ -152,13 +141,19 @@ describe("Open Responses basic-item lifecycles", () => { { type: "reasoning-start", id: "rs_1", - providerMetadata: { "openai-compatible": { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } }, + providerMetadata: undefined, }, { type: "reasoning-end", id: "rs_1", text: "Not streamed", - providerMetadata: { "openai-compatible": { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } }, + providerMetadata: { + "openai-compatible": { + itemId: "rs_1", + reasoningEncryptedContent: "encrypted-state", + reasoningItem: item, + }, + }, }, ]) }), @@ -340,7 +335,7 @@ describe("Open Responses basic-item lifecycles", () => { providerExecuted: undefined, providerMetadata: { "openai-compatible": { itemId: "fc_1" } }, }, - { type: "reasoning-end", id: "rs_1:0" }, + { type: "reasoning-end", id: "rs_1" }, ]) }), ) @@ -377,7 +372,8 @@ describe("Open Responses basic-item lifecycles", () => { expect(events.filter(LLMEvent.is.reasoningEnd)).toEqual([ { type: "reasoning-end", - id: ":0", + id: "", + text: "Thinking", providerMetadata: { "openai-compatible": { itemId: "", reasoningEncryptedContent: "state" } }, }, ]) @@ -452,7 +448,7 @@ describe("Open Responses basic-item lifecycles", () => { ) expect(events.filter(LLMEvent.is.toolInputEnd)).toEqual([]) expect(events.filter(LLMEvent.is.toolCall)).toEqual([]) - expect(events.filter(LLMEvent.is.reasoningEnd)).toEqual([{ type: "reasoning-end", id: "rs_1:0" }]) + expect(events.filter(LLMEvent.is.reasoningEnd)).toEqual([{ type: "reasoning-end", id: "rs_1" }]) expect(events.filter(LLMEvent.is.finish)).toEqual([ { type: "finish", diff --git a/packages/ai/test/provider/openai-compatible-responses.test.ts b/packages/ai/test/provider/openai-compatible-responses.test.ts index ced45842d26b..c06269aa929d 100644 --- a/packages/ai/test/provider/openai-compatible-responses.test.ts +++ b/packages/ai/test/provider/openai-compatible-responses.test.ts @@ -406,7 +406,7 @@ describe("Open Responses-compatible route", () => { }) routings.forEach((routing) => { - it.effect(`preserves reasoning summary boundaries without terminal reconciliation with ${routing.name}`, () => + it.effect(`keeps one reasoning block without terminal reconciliation with ${routing.name}`, () => Effect.gen(function* () { const address = { item_id: routing.item_id, output_index: routing.output_index } const response = yield* LLMClient.generate(request).pipe( @@ -437,26 +437,10 @@ describe("Open Responses-compatible route", () => { expect(response.message.content).toEqual([ { type: "reasoning", - text: "First.", - providerMetadata: { "openai-compatible": { itemId: routing.id } }, + text: "First.\n\nSecond.", }, - { - type: "reasoning", - text: "Second.", - providerMetadata: { - "openai-compatible": { itemId: routing.id, reasoningEncryptedContent: null }, - }, - }, - ]) - expect(response.events.filter(LLMEvent.is.reasoningEnd)).toEqual([ - { - type: "reasoning-end", - id: `${routing.id}:0`, - text: undefined, - providerMetadata: { "openai-compatible": { itemId: routing.id } }, - }, - { type: "reasoning-end", id: `${routing.id}:1` }, ]) + expect(response.events.filter(LLMEvent.is.reasoningEnd)).toEqual([{ type: "reasoning-end", id: routing.id }]) }), ) }) @@ -696,7 +680,7 @@ describe("Open Responses-compatible route", () => { expect(response.events.find((event) => event.type === "reasoning-end")).toEqual({ type: "reasoning-end", - id: "rs_raw:0", + id: "rs_raw", }) }), ) diff --git a/packages/ai/test/provider/openai-responses-images.recorded.test.ts b/packages/ai/test/provider/openai-responses-images.recorded.test.ts index 51ca8f1729fe..9e9937176447 100644 --- a/packages/ai/test/provider/openai-responses-images.recorded.test.ts +++ b/packages/ai/test/provider/openai-responses-images.recorded.test.ts @@ -2,7 +2,8 @@ import { describe, expect } from "bun:test" import { Effect } from "effect" import { LLM, LLMEvent, Message } from "../../src/index.js" import { OpenAI } from "../../src/providers.js" -import { recordedTests } from "../recorded-test.js" +import { ProviderShared } from "../../src/protocols/shared.js" +import { matchOptionalEmptyReasoningContent, recordedTests } from "../recorded-test.js" const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY ?? "fixture", @@ -13,6 +14,9 @@ const recorded = recordedTests({ provider: "openai", protocol: "openai-responses", requires: ["OPENAI_API_KEY"], + options: { + match: matchOptionalEmptyReasoningContent, + }, }) describe("OpenAI Responses image generation recorded", () => { @@ -38,6 +42,12 @@ describe("OpenAI Responses image generation recorded", () => { }), ) + const reasoning = response.message.content.find((part) => part.type === "reasoning") + const reasoningItem = reasoning?.providerMetadata?.openai?.reasoningItem + expect(ProviderShared.isRecord(reasoningItem) && Array.isArray(reasoningItem.summary)).toBe(true) + if (!ProviderShared.isRecord(reasoningItem) || !Array.isArray(reasoningItem.summary)) return + expect(reasoningItem.summary).toHaveLength(2) + const result = response.events.find(LLMEvent.is.toolResult) expect(result).toBeDefined() expect(result?.providerExecuted).toBe(true) diff --git a/packages/ai/test/provider/openai-responses-reasoning-items.test.ts b/packages/ai/test/provider/openai-responses-reasoning-items.test.ts new file mode 100644 index 000000000000..c9a76f6626cb --- /dev/null +++ b/packages/ai/test/provider/openai-responses-reasoning-items.test.ts @@ -0,0 +1,245 @@ +import { describe, expect } from "bun:test" +import { Effect } from "effect" +import { LLM, LLMEvent, Message } from "../../src/index.js" +import { OpenAIResponses } from "../../src/protocols/openai-responses.js" +import { configure } from "../../src/providers/openai-compatible-responses.js" +import { Auth, LLMClient } from "../../src/route.js" +import { compileRequest } from "../../src/route/client.js" +import { it } from "../lib/effect.js" +import { fixedResponse } from "../lib/http.js" +import { sseEvents } from "../lib/sse.js" + +const model = OpenAIResponses.route + .with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") }) + .model({ id: "reasoning-model" }) +const request = LLM.request({ model, prompt: "Think it through." }) +const completed = { type: "response.completed", response: { id: "resp_1" } } +const generate = (...events: OpenAIResponses.Event[]) => + LLMClient.generate(request).pipe(Effect.provide(fixedResponse(sseEvents(...events)))) + +describe("OpenAI Responses reasoning items", () => { + it.effect("streams one block and replays the completed item by route", () => + Effect.gen(function* () { + const item = { + type: "reasoning", + id: "rs_1", + summary: [ + { type: "summary_text", text: "Corrected summary." }, + { type: "summary_text", text: "Second summary." }, + ], + content: [{ type: "reasoning_text", text: "Completed raw." }], + status: "completed", + future_field: { retained: true }, + } as const + const response = yield* generate( + { + type: "response.output_item.added", + output_index: 2, + item: { type: "reasoning", id: "rs_1", encrypted_content: "added-state" }, + }, + { + type: "response.reasoning_text.delta", + output_index: 2, + item_id: "wrong", + content_index: 0, + delta: "Raw delta. ", + }, + { + type: "response.reasoning_summary_text.delta", + output_index: 2, + item_id: "wrong", + summary_index: 0, + delta: "Summary delta.", + }, + { type: "response.reasoning_summary_part.added", item_id: "rs_1", summary_index: 1 }, + { + type: "response.reasoning_summary_text.done", + item_id: "rs_1", + summary_index: 1, + text: "Second summary.", + }, + { type: "response.reasoning.delta", item_id: "rs_1", content_index: 1, delta: " Raw tail." }, + { type: "response.output_item.done", item }, + completed, + ) + const stored = { ...item, encrypted_content: "added-state" } + + expect(response.events.filter(LLMEvent.is.reasoningStart)).toEqual([ + { type: "reasoning-start", id: "rs_1", providerMetadata: undefined }, + ]) + expect(response.events.filter(LLMEvent.is.reasoningDelta).map((event) => event.text)).toEqual([ + "Raw delta. ", + "Summary delta.", + "\n\nSecond summary.", + " Raw tail.", + ]) + expect(response.events.filter(LLMEvent.is.reasoningEnd)).toEqual([ + { + type: "reasoning-end", + id: "rs_1", + text: "Corrected summary.\n\nSecond summary.", + providerMetadata: { + openai: { + itemId: "rs_1", + reasoningEncryptedContent: "added-state", + reasoningItem: stored, + }, + }, + }, + ]) + expect(response.message.content).toEqual([ + { + type: "reasoning", + text: "Corrected summary.\n\nSecond summary.", + providerMetadata: { + openai: { + itemId: "rs_1", + reasoningEncryptedContent: "added-state", + reasoningItem: stored, + }, + }, + }, + ]) + + const native = yield* compileRequest(LLM.request({ model, messages: [response.message] })) + expect(native.body.input).toEqual([stored]) + + const reasoning = response.message.content[0] + if (reasoning?.type !== "reasoning") return + const sharedModel = configure({ apiKey: "test", baseURL: "https://responses.test/v1" }).model("shared-model") + const shared = yield* compileRequest( + LLM.request({ + model: sharedModel, + messages: [ + Message.assistant({ + ...reasoning, + providerMetadata: { "openai-compatible": reasoning.providerMetadata?.openai ?? {} }, + }), + ], + }), + ) + expect(shared.body.input).toEqual([ + { + type: "reasoning", + id: "rs_1", + summary: item.summary, + encrypted_content: "added-state", + }, + ]) + }), + ) + + it.effect("uses completed raw content and then streamed fallback", () => + Effect.gen(function* () { + const raw = yield* generate( + { type: "response.output_item.added", item: { type: "reasoning", id: "rs_raw" } }, + { type: "response.reasoning_summary_text.delta", item_id: "rs_raw", delta: "Provisional" }, + { + type: "response.output_item.done", + item: { + type: "reasoning", + id: "rs_raw", + summary: [{ type: "summary_text", text: "" }], + content: [{ type: "reasoning_text", text: "Raw final" }], + }, + }, + completed, + ) + const fallback = yield* generate( + { type: "response.output_item.added", item: { type: "reasoning", id: "rs_fallback" } }, + { type: "response.reasoning_summary_text.delta", item_id: "rs_fallback", delta: "Streamed text" }, + { + type: "response.output_item.done", + item: { type: "reasoning", id: "rs_fallback", summary: [], content: [] }, + }, + completed, + ) + + expect(raw.reasoning).toBe("Raw final") + expect(fallback.reasoning).toBe("Streamed text") + expect(raw.events.filter(LLMEvent.is.reasoningEnd).map((event) => event.text)).toEqual(["Raw final"]) + expect(fallback.events.filter(LLMEvent.is.reasoningEnd).map((event) => event.text)).toEqual(["Streamed text"]) + }), + ) + + it.effect("tracks raw and summary indexes independently with an empty item ID", () => + Effect.gen(function* () { + const response = yield* generate( + { type: "response.output_item.added", item: { type: "reasoning", id: "" } }, + { type: "response.reasoning_text.delta", item_id: "", content_index: 0, delta: "Raw" }, + { type: "response.reasoning_text.done", item_id: "", content_index: 0, text: "Raw duplicate" }, + { type: "response.reasoning_summary_text.done", item_id: "", summary_index: 0, text: "Summary" }, + { type: "response.reasoning.delta", item_id: "", content_index: 1, delta: " raw tail" }, + { type: "response.reasoning.done", item_id: "", content_index: 1, text: "raw duplicate" }, + { type: "response.reasoning_summary_text.delta", item_id: "", summary_index: 1, delta: "Second" }, + { type: "response.reasoning_summary_text.done", item_id: "", summary_index: 1, text: "duplicate" }, + { type: "response.output_item.done", item: { type: "reasoning", id: "", summary: [], content: [] } }, + completed, + ) + + expect(response.events.filter(LLMEvent.is.reasoningDelta).map((event) => event.text)).toEqual([ + "Raw", + "Summary", + " raw tail", + "\n\nSecond", + ]) + expect(response.events.filter(LLMEvent.is.reasoningStart).map((event) => event.id)).toEqual([""]) + expect(response.events.filter(LLMEvent.is.reasoningEnd).map((event) => event.id)).toEqual([""]) + }), + ) + + it.effect("does not replay reasoning without output item completion", () => + Effect.gen(function* () { + const response = yield* generate( + { + type: "response.output_item.added", + item: { type: "reasoning", id: "rs_partial", encrypted_content: "partial-state" }, + }, + { type: "response.reasoning_summary_text.delta", item_id: "rs_partial", delta: "Partial" }, + { + type: "response.completed", + response: { + id: "resp_1", + output: [ + { + type: "reasoning", + id: "rs_partial", + summary: [{ type: "summary_text", text: "Terminal" }], + encrypted_content: "terminal-state", + }, + ], + }, + }, + ) + + expect(response.reasoning).toBe("Partial") + expect(response.message.content).toEqual([{ type: "reasoning", text: "Partial" }]) + expect(response.events.filter(LLMEvent.is.reasoningEnd)).toEqual([{ type: "reasoning-end", id: "rs_partial" }]) + const replay = yield* compileRequest(LLM.request({ model, messages: [response.message] })) + expect(replay.body.input).toEqual([]) + }), + ) + + it.effect("keeps sequential reasoning items separate", () => + Effect.gen(function* () { + const response = yield* generate( + { + type: "response.output_item.done", + item: { type: "reasoning", id: "rs_1", summary: [{ type: "summary_text", text: "First" }] }, + }, + { + type: "response.output_item.done", + item: { type: "reasoning", id: "rs_2", summary: [{ type: "summary_text", text: "Second" }] }, + }, + completed, + ) + + expect(response.message.content.map((part) => (part.type === "reasoning" ? part.text : undefined))).toEqual([ + "First", + "Second", + ]) + expect(response.events.filter(LLMEvent.is.reasoningStart).map((event) => event.id)).toEqual(["rs_1", "rs_2"]) + expect(response.events.filter(LLMEvent.is.reasoningEnd).map((event) => event.id)).toEqual(["rs_1", "rs_2"]) + }), + ) +}) diff --git a/packages/ai/test/provider/openai-responses.test.ts b/packages/ai/test/provider/openai-responses.test.ts index c2d3347c00d1..2632a6d0ded9 100644 --- a/packages/ai/test/provider/openai-responses.test.ts +++ b/packages/ai/test/provider/openai-responses.test.ts @@ -2500,11 +2500,11 @@ describe("OpenAI Responses route", () => { expect(response.text).toBe("Hello") expect(response.events).toMatchObject([ { type: "step-start", index: 0 }, - { type: "reasoning-start", id: "rs_1:0" }, - { type: "reasoning-delta", id: "rs_1:0", text: "thinking" }, + { type: "reasoning-start", id: "rs_1" }, + { type: "reasoning-delta", id: "rs_1", text: "thinking" }, { type: "text-start", id: "msg_1" }, { type: "text-delta", id: "msg_1", text: "Hello" }, - { type: "reasoning-end", id: "rs_1:0" }, + { type: "reasoning-end", id: "rs_1" }, { type: "text-end", id: "msg_1" }, { type: "step-finish", index: 0, reason: { normalized: "stop", raw: undefined } }, { type: "finish", reason: { normalized: "stop", raw: undefined } }, @@ -2514,7 +2514,6 @@ describe("OpenAI Responses route", () => { { type: "reasoning", text: "thinking", - providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: null } }, }, { type: "text", text: "Hello", providerMetadata: { openai: { itemId: "msg_1" } } }, ]) @@ -2547,8 +2546,19 @@ describe("OpenAI Responses route", () => { expect(response.events).toContainEqual( expect.objectContaining({ type: "reasoning-end", - id: "rs_1:0", - providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } }, + id: "rs_1", + providerMetadata: { + openai: { + itemId: "rs_1", + reasoningEncryptedContent: "encrypted-state", + reasoningItem: { + type: "reasoning", + id: "rs_1", + encrypted_content: "encrypted-state", + summary: [{ type: "summary_text", text: "thinking" }], + }, + }, + }, }), ) }), @@ -2595,12 +2605,11 @@ describe("OpenAI Responses route", () => { expect(response.reasoning).toBe("Checked the diff.") expect(response.events.filter((event) => event.type === "reasoning-end")).toEqual([ - { type: "reasoning-end", id: "rs_1:0" }, + { type: "reasoning-end", id: "rs_1" }, ]) expect(response.message.content).toContainEqual({ type: "reasoning", text: "Checked the diff.", - providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: null } }, }) }), ) @@ -2668,7 +2677,7 @@ describe("OpenAI Responses route", () => { expect(response.events.find((event) => event.type === "reasoning-end")).toEqual({ type: "reasoning-end", - id: "rs_1:0", + id: "rs_1", }) expect(response.events.filter(LLMEvent.is.toolCall)).toEqual([ expect.objectContaining({ id: "call_1", input: { query: "weather" } }), @@ -2680,7 +2689,7 @@ describe("OpenAI Responses route", () => { }), ) - it.effect("streams each reasoning summary part as a separate block", () => + it.effect("streams reasoning summary parts in one block", () => Effect.gen(function* () { const response = yield* LLMClient.generate( LLMRequest.update(request, { providerOptions: { store: false } }), @@ -2708,25 +2717,20 @@ describe("OpenAI Responses route", () => { ), ) - expect(response.reasoning).toBe("FirstSecond") + expect(response.reasoning).toBe("First\n\nSecond") expect(response.events).toMatchObject([ { type: "step-start", index: 0 }, { type: "reasoning-start", - id: "rs_1:0", - providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: null } }, - }, - { type: "reasoning-delta", id: "rs_1:0", text: "First" }, - { type: "reasoning-end", id: "rs_1:0", providerMetadata: { openai: { itemId: "rs_1" } } }, - { - type: "reasoning-start", - id: "rs_1:1", - providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: null } }, + id: "rs_1", + providerMetadata: undefined, }, - { type: "reasoning-delta", id: "rs_1:1", text: "Second" }, + { type: "reasoning-delta", id: "rs_1", text: "First" }, + { type: "reasoning-delta", id: "rs_1", text: "\n\nSecond" }, { type: "reasoning-end", - id: "rs_1:1", + id: "rs_1", + text: "First\n\nSecond", providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } }, }, { type: "step-finish", index: 0, reason: { normalized: "stop", raw: undefined } }, @@ -2735,73 +2739,6 @@ describe("OpenAI Responses route", () => { }), ) - it.effect("concludes reasoning at implicit summary boundaries", () => - Effect.gen(function* () { - const response = yield* LLMClient.generate( - LLMRequest.update(request, { providerOptions: { store: false } }), - ).pipe( - Effect.provide( - fixedResponse( - sseEvents( - { - type: "response.output_item.added", - item: { type: "reasoning", id: "rs_1", encrypted_content: null }, - }, - { type: "response.reasoning_summary_part.added", item_id: "rs_1", summary_index: 0 }, - { type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 0, delta: "First" }, - // The next part is enough to conclude the previous one even when - // its done event is delayed. - { type: "response.reasoning_summary_part.added", item_id: "rs_1", summary_index: 1 }, - { type: "response.reasoning_summary_part.done", item_id: "rs_1", summary_index: 0 }, - { type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 1, delta: "Second" }, - { type: "response.reasoning_summary_part.done", item_id: "rs_1", summary_index: 1 }, - // Some compatible providers begin the next part with its first delta. - { type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 2, delta: "Third" }, - { - type: "response.output_item.done", - item: { type: "reasoning", id: "rs_1", encrypted_content: "encrypted-state" }, - }, - { type: "response.completed", response: { id: "resp_1" } }, - ), - ), - ), - ) - - expect(response.reasoning).toBe("FirstSecondThird") - expect(response.events.filter((event) => event.type.startsWith("reasoning-"))).toEqual([ - { - type: "reasoning-start", - id: "rs_1:0", - providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: null } }, - }, - { type: "reasoning-delta", id: "rs_1:0", text: "First", providerMetadata: undefined }, - { type: "reasoning-end", id: "rs_1:0", providerMetadata: { openai: { itemId: "rs_1" } } }, - { - type: "reasoning-start", - id: "rs_1:1", - providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: null } }, - }, - { type: "reasoning-delta", id: "rs_1:1", text: "Second", providerMetadata: undefined }, - { - type: "reasoning-end", - id: "rs_1:1", - providerMetadata: { openai: { itemId: "rs_1" } }, - }, - { - type: "reasoning-start", - id: "rs_1:2", - providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: null } }, - }, - { type: "reasoning-delta", id: "rs_1:2", text: "Third", providerMetadata: undefined }, - { - type: "reasoning-end", - id: "rs_1:2", - providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } }, - }, - ]) - }), - ) - it.effect("rejects a reasoning item that starts before the previous item ends", () => Effect.gen(function* () { const error = yield* LLMClient.generate(request).pipe( @@ -2997,15 +2934,26 @@ describe("OpenAI Responses route", () => { expect(response.events.filter((event) => event.type.startsWith("reasoning-"))).toEqual([ { type: "reasoning-start", - id: "rs_1:0", - providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: null } }, + id: "rs_1", + providerMetadata: undefined, }, - { type: "reasoning-delta", id: "rs_1:0", text: "Checked the diff.", providerMetadata: undefined }, + { type: "reasoning-delta", id: "rs_1", text: "Checked the diff.", providerMetadata: undefined }, { type: "reasoning-end", - id: "rs_1:0", + id: "rs_1", text: "Checked the diff.", - providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } }, + providerMetadata: { + openai: { + itemId: "rs_1", + reasoningEncryptedContent: "encrypted-state", + reasoningItem: { + type: "reasoning", + id: "rs_1", + summary: [{ type: "summary_text", text: "Checked the diff." }], + encrypted_content: "encrypted-state", + }, + }, + }, }, ]) @@ -3051,7 +2999,7 @@ describe("OpenAI Responses route", () => { expect(response.reasoning).toBe("Streamed") expect(response.events.filter((event) => event.type === "reasoning-delta")).toEqual([ - { type: "reasoning-delta", id: "rs_1:0", text: "Streamed", providerMetadata: undefined }, + { type: "reasoning-delta", id: "rs_1", text: "Streamed", providerMetadata: undefined }, ]) }), ) @@ -3074,7 +3022,15 @@ describe("OpenAI Responses route", () => { { type: "response.reasoning_summary_part.done", item_id: "rs_1", summary_index: 1 }, { type: "response.output_item.done", - item: { type: "reasoning", id: "rs_1", encrypted_content: "encrypted-state" }, + item: { + type: "reasoning", + id: "rs_1", + summary: [ + { type: "summary_text", text: "First" }, + { type: "summary_text", text: "Second" }, + ], + encrypted_content: "encrypted-state", + }, }, { type: "response.completed", response: { id: "resp_1" } }, ), @@ -3083,11 +3039,25 @@ describe("OpenAI Responses route", () => { ) expect(response.events.filter((event) => event.type === "reasoning-end")).toEqual([ - { type: "reasoning-end", id: "rs_1:0", providerMetadata: { openai: { itemId: "rs_1" } } }, { type: "reasoning-end", - id: "rs_1:1", - providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } }, + id: "rs_1", + text: "First\n\nSecond", + providerMetadata: { + openai: { + itemId: "rs_1", + reasoningEncryptedContent: "encrypted-state", + reasoningItem: { + type: "reasoning", + id: "rs_1", + summary: [ + { type: "summary_text", text: "First" }, + { type: "summary_text", text: "Second" }, + ], + encrypted_content: "encrypted-state", + }, + }, + }, }, ]) }), diff --git a/packages/ai/test/recorded-test.ts b/packages/ai/test/recorded-test.ts index 18dffa805a2c..7a17e460b734 100644 --- a/packages/ai/test/recorded-test.ts +++ b/packages/ai/test/recorded-test.ts @@ -9,6 +9,7 @@ import { ImageClient } from "../src/image-client.js" import type { Service as ImageClientService } from "../src/image-client.js" import type { Service as LLMClientService } from "../src/route/client.js" import type { Service as RequestExecutorService } from "../src/route/executor.js" +import { ProviderShared } from "../src/protocols/shared.js" import { recordedEffectGroup, type RecordedCaseOptions as RunnerCaseOptions, @@ -28,6 +29,30 @@ type RecordedCaseOptions = RunnerCaseOptions & { readonly options?: HttpRecorder.RecorderOptions } +const reasoningRequestBody = (body: string) => { + const value = ProviderShared.decodeJson(body) + if (!ProviderShared.isRecord(value) || !Array.isArray(value.input)) return body + return ProviderShared.encodeJson({ + ...value, + input: value.input.map((item) => { + if (!ProviderShared.isRecord(item) || item.type !== "reasoning") return item + if (!Array.isArray(item.content) || item.content.length > 0) return item + return Object.fromEntries(Object.entries(item).filter(([key]) => key !== "content")) + }), + }) +} + +export const matchOptionalEmptyReasoningContent: HttpRecorder.RequestMatcher = (incoming, expected) => + incoming.method === expected.method && + incoming.url === expected.url && + [...new Set([...Object.keys(incoming.headers), ...Object.keys(expected.headers)])].every( + (key) => incoming.headers[key] === expected.headers[key], + ) && + Bun.deepEquals( + ProviderShared.decodeJson(reasoningRequestBody(incoming.body)), + ProviderShared.decodeJson(reasoningRequestBody(expected.body)), + ) + const mergeOptions = ( base: HttpRecorder.RecorderOptions | undefined, override: HttpRecorder.RecorderOptions | undefined,