diff --git a/CHANGELOG.md b/CHANGELOG.md index 752a924a..3d527c1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ All notable changes to DreamCoder will be documented in this file. - **Providers in OpenAI format reuse their prompt cache.** Later turns of a session now hit the cache for about 99% of the input (measured on OpenRouter with GPT-4.1 mini and GLM-4.6), and cache hits that DeepSeek and Moonshot report are counted as cache reads (#262). - Providers in OpenAI format can run several tool calls in one step and get tool definitions without schema fields they do not use, and replies show the model you chose instead of the dated version the provider reports (#262). - Every request the app itself sends to a provider (through the local proxy, for session titles and for connection tests) is recorded in the diagnostics log with the provider, model, status, timings, the provider's request id and how it ended, including errors in the middle of a stream and clients that disconnect; never the key or the request body (#262). -- **A stalled reply ends after the AI request timeout.** When a provider stops sending anything for longer than the timeout in the network settings, the agent gives up on that reply and asks again without streaming, instead of waiting until you stop it. This now applies to every provider, whether it is reached directly or through the local proxy; keep-alive signals alone do not count as progress (#262). +- **A stalled reply ends after the AI request timeout.** When a provider stops sending anything for longer than the timeout in the network settings, the agent gives up on that reply and starts it again, instead of waiting until you stop it. This now applies to every provider, whether it is reached directly or through the local proxy; keep-alive signals alone do not count as progress (#262). - Providers in OpenAI format can carry extra request headers and query parameters, such as Azure's `api-version`. The settings form does not offer them yet; for now they are set by editing the provider in `providers.json`. Authentication and the content type still come from the provider settings and override a header of the same name (#262). ### Fixed @@ -24,6 +24,14 @@ All notable changes to DreamCoder will be documented in this file. - With a manual HTTP proxy set in the network settings, the agent reaches the app's local proxy directly instead of sending those requests to the HTTP proxy (#262). - Replies from some providers no longer come back empty; the provider's error message is shown instead (#262). - `/context` in the terminal no longer sends a request to the model for each part of the context when the provider is reached through the local proxy; it uses the local estimate (#262). +- **A session that outgrows a third-party model's context can be compacted again.** Context overflow errors from OpenAI, OpenRouter, DashScope, Moonshot, Zhipu and Gemini are recognized as "Prompt is too long", so `/compact` drops the oldest turns and succeeds instead of failing the same way (#263). +- Auto-compaction also triggers for providers that do not report token usage; the agent estimates the context itself instead of counting it as empty (#263). +- Errors from providers in OpenAI format show the provider's own message, such as the reason for a 429, instead of a cut-off JSON fragment, and the agent waits as long as the provider's `retry-after` asks before trying again (#263). +- A reply that the provider stops with its content filter or because of an error on its side, a stream that closes before the reply is complete, and a reply with no content now show an error instead of looking finished (#263). +- Replies from providers in OpenAI format no longer lose text when one chunk carries reasoning, text and a tool call together, and Responses-format replies keep text and tool arguments that only arrive in completed events (#263). +- **A wrong API key or an empty balance is reported at once.** For providers that use an API key, the agent no longer retries authentication failures and quota or balance errors for minutes before showing them; it shows the provider's message right away (#263). +- When the output limit cuts off a tool call in the middle of its arguments, the agent no longer runs the tool with missing arguments; the model is told the call was cut off and retries with smaller pieces (#263). +- **A reply that breaks off in the middle is retried as a stream.** When a provider's stream fails partway, the agent removes the half-finished reply from the conversation, the app and the session history, and asks again with streaming, instead of switching to a slower request without streaming that long answers could not finish within the timeout. Retries before and during a stream share one limit (#263). ## [0.9.0] — 2026-11-17 diff --git a/desktop/src/stores/chatStore.test.ts b/desktop/src/stores/chatStore.test.ts index 598a163f..54dab760 100644 --- a/desktop/src/stores/chatStore.test.ts +++ b/desktop/src/stores/chatStore.test.ts @@ -526,6 +526,62 @@ describe('chatStore history mapping', () => { vi.useRealTimers() }) + it('drops what a retried reply already streamed and keeps the earlier turn', () => { + vi.useFakeTimers() + const earlier: UIMessage[] = [ + { id: 'u1', type: 'user_text', content: 'do it', timestamp: 1 }, + { id: 'a1', type: 'assistant_text', content: 'Reading the file.', transcriptMessageId: 'cli-a1', timestamp: 2 }, + { id: 't1', type: 'tool_use', toolName: 'Read', toolUseId: 'toolu_read', input: {}, timestamp: 3 }, + { id: 'r1', type: 'tool_result', toolUseId: 'toolu_read', content: 'file', isError: false, timestamp: 4 }, + ] + useChatStore.setState({ sessions: { [TEST_SESSION_ID]: makeSession({ messages: earlier, chatState: 'streaming' }) } }) + const send = (msg: Parameters['handleServerMessage']>[1]) => + useChatStore.getState().handleServerMessage(TEST_SESSION_ID, msg) + + send({ type: 'thinking', text: 'planning' }) + send({ type: 'content_start', blockType: 'text' }) + send({ type: 'content_delta', text: 'Now I write' }) + send({ type: 'message_id', id: 'cli-a2' }) + send({ type: 'content_start', blockType: 'tool_use', toolName: 'Write', toolUseId: 'toolu_write' }) + send({ type: 'content_delta', toolInput: '{"file_pa' }) + send({ type: 'content_start', blockType: 'tool_use', toolName: 'Grep', toolUseId: 'toolu_sub', parentToolUseId: 'toolu_agent' }) + send({ type: 'tool_use_complete', toolName: 'Grep', toolUseId: 'toolu_sub', input: { pattern: 'x' }, parentToolUseId: 'toolu_agent' }) + send({ type: 'content_start', blockType: 'text' }) + send({ type: 'content_delta', text: 'half a sen' }) + send({ type: 'content_start', blockType: 'tool_use', toolName: 'Edit', toolUseId: 'toolu_edit' }) + send({ type: 'content_delta', toolInput: '{"old_str' }) + vi.runOnlyPendingTimers() + send({ type: 'thinking', text: 'checking again' }) + vi.runOnlyPendingTimers() + send({ type: 'content_delta', text: 'and then' }) + vi.runOnlyPendingTimers() + expect(useChatStore.getState().sessions[TEST_SESSION_ID]).toMatchObject({ + activeToolUseId: 'toolu_edit', + activeToolName: 'Edit', + activeThinkingId: expect.any(String), + streamingText: 'and then', + streamingToolInput: '{"old_str', + }) + + send({ type: 'reply_retracted' }) + vi.runOnlyPendingTimers() + + const session = useChatStore.getState().sessions[TEST_SESSION_ID]! + expect(session.messages).toEqual([ + ...earlier, + expect.objectContaining({ type: 'tool_use', toolUseId: 'toolu_sub', parentToolUseId: 'toolu_agent' }), + ]) + expect(session).toMatchObject({ + chatState: 'thinking', + streamingText: '', + streamingToolInput: '', + activeToolUseId: null, + activeToolName: null, + activeThinkingId: null, + }) + vi.useRealTimers() + }) + it('builds on history and keeps only live messages history does not have yet', async () => { vi.mocked(sessionsApi.getTranscript).mockResolvedValueOnce(transcriptOf([ { id: 'u1', type: 'user_text', content: 'prompt', transcriptMessageId: 'u1', timestamp: 1 }, diff --git a/desktop/src/stores/chatStore.ts b/desktop/src/stores/chatStore.ts index 53a102ba..e9ae5c22 100644 --- a/desktop/src/stores/chatStore.ts +++ b/desktop/src/stores/chatStore.ts @@ -583,6 +583,18 @@ function mergeHistoryWithLiveMessages(history: UIMessage[], live: UIMessage[]): const liveStartBySession = new Map() +const STREAMED_REPLY_TYPES = new Set(['assistant_text', 'thinking', 'tool_use']) + +function dropRetractedReply(messages: UIMessage[]): UIMessage[] { + const replyStart = messages.findLastIndex((message) => message.type === 'user_text' || message.type === 'tool_result') + 1 + const kept = messages.filter((message, index) => + index < replyStart || + !STREAMED_REPLY_TYPES.has(message.type) || + (message.type === 'tool_use' && !!message.parentToolUseId), + ) + return kept.length === messages.length ? messages : kept +} + function tagLatestAssistantText(messages: UIMessage[], pendingText: string, id: string, timestamp: number): UIMessage[] { for (let index = messages.length - 1; index >= 0; index--) { const message = messages[index]! @@ -1347,6 +1359,21 @@ export const useChatStore = create((set, get) => ({ break } + case 'reply_retracted': + textDeltaBuffer.clear(sessionId) + thinkingDeltaBuffer.clear(sessionId) + toolInputDeltaBuffer.clear(sessionId) + update((s) => ({ + messages: dropRetractedReply(s.messages), + streamingText: '', + streamingToolInput: '', + activeThinkingId: null, + activeToolUseId: null, + activeToolName: null, + chatState: 'thinking', + })) + break + case 'api_retry': { const attempt = Math.max(1, Math.trunc(msg.attempt)) const maxRetries = Math.max(attempt, Math.trunc(msg.maxRetries)) diff --git a/docs/features/harness/README.md b/docs/features/harness/README.md index 73a64039..1dd39f9c 100644 --- a/docs/features/harness/README.md +++ b/docs/features/harness/README.md @@ -66,9 +66,9 @@ DreamCoder 的 CLI 派生自 Claude Code 2.1.92(`src/constants/claudeCodeCompa - 推理强度:CLI 对自定义提供方发 `thinking:{type:'adaptive'}`,代理只在带预算或类型为 `enabled` 时写 `reasoning_effort`(`src/server/proxy/transform/anthropicToOpenaiChat.ts:64-72`),桌面上选的 Effort 不起作用。 - 输出上限:`max_tokens` 不转发,实际用上游默认值。 -- 错误:上游错误统一包成 `api_error`,`retry-after` 丢失;CLI 针对 Anthropic 的错误识别(上下文过长、过载)对翻译后的错误不命中。 +- 错误:上游错误统一包成 `api_error`,`retry-after` 丢失;CLI 针对 Anthropic 的错误识别(上下文过长、过载)对翻译后的错误不命中。已在 #263 修复:代理解析上游错误并透传 `retry-after`,代理和 CLI 按同一张表分类(见 proxy/transport.md 的 P-NET-03、P-NET-04)。 - 推理内容:只对 DeepSeek 一类上游写回历史里的 `reasoning_content`,Kimi、GLM 的会丢。 -- 图片:工具结果里的图片被换成 `[image omitted]`(`src/server/proxy/transform/toolArguments.ts:38`)。 +- 图片:工具结果里的图片被换成 `[image omitted]`(`src/server/proxy/transform/toolArguments.ts:31`)。 - 工具调用:chat 流里没有 id 的工具调用整个被丢掉。 详见 [model-adaptation.md](comparison/model-adaptation.md)。 @@ -100,6 +100,7 @@ DreamCoder 的 CLI 派生自 Claude Code 2.1.92(`src/constants/claudeCodeCompa - 外部构建里没有「完成前先验证」的条文。 - 没有空响应重试和重复调用提醒(ZCode 有)。 - 不在内置表里的模型(如 GLM-5.3、Kimi-K3),上下文窗口按 200K 计。 +- 工具在一次模型回复读完之后才执行:`tengu_streaming_tool_execution2` 在 DreamCoder 里打不开(见 loop-execution.md)。 详见 [loop-execution.md](comparison/loop-execution.md)。 diff --git a/docs/features/harness/comparison/loop-execution.md b/docs/features/harness/comparison/loop-execution.md index dd229a2e..7a488b13 100644 --- a/docs/features/harness/comparison/loop-execution.md +++ b/docs/features/harness/comparison/loop-execution.md @@ -56,7 +56,7 @@ - Codex:`handle_output_item_done` 构造 future,`tokio::spawn` 执行(`codex:core/src/stream_events_utils.rs:~345-372`;`codex:core/src/tools/parallel.rs:201`);读写锁分配(`parallel.rs:213-217`);`exec_command` 声明可并行(`codex:core/src/tools/handlers/unified_exec/exec_command.rs:135`),apply_patch handler 默认不可并行(`codex:tools/src/tool_executor.rs:122`)。错误输出(`parallel.rs:338-384`;`codex:core/src/tools/registry.rs:963-966`)。exec 审批策略固定 Never(`codex:exec/src/lib.rs:591`),`request_user_input` 与文件变更审批被拒(`exec/src/lib.rs:2122-2143`);需要提权时返回「approval policy is Never; reject command — you cannot ask for escalated permissions ...」(`exec_command.rs:~330`)。 - ZCode:`ToolScheduler` 并发 10,判定规则(`zcode:core/src/tool/scheduler.ts:45, 90-185`);流中执行条件 `modelStreaming === "on"`(headless 为 on,`zcode:cli/src/prompt-command.ts:270`)且 `streamingToolExecution` 默认 `readOnly`(`runtime/methods/streaming-tool-coordinator.ts:383-406`);排序(`runtime/methods/turn-tools.ts:47-57`)。错误措辞与 DreamCoder 同一套(`tool/input-validation-model-content.ts:12-68`;`tool/executor/errors.ts:44-58`)。`zcode -p` 默认 yolo(`zcode:cli/src/run.ts:52`),但判为 ask 的调用经 deny broker 拒绝(`permission/broker.ts:22-37`;`permission/service.ts:121-150`;`prompt-command.ts:250-253`;`tool/executor/errors.ts:136-160`)。 -- DreamCoder:流式执行门 `tengu_streaming_tool_execution2`(`src/query/config.ts:33`),GrowthBook 无 clientKey 时关闭(`src/services/analytics/growthbook.ts:173, 422-427, 819-821`),所以由 `runTools` 在响应结束后执行(`src/query.ts:1368-1370`)。并发上限 `CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY` 默认 10(`src/services/tools/toolOrchestration.ts:8-112`)。错误类型(`src/services/tools/toolExecution.ts:402, 474-482, 671, 724`)。权限拒绝返回 is_error 的 tool_result(`toolExecution.ts:1030-1040`),文案见 `src/utils/messages.ts:217-241`。应用用 `--sdk-url` 强制 stdio 权限提示并转发服务器(`src/cli/print.ts:851-853, 4315-4317`);无人响应时是等待而不是自动拒绝(推断)。 +- DreamCoder:流式执行门 `tengu_streaming_tool_execution2`(`src/query/config.ts:33`),GrowthBook 无 clientKey 时关闭(`src/services/analytics/growthbook.ts:173, 422-427, 819-821`),所以由 `runTools` 在响应结束后执行(`src/query.ts:1369-1371`)。#263 的流式重试依赖这一点:读流中途出错时,失败那次的工具调用还没有执行,撤回后重来不会重复执行;以后要打开这个开关,必须先补上保护,已有工具开始执行时不再重试这次流。并发上限 `CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY` 默认 10(`src/services/tools/toolOrchestration.ts:8-112`)。错误类型(`src/services/tools/toolExecution.ts:402, 474-482, 671, 724`)。权限拒绝返回 is_error 的 tool_result(`toolExecution.ts:1030-1040`),文案见 `src/utils/messages.ts:217-241`。应用用 `--sdk-url` 强制 stdio 权限提示并转发服务器(`src/cli/print.ts:851-853, 4315-4317`);无人响应时是等待而不是自动拒绝(推断)。 ### 2.3 编辑文件 diff --git a/docs/features/harness/proxy/request.md b/docs/features/harness/proxy/request.md index 23525553..ed02df5d 100644 --- a/docs/features/harness/proxy/request.md +++ b/docs/features/harness/proxy/request.md @@ -52,7 +52,7 @@ - 严重程度:高 - 现状: 1. Read 工具读整份 PDF 时,tool_result 只写 `PDF file read: <路径> (<大小>)`(`src/tools/FileReadTool/FileReadTool.ts:670-676`),PDF 内容放在随后一条 meta user 消息的 `document` 块里(`FileReadTool.ts:997-1011`)。是否走这条路只看 `isPDFSupported()`,它只排除 `claude-3-haiku`(`src/utils/pdfUtils.ts:59-61`),对第三方模型都为真。 - 2. chat 转换 `convertUserMessage` 只处理 text、image、tool_result(`anthropicToOpenaiChat.ts:108-121`),`document` 块没有分支,被直接跳过;Responses 转换同样只处理 text、image、tool_use、tool_result(`anthropicToOpenaiResponses.ts:82-105`)。只有出现在 tool_result 内部时才会留下 `[omitted content: document]`(`src/server/proxy/transform/toolArguments.ts:43-44`)。 + 2. chat 转换 `convertUserMessage` 只处理 text、image、tool_result(`anthropicToOpenaiChat.ts:108-121`),`document` 块没有分支,被直接跳过;Responses 转换同样只处理 text、image、tool_use、tool_result(`anthropicToOpenaiResponses.ts:82-105`)。只有出现在 tool_result 内部时才会留下 `[omitted content: document]`(`src/server/proxy/transform/toolArguments.ts:36-37`)。 - 场景:Qwen 预设或任何自定义 OpenAI 格式提供方,用户让模型「读一下 spec.pdf 并总结」。模型调用 Read,收到「PDF file read: spec.pdf (120KB)」,后面没有任何内容,也没有提示内容缺失;模型据此编造总结,或认为已读而不再尝试 `pages` 参数(该参数会把页面转成图片,见 `FileReadTool.ts:886-944`)。 - 参考:ZCode 对 file 块:模型声明支持 PDF 时作为 SDK file part 发送,不支持时换成明确的「不支持该媒体」文本(`zcode:apps/zcode-cli/packages/adapters/src/model/transform.ts:540-563`;`zcode:apps/zcode-cli/packages/adapters/src/model/media-transform-policy.ts:8-15`;基线规则 `supportsPdf:false`,`modelRules[0]`)。Codex:无 PDF 读取。 - 改法方向:代理对 `document` 块不再静默跳过:模型支持 PDF 输入时转成 chat 的 file 内容块 / Responses 的 `input_file`,否则换成一段明确文本(说明 PDF 未送达、可用 `pages` 参数按页读取);或由服务端告知 CLI 当前提供方不支持 PDF,让 `isPDFSupported` 为假。 @@ -62,7 +62,7 @@ - 类型:问题 - 严重程度:高 -- 现状:tool_result 的内容块经 `extractToolResultContent` 拍平成字符串,图片变成 `[image omitted: ]`(`toolArguments.ts:29-38`),chat(`anthropicToOpenaiChat.ts:114-120`)和 Responses(`anthropicToOpenaiResponses.ts:98-104`)都这样。产生图片结果的工具:Read 读图片(`FileReadTool.ts:650-666`)、Bash 输出 data URI(`src/tools/BashTool/utils.ts:71-90`)、内置浏览器截图(MCP 内容 `{type:'image'}`,`src/server/browserHost/service.ts:826`)以及其他返回图片的 MCP 工具。 +- 现状:tool_result 的内容块经 `extractToolResultContent` 拍平成字符串,图片变成 `[image omitted: ]`(`toolArguments.ts:22-31`),chat(`anthropicToOpenaiChat.ts:114-120`)和 Responses(`anthropicToOpenaiResponses.ts:98-104`)都这样。产生图片结果的工具:Read 读图片(`FileReadTool.ts:650-666`)、Bash 输出 data URI(`src/tools/BashTool/utils.ts:71-90`)、内置浏览器截图(MCP 内容 `{type:'image'}`,`src/server/browserHost/service.ts:826`)以及其他返回图片的 MCP 工具。 - 场景:自定义 `openai_chat` 接一个视觉模型(例如 `qwen3-vl-plus` 或 OpenRouter 上的 Claude/GPT),用户让 agent 在内置浏览器里检查页面布局。截图工具返回 JPEG,上游只收到 `[image omitted: image/jpeg]`,模型无法完成任务,通常会反复截图,多花轮数。 - 参考: - ZCode(chat):tool 消息里保留文本,图片移到紧随其后的一条 user 消息,并按模型输入能力决定保留还是换成说明文字(`zcode:apps/zcode-cli/packages/adapters/src/model/transform.ts:59-66,120-151`;`tool-result-media-projection.ts:26-39`)。 @@ -236,7 +236,7 @@ DreamCoder 缺少的配置维度:推理字段的写法与档位、输出上限 - `tool_choice`:auto→auto、any→required、none→none、tool→指定函数(`anthropicToOpenaiChat.ts:175-187`;Responses 为扁平形状 `anthropicToOpenaiResponses.ts:125-137`)。CLI 唯一发 `type:'tool'` 的地方是 WebSearch 的小模型分支,受 `tengu_plum_vx3` 控制、默认关(`WebSearchTool.ts:395-414`)。 - `stop_sequences`:chat 转为 `stop`,Responses 丢弃(`anthropicToOpenaiChat.ts:43-45`)。CLI 只在 yolo 分类器和 sideQuery 中使用(`src/utils/permissions/yoloClassifier.ts:792`;`src/utils/sideQuery.ts:187`),本场景默认不触发。 - `temperature`:只在思考关闭时发 1(`claude.ts:1741-1745`),即 WebFetch 二次处理、hook 等辅助请求和用户关闭思考时;未找到会因此报错的具体模型。 -- `defer_loading` 与 `tool_reference`:代理场景下 ToolSearch 默认关闭(ANTHROPIC_BASE_URL 非 Anthropic 官方,`src/utils/toolSearch.ts:296-309`),服务端不设 `ENABLE_TOOL_SEARCH`;用户手动开启时 `tool_reference` 会变成 `[omitted content: tool_reference]`(`toolArguments.ts:43-44`),所有工具仍全量发送。 +- `defer_loading` 与 `tool_reference`:代理场景下 ToolSearch 默认关闭(ANTHROPIC_BASE_URL 非 Anthropic 官方,`src/utils/toolSearch.ts:296-309`),服务端不设 `ENABLE_TOOL_SEARCH`;用户手动开启时 `tool_reference` 会变成 `[omitted content: tool_reference]`(`toolArguments.ts:36-37`),所有工具仍全量发送。 - 工具名长度:MCP 工具名只做字符替换、不截断(`src/services/mcp/normalization.ts:17-23`;`mcpStringUtils.ts:50-52`),超过 64 字符时 Anthropic 与 OpenAI 端点都会拒绝,不是代理特有的问题。 - 空内容:空数组 content 转成 `content: ''`(`anthropicToOpenaiChat.ts:93-96`);ZCode 用 `(no content)` 防止提供方把空串当缺失(`zcode:apps/zcode-cli/packages/adapters/src/model/transform.ts:457-469`)。没找到 CLI 发出空 user content 的具体路径,未列为发现。 - 多个 assistant 文本块用 `+=` 直接相连、没有分隔(`anthropicToOpenaiChat.ts:144-145`);CLI 把 tool_use 重排为连续段(`messages.ts:2318-2330`),两段文本会被粘在一起。影响很小,未列为发现。 @@ -259,7 +259,7 @@ DreamCoder 缺少的配置维度:推理字段的写法与档位、输出上限 | `max_tokens` 和 effort 不转发 | 确认,并补充 | `types.ts:35-36,112` 有字段无赋值;`types.ts:161-181` 无 `output_config`。补充:`thinking: adaptive/disabled` 也不产生任何字段,带 budget 的映射在现有配置下走不到;直接转发 32000 会超出部分模型上限(P-REQ-02、P-REQ-05)。 | | 上游错误统一包成 `api_error`,响应头(含 `retry-after`)丢失 | 不在本部分范围 | 代码位置与描述一致(`handler.ts:66-71`;`client.ts:83-89`),留给错误方向的评审。 | | `reasoning_content` 只对 deepseek 与 opencode.ai 域名写回 | 确认,并补充 | `client.ts:206-210,251-256`;`anthropicToOpenaiChat.ts:146-147,168-170`。补充:同一判断还打开 `thinking.type` 透传,对 opencode.ai 上的所有模型生效;Responses 路径完全不回传(P-REQ-07、P-REQ-08)。 | -| 工具结果里的图片变成 `[image omitted]` | 确认,并补充 | `toolArguments.ts:37-38`,chat 与 Responses 两条路径都是。补充:PDF 的 `document` 块不在 tool_result 里,被完全静默丢弃(P-REQ-03);Responses 本可在 `function_call_output` 里带图片(P-REQ-04)。 | +| 工具结果里的图片变成 `[image omitted]` | 确认,并补充 | `toolArguments.ts:30-31`,chat 与 Responses 两条路径都是。补充:PDF 的 `document` 块不在 tool_result 里,被完全静默丢弃(P-REQ-03);Responses 本可在 `function_call_output` 里带图片(P-REQ-04)。 | | 流式工具调用没有 id 时被丢弃 | 不在本部分范围 | 响应方向。 | | `cache_control` 被丢弃,不发缓存键或会话头 | 确认,并补充 | `types.ts:149-181` 只声明不使用;`client.ts:213,221` 只有两个头。补充:CLI 已经给代理带了 `X-Claude-Code-Session-Id` 和 `metadata.user_id`,代理没读(P-REQ-10)。 | diff --git a/docs/features/harness/proxy/response.md b/docs/features/harness/proxy/response.md index 08153e6e..008daece 100644 --- a/docs/features/harness/proxy/response.md +++ b/docs/features/harness/proxy/response.md @@ -8,6 +8,7 @@ ### P-RES-01 上游没给 usage 时代理写 0,CLI 的上下文估算以 0 为基准,自动压缩不会触发 +- 状态:已在 #263 修复:代理在 chunk 顶层没有 usage 时读 `choices[0].usage`(流式和非流式);上游没给 usage 时 `message_delta` 不带 usage,不再写 0。CLI 的 `getTokenUsage` 把全 0 的 usage 当作没有,`tokenCountWithEstimation` 继续往前找有效 usage,找不到就整段本地估算,自动压缩照常按估算触发。 - 类型:问题 - 严重程度:高 - 现状: @@ -60,6 +61,7 @@ ### P-RES-04 上游在结束标记之前正常关闭连接,被当成正常结束,回答被截断且不重试 +- 状态:已在 #263 修复:chat 流既没有 `finish_reason` 也没有 `[DONE]` 就结束时、Responses 流没有 `response.completed`/`incomplete`/`failed` 就结束时,代理发 SSE `error`(「The provider closed the stream before the reply finished.」)。只有 `[DONE]` 没有 `finish_reason` 仍按正常结束。CLI 收到后撤回部分回复并重新建立流,见 P-RES-06。直连路径(anthropic 格式)上,已有块完成、但没有 message_delta 就正常关闭的流仍按正常结束处理,列为后续。 - 类型:问题 - 严重程度:中 - 现状: @@ -75,6 +77,7 @@ ### P-RES-05 finish_reason 映射过粗:内容过滤、资源不足、上下文用满都变成 end_turn;Responses 的内容过滤变成 max_tokens +- 状态:已在 #263 修复:chat 流和非流式共用一个映射:`model_context_window_exceeded` 原样传给 CLI;`content_filter`、`sensitive` 转成 `invalid_request_error`(「The provider's content filter stopped the reply ...」);`network_error`、`insufficient_system_resource` 转成 `api_error`;其余未知取值仍按 end_turn。Responses 的 `incomplete` 按 `incomplete_details.reason` 区分:`max_output_tokens` 或没有原因时为 max_tokens,`content_filter` 同上,其他原因转成 `api_error`。流式时以 SSE `error` 发出,非流式时为对应状态码的错误响应。CLI 专门处理的 stop_reason 只有 max_tokens、model_context_window_exceeded 和 refusal(src/services/api/claude.ts:2321-2355),其余都按正常结束;refusal 的文案指向 Anthropic 使用政策,所以内容过滤不映射到它。 - 类型:问题 - 严重程度:中 - 现状: @@ -95,6 +98,7 @@ ### P-RES-06 流中途的上游错误丢掉错误码和状态,CLI 一律改走非流式请求,长回答在 120 秒总时长内完不成 +- 状态:已在 #263 修复:流中途的错误用 P-NET-04 的同一张分类表映射:上下文溢出转成 `prompt is too long`,额度或余额不足为 `billing_error`,鉴权失败为 `authentication_error`,限流为 `rate_limit_error`,其余为 `api_error`。服务端给 CLI 设 `CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK=1`,流出错后不再改走非流式请求。报告里「设了这个变量就交给 withRetry 用流式重试」不成立:withRetry 只包住建立流这一步(src/services/api/claude.ts 的 withRetry 调用),读流在它外面,单设变量时流中途的错误直接成为错误消息。所以 #263 在 `queryModel` 里把「建立流加读流」放进一个外层循环:读流出错且 `isRetryableStreamError` 判为可重试(不是用户中止,不是 invalid_request、鉴权、权限、额度一类)时,撤回已产出的部分消息,按 withRetry 的退避重新建立流。外层重来和 withRetry 内部的重试共用一个次数:整个调用的请求总数不超过 `getDefaultMaxRetries() + 1`。撤回的去向见 docs/features/session-state/tech-design.md 的 `reply_retracted`。 - 类型:问题 - 严重程度:中 - 现状: @@ -113,6 +117,7 @@ ### P-RES-07 被截断或损坏的工具参数在 CLI 里变成 `{}`,模型看到“缺参数”而不是“参数被截断” +- 状态:已在 #263 修复:参数解析统一在 CLI 的 `normalizeContentFromAPI`(src/utils/messages.ts)。参数为空串或 JSON `null` 时视为没有参数(`{}`)。其余情况下,非流式代理在参数不是合法 JSON 对象时把原始字符串交给 CLI,不再生成 `{raw}`;CLI 解析失败或得到的不是对象时,input 写成保留键 `__dreamcoder_invalid_arguments: { receivedChars, parseError }`(只记长度和错误,不记原文)。`runToolUse` 见到保留键就不执行工具,回给模型一条 is_error 的说明:参数不完整,很可能被输出上限截断,请缩短参数重试。保留键的写入和判断都在 `src/utils/invalidToolArguments.ts`。Anthropic 官方路径上因 max_tokens 截断的工具调用走同一条路。桌面端把这种调用显示为没有摘要的工具卡片,输入区以 JSON 显示保留键,结果区显示错误说明。 - 类型:问题 - 严重程度:中 - 现状: @@ -130,6 +135,7 @@ ### P-RES-08 同一个 delta 同时带正文、推理或工具调用时,只处理其中一种,其余被丢弃 +- 状态:已在 #263 修复:同一个 delta 依次处理推理、正文和工具调用,每类按原有规则开关块。 - 类型:问题 - 严重程度:中 - 现状:`detectBlockTransition` 按 tool_calls → content → reasoning 的顺序只返回一种类型(openaiChatStreamToAnthropic.ts:270-292),`processChunk` 只调用对应的一个处理函数(:320-330)。同一 delta 里的其余字段直接丢弃。 @@ -144,6 +150,7 @@ ### P-RES-09 Responses 流转换只依赖增量事件,不读完整 item,也不在结束时关闭文本和工具块 +- 状态:已在 #263 修复:文本增量没有 `content_part.added` 时自动开块;`output_item.added`、`function_call_arguments.done`、`output_text.done` 和 `output_item.done` 里的完整参数或文本在没有收到增量时补发;`response.completed` 和 `incomplete` 时关闭所有还开着的块。模型名一项 #262 已改为使用请求的模型名。 - 类型:问题 - 严重程度:低 - 现状: @@ -179,6 +186,7 @@ ### P-RES-12 有合法 chunk 但没有任何内容的回复,CLI 静默结束 +- 状态:已在 #263 修复:chat 流正常结束(end_turn)时,如果没有任何内容块也没有 usage,代理发 SSE `error`(「The provider returned an empty reply.」)。因输出上限结束(max_tokens)的不算空回复。 - 类型:优化 - 严重程度:低 - 现状:chat 流里只要出现过带 `choices` 的 chunk 就算合法(openaiChatStreamToAnthropic.ts:121),结束时发 end_turn(:518-527)。CLI 的空流检查要求“没有块且没有 stop_reason”(claude.ts:2400),此时 stop_reason 已是 end_turn,检查不触发,本次请求不产出任何助手消息。 diff --git a/docs/features/harness/proxy/transport.md b/docs/features/harness/proxy/transport.md index 99733036..26c489fe 100644 --- a/docs/features/harness/proxy/transport.md +++ b/docs/features/harness/proxy/transport.md @@ -31,6 +31,7 @@ ### P-NET-02 提供方的上下文溢出错误认不出来,会话卡死,手动 /compact 也救不回来 +- 状态:已在 #263 修复:溢出错误的识别表放在 `src/utils/providerErrors.ts`,代理和 CLI 共用。代理把识别出的溢出错误改写成 400 `invalid_request_error`,消息以 `prompt is too long` 开头,能解析出数字时带上 `N tokens > M maximum`;CLI 的 `errors.ts` 对直连路径用同一张表识别,并能从 OpenAI、OpenRouter、Gemini、Moonshot 的文案里解析出 token 数。`/compact` 的截头重试因此能按超出的 token 数删掉最早的几轮。OpenRouter 实测文案:`This endpoint's maximum context length is 128000 tokens. However, you requested about 270016 tokens ...`,它在转发前就拒绝,流式、非流式和 Responses 端点相同。本构建里被动压缩是空实现,溢出后需要用户手动 `/compact`。 - 类型:问题 - 严重程度:高 - 现状: @@ -46,6 +47,7 @@ ### P-NET-03 代理转发错误时把原因转义进字符串、丢掉响应头,429 时用户看不到原因 +- 状态:已在 #263 修复:代理解析上游错误正文,取出 message 和 code,`error.type` 按状态码映射,原文作为 `error.message`;透传 `retry-after`、`retry-after-ms` 和 `x-request-id`,并把 request id 也写进 `request-id`(SDK 读这个头)。按错误内容决定是否重试见 P-NET-04。 - 类型:问题 - 严重程度:中 - 现状: @@ -61,6 +63,7 @@ ### P-NET-04 重试分类按 Anthropic 设计,对第三方提供方会重试不该重试的错误、等待过久 +- 状态:已在 #263 修复:分类表在 `src/utils/providerErrors.ts`,代理和 CLI 直连路径共用。代理:额度或余额不足(HTTP 402,`insufficient_quota`、`exceeded_current_quota_error`、`billing_hard_limit_reached`、智谱 1113 等,原来是 429 的也算)改成 402 `billing_error`;401、403 保留状态码;这两类都加 `x-should-retry: false`,CLI 不再重试。限流保留 429 和 `retry-after`;5xx 照常重试,503 不改成 `overloaded_error`;2xx 正文里的错误按内容分类,分不出的为 502。CLI 直连路径:非 Anthropic 官方地址上的额度类错误不重试;401 只在能刷新凭据时重试(Anthropic 官方地址、配置了 apiKeyHelper、Claude 登录或 OpenAI 登录)。已知限制:代理自己的超时和网络错误仍按 5xx 重试,上游完全不回响应头时,非流式请求和等待响应头的阶段最坏要等「重试次数 × AI 请求超时」。 - 类型:问题 - 严重程度:中 - 现状(主循环请求 `maxRetries` 默认 10,即最多 11 次,src/services/api/withRetry.ts:52, 789-794;SDK 自身重试关闭,claude.ts:1831): diff --git a/docs/features/session-state/tech-design.md b/docs/features/session-state/tech-design.md index 8e829f84..5cdb5a50 100644 --- a/docs/features/session-state/tech-design.md +++ b/docs/features/session-state/tech-design.md @@ -44,7 +44,7 @@ DreamCoder 的每个会话由服务端拉起一个 CLI 进程执行,桌面端 客户端发:`prewarm_session`、`user_message`、`permission_response`、`set_permission_mode`、`set_runtime_config`、`stop_generation`、`ping`。 -服务端发:流式内容(`content_start`、`content_delta`、`tool_use_complete`、`tool_result`、`thinking`、`message_complete`),状态类(`status`、`permission_mode`、`permission_request`、`session_title_updated`、`team_*`、`task_update`、`background_task`),以及 `error`、`system_notification`、`transcript_item`、`api_retry`。定义在 `src/server/ws/protocol.ts`。 +服务端发:流式内容(`content_start`、`content_delta`、`tool_use_complete`、`tool_result`、`thinking`、`message_complete`、`reply_retracted`),状态类(`status`、`permission_mode`、`permission_request`、`session_title_updated`、`team_*`、`task_update`、`background_task`),以及 `error`、`system_notification`、`transcript_item`、`api_retry`。定义在 `src/server/ws/protocol.ts`。 ### 已核实的主要问题 @@ -300,6 +300,16 @@ CLI 源码注释提到,末尾的 `idle` 事件曾让一些客户端卡在「 - `session_patch`:`version` 是应用这次变化之后的版本号。客户端收到的 `version` 不等于本地版本加一时,丢弃这个 patch 并发 `request_snapshot`。 - `changes` 只包含变化的顶层字段,每个顶层字段整体替换,不做深层合并 `[assumption]`。这样客户端的应用逻辑最简单,代价是 `pendingPermissions` 这类数组每次整体下发,数据量很小,可以接受。 +流式内容另有一条撤回消息(#263): + +```ts +| { type: 'reply_retracted' } +``` + +- 来源:CLI 读流时出错并重新建立流之前,输出 stream-json 系统消息 `{ type: 'system', subtype: 'stream_retracted' }`(`src/QueryEngine.ts`)。 +- 服务端 `translate` 收到后清空这次回复的流式翻译状态(进行中的块、待补全的工具块、已流式发出的工具 id),广播 `reply_retracted`,并用现有的 `turn_activity_reported` 事件经 reducer 把 `turn.activity` 改回 `thinking`。消息内容不在 `SessionState` 里,所以撤回不需要新的状态事件。 +- 客户端(桌面与 H5 同一份 `chatStore`)收到后丢掉流式缓冲,并删掉最后一条用户消息或工具结果之后、属于主线程的 `assistant_text`、`thinking` 和 `tool_use`。 + ### WebSocket:客户端 → 服务端 - 新增 `{ type: 'request_snapshot' }`。 @@ -425,6 +435,7 @@ Store --> C : patch(effective=bypass, pendingRestart=无) - **重启期间的请求**:带着旧代号的强杀、删除操作被忽略;新请求排在 `runtimeTransition` 之后执行(沿用现有的串行队列思路)。 - **持久化写入失败**:内存状态照常生效并广播,记一条诊断事件(`diagnosticsService`),下一次写入重试整条记录。不会因为写盘失败回滚界面。 - **恢复时 transcript 损坏或缺字段**:按恢复顺序退回全局默认,记诊断事件。 +- **读流中途出错后重新建立流**:CLI 在 `mutableMessages` 和 transcript 里删掉失败那次已经完整产出的部分消息(`QueryEngine` 处理 `tombstone`),服务端和客户端按上面的 `reply_retracted` 丢掉已显示的流式内容。重试成功后,客户端只看到重试得到的完整回复,下一轮历史里也没有失败那次的部分消息。 ## 横切关注点 diff --git a/src/QueryEngine.ts b/src/QueryEngine.ts index 9dcb1861..aa2485a8 100644 --- a/src/QueryEngine.ts +++ b/src/QueryEngine.ts @@ -86,6 +86,7 @@ import { setCwd } from './utils/Shell.js' import { flushSessionStorage, recordTranscript, + removeTranscriptMessage, } from './utils/sessionStorage.js' import { asSystemPrompt } from './utils/systemPromptType.js' import { resolveThemeSetting } from './utils/systemTheme.js' @@ -710,6 +711,7 @@ export class QueryEngine { let structuredOutputFromTool: unknown // Track the last stop_reason from assistant messages let lastStopReason: string | null = null + let pendingAssistantRecord: Promise = Promise.resolve() // Reference-based watermark so error_during_execution's errors[] is // turn-scoped. A length-based index breaks when the 100-entry ring buffer // shift()s during the turn — the index slides. If this entry is rotated @@ -773,7 +775,7 @@ export class QueryEngine { // useLogMessages.ts fire-and-forgets. enqueueWrite is // order-preserving so fire-and-forget here is safe. if (message.type === 'assistant') { - void recordTranscript(messages) + pendingAssistantRecord = recordTranscript(messages) } else { await recordTranscript(messages) } @@ -803,9 +805,17 @@ export class QueryEngine { } switch (message.type) { - case 'tombstone': - // Tombstone messages are control signals for removing messages, skip them + case 'tombstone': { + const retractedUuid = message.message.uuid + removeMessageByUuid(this.mutableMessages, retractedUuid) + removeMessageByUuid(messages, retractedUuid) + if (persistSession) { + await pendingAssistantRecord + await flushSessionStorage() + await removeTranscriptMessage(retractedUuid) + } break + } case 'assistant': // Capture stop_reason if already set (synthetic messages). For // streamed responses, this is null at content_block_stop time; @@ -998,6 +1008,14 @@ export class QueryEngine { compact_metadata: toSDKCompactMetadata(message.compactMetadata), } } + if (message.subtype === 'api_error' && message.streamRestart) { + yield { + type: 'system', + subtype: 'stream_retracted' as const, + session_id: getSessionId(), + uuid: message.uuid, + } + } if (message.subtype === 'api_error') { yield { type: 'system', @@ -1397,3 +1415,8 @@ export async function* ask({ setReadFileCache(engine.getReadFileState()) } } + +function removeMessageByUuid(messages: Message[], uuid: string): void { + const index = messages.findIndex(message => message.uuid === uuid) + if (index !== -1) messages.splice(index, 1) +} diff --git a/src/__tests__/queryEngineRetraction.test.ts b/src/__tests__/queryEngineRetraction.test.ts new file mode 100644 index 00000000..38d936e4 --- /dev/null +++ b/src/__tests__/queryEngineRetraction.test.ts @@ -0,0 +1,93 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test' +import { readFile } from 'node:fs/promises' +import { APIError } from '@anthropic-ai/sdk' +import { QueryEngine } from '../QueryEngine.js' +import * as claudeApi from '../services/api/claude.js' +import { getDefaultAppState } from '../state/AppStateStore.js' +import { createFileStateCacheWithSizeLimit } from '../utils/fileStateCache.js' +import { createAssistantMessage, createSystemAPIErrorMessage } from '../utils/messages.js' +import { flushSessionStorage, getTranscriptPath } from '../utils/sessionStorage.js' + +function reply(content: Parameters[0]['content']) { + const message = createAssistantMessage({ content }) + message.message.model = 'claude-sonnet-4-5' + return message +} + +describe('QueryEngine when a stream is retried', () => { + let modelSpy: ReturnType | undefined + const envKeys = ['ANTHROPIC_API_KEY', 'TEST_ENABLE_SESSION_PERSISTENCE'] as const + let savedEnv: Partial> + + beforeEach(() => { + savedEnv = Object.fromEntries(envKeys.map((key) => [key, process.env[key]])) + process.env.ANTHROPIC_API_KEY = 'sk-test' + process.env.TEST_ENABLE_SESSION_PERSISTENCE = '1' + }) + + afterEach(() => { + modelSpy?.mockRestore() + for (const key of envKeys) { + if (savedEnv[key] === undefined) delete process.env[key] + else process.env[key] = savedEnv[key] + } + }) + + test('keeps the whole retried reply, drops the partial one and tells the client', async () => { + const partial = reply('Half an ans') + const text = reply('Let me check.') + const toolUse = reply([{ type: 'tool_use', id: 'toolu_1', name: 'Missing', input: {} }] as never) + const final = reply('Done.') + let calls = 0 + modelSpy = spyOn(claudeApi, 'queryModelWithStreaming').mockImplementation(async function* ({ options }) { + calls += 1 + if (calls > 1) { + yield final + return + } + yield partial + options.onStreamingFallback?.() + yield { + ...createSystemAPIErrorMessage(APIError.generate(undefined, undefined, 'the stream broke', undefined), 500, 1, 10), + streamRestart: true, + } + yield { type: 'stream_event', event: { type: 'message_start', message: { usage: { input_tokens: 10, output_tokens: 0 } } } } + yield text + yield toolUse + yield { type: 'stream_event', event: { type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 5 } } } + } as never) + + let appState = getDefaultAppState() + const engine = new QueryEngine({ + cwd: process.cwd(), + tools: [], + commands: [], + mcpClients: [], + agents: [], + canUseTool: async () => ({ behavior: 'allow' }) as never, + getAppState: () => appState, + setAppState: (update) => { + appState = update(appState) + }, + readFileCache: createFileStateCacheWithSizeLimit(10), + userSpecifiedModel: 'claude-sonnet-4-5', + }) + + const output: Array> = [] + for await (const message of engine.submitMessage('hello')) output.push(message as Record) + await flushSessionStorage() + + const retracted = output.findIndex((m) => m.type === 'system' && m.subtype === 'stream_retracted') + const retry = output.findIndex((m) => m.type === 'system' && m.subtype === 'api_retry') + expect(retracted).toBeGreaterThan(-1) + expect(retracted).toBeLessThan(retry) + const history = engine.getMessages().map((m) => m.uuid) + expect(history).not.toContain(partial.uuid) + expect(history).toEqual(expect.arrayContaining([text.uuid, toolUse.uuid, final.uuid])) + expect(calls).toBe(2) + const transcript = await readFile(getTranscriptPath(), 'utf8') + expect(transcript).not.toContain(partial.uuid) + expect(transcript).toContain(toolUse.uuid) + expect(transcript).toContain(final.uuid) + }) +}) diff --git a/src/entrypoints/sdk/coreSchemas.ts b/src/entrypoints/sdk/coreSchemas.ts index 9204796d..9ade409f 100644 --- a/src/entrypoints/sdk/coreSchemas.ts +++ b/src/entrypoints/sdk/coreSchemas.ts @@ -1646,6 +1646,19 @@ export const SDKAPIRetryMessageSchema = lazySchema(() => ), ) +export const SDKStreamRetractedMessageSchema = lazySchema(() => + z + .object({ + type: z.literal('system'), + subtype: z.literal('stream_retracted'), + uuid: UUIDPlaceholder(), + session_id: z.string(), + }) + .describe( + 'Emitted when a stream broke in the middle and will be opened again. Everything streamed since the last user message or tool result belongs to the failed attempt and is replaced by the retried reply.', + ), +) + export const SDKLocalCommandOutputMessageSchema = lazySchema(() => z .object({ @@ -1922,6 +1935,7 @@ export const SDKMessageSchema = lazySchema(() => SDKCompactBoundaryMessageSchema(), SDKStatusMessageSchema(), SDKAPIRetryMessageSchema(), + SDKStreamRetractedMessageSchema(), SDKLocalCommandOutputMessageSchema(), SDKHookStartedMessageSchema(), SDKHookProgressMessageSchema(), diff --git a/src/query.ts b/src/query.ts index ca2ec844..2a4685da 100644 --- a/src/query.ts +++ b/src/query.ts @@ -739,6 +739,7 @@ async function* queryLoop( toolUseContext, ) } + streamingFallbackOccured = false } // Backfill tool_use inputs on a cloned message before yield so // SDK stream output and transcript serialization see legacy/derived diff --git a/src/server/__tests__/conversation-service.test.ts b/src/server/__tests__/conversation-service.test.ts index 5f7d12f7..ccfd18f0 100644 --- a/src/server/__tests__/conversation-service.test.ts +++ b/src/server/__tests__/conversation-service.test.ts @@ -245,6 +245,7 @@ describe('ConversationService', () => { expect(env.API_TIMEOUT_MS).toBe('180000') expect(env.CLAUDE_ENABLE_STREAM_WATCHDOG).toBe('1') + expect(env.CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK).toBe('1') expect(env.CLAUDE_STREAM_IDLE_TIMEOUT_MS).toBe('180000') expect(env.HTTP_PROXY).toBe('http://127.0.0.1:7890') expect(env.HTTPS_PROXY).toBe('http://127.0.0.1:7890') diff --git a/src/server/__tests__/cron-scheduler-launcher.test.ts b/src/server/__tests__/cron-scheduler-launcher.test.ts index 11b224b8..c328d736 100644 --- a/src/server/__tests__/cron-scheduler-launcher.test.ts +++ b/src/server/__tests__/cron-scheduler-launcher.test.ts @@ -306,10 +306,12 @@ describe('cron scheduler launcher resolution', () => { expect({ API_TIMEOUT_MS: env.API_TIMEOUT_MS, CLAUDE_ENABLE_STREAM_WATCHDOG: env.CLAUDE_ENABLE_STREAM_WATCHDOG, + CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK: env.CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK, CLAUDE_STREAM_IDLE_TIMEOUT_MS: env.CLAUDE_STREAM_IDLE_TIMEOUT_MS, }).toEqual({ API_TIMEOUT_MS: '180000', CLAUDE_ENABLE_STREAM_WATCHDOG: '1', + CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK: '1', CLAUDE_STREAM_IDLE_TIMEOUT_MS: '180000', }) } finally { diff --git a/src/server/__tests__/network-settings.test.ts b/src/server/__tests__/network-settings.test.ts index 11da5e51..f74a69e1 100644 --- a/src/server/__tests__/network-settings.test.ts +++ b/src/server/__tests__/network-settings.test.ts @@ -92,6 +92,7 @@ describe('network settings', () => { expect(getManualNetworkProxyUrl(settings)).toBe('http://127.0.0.1:7890') expect(buildNetworkEnvironment(settings, { NO_PROXY: 'corp.internal' })).toEqual({ API_TIMEOUT_MS: '180000', + CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK: '1', CLAUDE_ENABLE_STREAM_WATCHDOG: '1', CLAUDE_STREAM_IDLE_TIMEOUT_MS: '180000', HTTP_PROXY: 'http://127.0.0.1:7890', diff --git a/src/server/__tests__/providers.test.ts b/src/server/__tests__/providers.test.ts index 8dfba48b..59103aaa 100644 --- a/src/server/__tests__/providers.test.ts +++ b/src/server/__tests__/providers.test.ts @@ -946,6 +946,35 @@ describe('ProviderService', () => { expect(res.status).toBe(404) expect(await res.json()).toMatchObject({ type: 'error', error: { type: 'not_found_error' } }) }) + + test('answers an upstream error in Anthropic form with the retry and request id headers', async () => { + const originalFetch = globalThis.fetch + globalThis.fetch = mock(async () => Response.json( + { error: { message: 'Rate limit reached, please try again in 20s.', code: 'rate_limit_exceeded' } }, + { status: 429, headers: { 'retry-after': '20', 'x-request-id': 'req_abc' } }, + )) as unknown as typeof fetch + + try { + const svc = new ProviderService() + const provider = await svc.addProvider(sampleInput({ apiFormat: 'openai_chat' })) + const req = new Request(`http://localhost:3456/proxy/providers/${provider.id}/v1/messages`, { + method: 'POST', + body: JSON.stringify({ model: 'model-main', max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }), + }) + + const res = await handleProxyRequest(req, new URL(req.url), { timeout: () => {} }) + + expect(res.status).toBe(429) + expect(res.headers.get('retry-after')).toBe('20') + expect(res.headers.get('request-id')).toBe('req_abc') + expect(await res.json()).toEqual({ + type: 'error', + error: { type: 'rate_limit_error', message: 'Rate limit reached, please try again in 20s.' }, + }) + } finally { + globalThis.fetch = originalFetch + } + }) }) describe('discoverModels', () => { @@ -1173,6 +1202,44 @@ describe('ProviderService', () => { } }) + test('shows only the HTTP status when a rejected connection test has an empty body', async () => { + const originalFetch = globalThis.fetch + globalThis.fetch = mock(async () => new Response('', { status: 502 })) as unknown as typeof fetch + + try { + const svc = new ProviderService() + const result = await svc.testProviderConfig({ + baseUrl: 'https://api.example.com', + apiKey: 'sk-test', + modelId: 'gpt-main', + apiFormat: 'openai_chat', + }) + + expect(result.connectivity).toMatchObject({ success: false, error: 'HTTP 502', httpStatus: 502 }) + } finally { + globalThis.fetch = originalFetch + } + }) + + test('shows the HTTP status with the provider message when the connection test is rejected', async () => { + const originalFetch = globalThis.fetch + globalThis.fetch = mock(async () => Response.json({ error: { message: 'Incorrect API key provided' } }, { status: 401 })) as unknown as typeof fetch + + try { + const svc = new ProviderService() + const result = await svc.testProviderConfig({ + baseUrl: 'https://api.example.com', + apiKey: 'sk-test', + modelId: 'gpt-main', + apiFormat: 'openai_chat', + }) + + expect(result.connectivity).toMatchObject({ success: false, error: 'HTTP 401: Incorrect API key provided', httpStatus: 401 }) + } finally { + globalThis.fetch = originalFetch + } + }) + test.each([ ['a JSON error body', '{"error":{"message":"invalid api key"}}', 'invalid api key'], ['an HTML page', 'hi', 'no valid Chat Completions chunks'], @@ -1387,10 +1454,11 @@ describe('Providers API', () => { apiFormat: 'openai_chat', }) const res = await handleProvidersApi(req, url, segments) - const body = (await res.json()) as { error: string } + const body = (await res.json()) as { error: string; message: string } expect(res.status).toBe(400) expect(body.error).toBe('MODEL_DISCOVERY_FAILED') + expect(body.message).toBe('HTTP 401: unauthorized') } finally { server.stop(true) } diff --git a/src/server/__tests__/proxy-streaming.test.ts b/src/server/__tests__/proxy-streaming.test.ts index 6ebcc156..74116358 100644 --- a/src/server/__tests__/proxy-streaming.test.ts +++ b/src/server/__tests__/proxy-streaming.test.ts @@ -1,10 +1,8 @@ import { describe, test, expect } from 'bun:test' import { openaiChatStreamToAnthropic } from '../proxy/streaming/openaiChatStreamToAnthropic.js' import { openaiResponsesStreamToAnthropic } from '../proxy/streaming/openaiResponsesStreamToAnthropic.js' -import { - OpenAIResponsesStreamError, - openaiResponsesStreamToAnthropicResponse, -} from '../proxy/streaming/openaiResponsesStreamToAnthropicResponse.js' +import { openaiResponsesStreamToAnthropicResponse } from '../proxy/streaming/openaiResponsesStreamToAnthropicResponse.js' +import { UpstreamReplyError } from '../upstream/upstreamError.js' import { SseDecoder } from '../proxy/streaming/sse.js' function makeStream(chunks: string[]): ReadableStream { @@ -588,7 +586,7 @@ describe('openaiResponsesStreamToAnthropicResponse upstream errors', () => { 'm', ) - await expect(result).rejects.toBeInstanceOf(OpenAIResponsesStreamError) + await expect(result).rejects.toBeInstanceOf(UpstreamReplyError) await expect(result).rejects.toThrow(message) }) @@ -745,3 +743,280 @@ describe('stream cancellation', () => { expect(upstreamCancelReason).toBe('client gone') }) }) + +function chatChunk(choice: Record, extra?: Record): string { + return `data: ${JSON.stringify({ id: 'c1', object: 'chat.completion.chunk', created: 0, model: 'm', choices: [{ index: 0, ...choice }], ...extra })}\n\n` +} + +describe('chat stream usage', () => { + test('reads usage that the provider puts inside the choice (Moonshot)', async () => { + const events = await collectSse(openaiChatStreamToAnthropic(makeStream([ + chatChunk({ delta: { content: 'Hi' }, finish_reason: null }), + chatChunk({ delta: {}, finish_reason: 'stop', usage: { prompt_tokens: 1000, completion_tokens: 20, total_tokens: 1020 } }), + 'data: [DONE]\n\n', + ]), 'kimi-k2.6')) + + expect(events.find((e) => e.event === 'message_delta')!.data.usage).toEqual({ input_tokens: 1000, output_tokens: 20, cache_read_input_tokens: 0 }) + }) + + test('leaves usage out instead of reporting zero when the provider sends none', async () => { + const events = await collectSse(openaiChatStreamToAnthropic(makeStream([ + chatChunk({ delta: { content: 'Hi' }, finish_reason: null }), + chatChunk({ delta: {}, finish_reason: 'stop' }), + 'data: [DONE]\n\n', + ]), 'm')) + + expect(events.find((e) => e.event === 'message_delta')!.data).not.toHaveProperty('usage') + }) +}) + +function eventNames(events: Array<{ event: string }>): string[] { + return events.map((e) => e.event) +} + +function lastError(events: Array<{ event: string; data: Record }>): { type: string; message: string } { + const last = events.at(-1)! + expect(last.event).toBe('error') + return (last.data as { error: { type: string; message: string } }).error +} + +describe('chat stream finish reasons', () => { + test('passes a full context window through so the CLI can tell the user', async () => { + const events = await collectSse(openaiChatStreamToAnthropic(makeStream([ + chatChunk({ delta: { content: 'Hi' }, finish_reason: null }), + chatChunk({ delta: {}, finish_reason: 'model_context_window_exceeded' }), + 'data: [DONE]\n\n', + ]), 'glm-4.6')) + + expect((events.find((e) => e.event === 'message_delta')!.data.delta as { stop_reason: string }).stop_reason) + .toBe('model_context_window_exceeded') + }) + + test.each(['content_filter', 'sensitive'])('reports %s as an error instead of a normal end', async (reason) => { + const events = await collectSse(openaiChatStreamToAnthropic(makeStream([ + chatChunk({ delta: { content: 'Hi' }, finish_reason: null }), + chatChunk({ delta: {}, finish_reason: reason }), + 'data: [DONE]\n\n', + ]), 'm')) + + expect(eventNames(events)).not.toContain('message_stop') + expect(lastError(events)).toEqual({ + type: 'invalid_request_error', + message: `The provider's content filter stopped the reply (finish_reason: ${reason}).`, + }) + }) + + test.each(['network_error', 'insufficient_system_resource'])('reports %s as a server error', async (reason) => { + const events = await collectSse(openaiChatStreamToAnthropic(makeStream([ + chatChunk({ delta: { content: 'Hi' }, finish_reason: null }), + chatChunk({ delta: {}, finish_reason: reason }), + 'data: [DONE]\n\n', + ]), 'm')) + + expect(eventNames(events)).not.toContain('message_stop') + expect(lastError(events)).toEqual({ + type: 'api_error', + message: `The provider stopped the reply because of an error on its side (finish_reason: ${reason}).`, + }) + }) +}) + +describe('chat stream errors and early ends', () => { + test('keeps a context overflow code from an error in the middle of the stream', async () => { + const events = await collectSse(openaiChatStreamToAnthropic(makeStream([ + chatChunk({ delta: { content: 'Hi' }, finish_reason: null }), + 'data: {"error":{"message":"Your input exceeds the context window of this model.","code":"context_length_exceeded"}}\n\n', + ]), 'm')) + + expect(lastError(events)).toEqual({ + type: 'invalid_request_error', + message: 'prompt is too long · Your input exceeds the context window of this model.', + }) + }) + + test('reports a stream that closes before any end marker', async () => { + const events = await collectSse(openaiChatStreamToAnthropic(makeStream([ + chatChunk({ delta: { content: 'Half an ans' }, finish_reason: null }), + ]), 'm')) + + expect(eventNames(events)).not.toContain('message_stop') + expect(lastError(events)).toEqual({ + type: 'api_error', + message: 'The provider closed the stream before the reply finished.', + }) + }) + + test('accepts [DONE] without a finish reason as a normal end', async () => { + const events = await collectSse(openaiChatStreamToAnthropic(makeStream([ + chatChunk({ delta: { content: 'Hi' }, finish_reason: null }), + 'data: [DONE]\n\n', + ]), 'm')) + + expect(eventNames(events).at(-1)).toBe('message_stop') + }) + + test('reports a reply with chunks but no content and no usage', async () => { + const events = await collectSse(openaiChatStreamToAnthropic(makeStream([ + chatChunk({ delta: { role: 'assistant' }, finish_reason: null }), + chatChunk({ delta: {}, finish_reason: 'stop' }), + 'data: [DONE]\n\n', + ]), 'm')) + + expect(lastError(events)).toEqual({ type: 'api_error', message: 'The provider returned an empty reply.' }) + }) +}) + +describe('chat stream deltas with several kinds of content', () => { + test('keeps reasoning, text and a tool call that arrive in one delta', async () => { + const events = await collectSse(openaiChatStreamToAnthropic(makeStream([ + chatChunk({ + delta: { + reasoning_content: 'think', + content: 'Let me look.', + tool_calls: [{ index: 0, id: 'call_1', type: 'function', function: { name: 'Read', arguments: '{"file_path":"a.ts"}' } }], + }, + finish_reason: 'tool_calls', + }), + 'data: [DONE]\n\n', + ]), 'm')) + + const starts = events.filter((e) => e.event === 'content_block_start').map((e) => (e.data.content_block as { type: string }).type) + const deltas = events.filter((e) => e.event === 'content_block_delta').map((e) => e.data.delta) + expect(starts).toEqual(['thinking', 'text', 'tool_use']) + expect(deltas).toEqual([ + { type: 'thinking_delta', thinking: 'think' }, + { type: 'text_delta', text: 'Let me look.' }, + { type: 'input_json_delta', partial_json: '{"file_path":"a.ts"}' }, + ]) + expect(eventNames(events).at(-1)).toBe('message_stop') + }) +}) + +function responsesEvent(type: string, data: Record): string { + return `event: ${type}\ndata: ${JSON.stringify({ type, ...data })}\n\n` +} + +const responseCreated = responsesEvent('response.created', { response: { id: 'r1', model: 'gpt-4.1' } }) + +function completed(usage?: Record): string { + return responsesEvent('response.completed', { response: { id: 'r1', status: 'completed', ...(usage && { usage }) } }) +} + +describe('responses stream ends', () => { + test('reports an incomplete reply stopped by the content filter as an error', async () => { + const events = await collectSse(openaiResponsesStreamToAnthropic(makeStream([ + responseCreated, + responsesEvent('response.incomplete', { response: { status: 'incomplete', incomplete_details: { reason: 'content_filter' } } }), + ]), 'gpt-4.1')) + + expect(eventNames(events)).not.toContain('message_stop') + expect(lastError(events)).toEqual({ + type: 'invalid_request_error', + message: "The provider's content filter stopped the reply (incomplete: content_filter).", + }) + }) + + test('keeps a context overflow code from response.failed', async () => { + const events = await collectSse(openaiResponsesStreamToAnthropic(makeStream([ + responseCreated, + responsesEvent('response.failed', { response: { status: 'failed', error: { code: 'context_length_exceeded', message: 'Your input exceeds the context window of this model.' } } }), + ]), 'gpt-4.1')) + + expect(lastError(events)).toEqual({ + type: 'invalid_request_error', + message: 'prompt is too long · Your input exceeds the context window of this model.', + }) + }) + + test('reports a stream that closes before response.completed', async () => { + const events = await collectSse(openaiResponsesStreamToAnthropic(makeStream([ + responseCreated, + responsesEvent('response.content_part.added', { output_index: 0, content_index: 0, part: { type: 'output_text', text: '' } }), + responsesEvent('response.output_text.delta', { output_index: 0, content_index: 0, delta: 'Half an ans' }), + ]), 'gpt-4.1')) + + expect(eventNames(events)).not.toContain('message_stop') + expect(lastError(events)).toEqual({ type: 'api_error', message: 'The provider closed the stream before the reply finished.' }) + }) + + test('leaves usage out when response.completed has none', async () => { + const events = await collectSse(openaiResponsesStreamToAnthropic(makeStream([responseCreated, completed()]), 'gpt-4.1')) + + expect(events.find((e) => e.event === 'message_delta')!.data).not.toHaveProperty('usage') + }) +}) + +describe('responses stream without every delta event', () => { + function blocksAndDeltas(events: Array<{ event: string; data: Record }>) { + return events + .filter((e) => e.event.startsWith('content_block')) + .map((e) => e.event === 'content_block_delta' ? e.data.delta : e.event === 'content_block_start' ? e.data.content_block : 'stop') + } + + test('opens a text block for a text delta without content_part.added and closes it on completion', async () => { + const events = await collectSse(openaiResponsesStreamToAnthropic(makeStream([ + responseCreated, + responsesEvent('response.output_text.delta', { output_index: 0, content_index: 0, delta: 'Hello' }), + completed({ input_tokens: 5, output_tokens: 1 }), + ]), 'gpt-4.1')) + + expect(blocksAndDeltas(events)).toEqual([{ type: 'text', text: '' }, { type: 'text_delta', text: 'Hello' }, 'stop']) + expect(eventNames(events).slice(-2)).toEqual(['message_delta', 'message_stop']) + }) + + test('takes the arguments from function_call_arguments.done when no deltas came', async () => { + const events = await collectSse(openaiResponsesStreamToAnthropic(makeStream([ + responseCreated, + responsesEvent('response.output_item.added', { output_index: 0, item: { type: 'function_call', id: 'fc_1', call_id: 'call_1', name: 'Read' } }), + responsesEvent('response.function_call_arguments.done', { item_id: 'fc_1', output_index: 0, arguments: '{"file_path":"a.ts"}' }), + completed({ input_tokens: 5, output_tokens: 1 }), + ]), 'gpt-4.1')) + + expect(blocksAndDeltas(events)).toEqual([ + { type: 'tool_use', id: 'call_1', name: 'Read', input: {} }, + { type: 'input_json_delta', partial_json: '{"file_path":"a.ts"}' }, + 'stop', + ]) + }) + + test('reads a message that only arrives as a completed output item', async () => { + const events = await collectSse(openaiResponsesStreamToAnthropic(makeStream([ + responseCreated, + responsesEvent('response.output_item.done', { + output_index: 0, + item: { type: 'message', id: 'msg_1', role: 'assistant', content: [{ type: 'output_text', text: 'Hello there' }] }, + }), + completed({ input_tokens: 5, output_tokens: 2 }), + ]), 'gpt-4.1')) + + expect(blocksAndDeltas(events)).toEqual([{ type: 'text', text: '' }, { type: 'text_delta', text: 'Hello there' }, 'stop']) + }) + + test('closes a tool block on completion when function_call_arguments.done never came', async () => { + const events = await collectSse(openaiResponsesStreamToAnthropic(makeStream([ + responseCreated, + responsesEvent('response.output_item.added', { output_index: 0, item: { type: 'function_call', id: 'fc_1', call_id: 'call_1', name: 'Read' } }), + responsesEvent('response.function_call_arguments.delta', { item_id: 'fc_1', output_index: 0, delta: '{"file_path":"a.ts"}' }), + completed({ input_tokens: 5, output_tokens: 1 }), + ]), 'gpt-4.1')) + + expect(blocksAndDeltas(events).at(-1)).toBe('stop') + expect((events.find((e) => e.event === 'message_delta')!.data.delta as { stop_reason: string }).stop_reason).toBe('tool_use') + }) +}) + +describe('stream error classification', () => { + test.each([ + ['insufficient_quota', 'You exceeded your current quota.', 'billing_error'], + ['rate_limit_exceeded', 'Rate limit reached.', 'rate_limit_error'], + ['invalid_api_key', 'Incorrect API key provided', 'authentication_error'], + ['server_error', 'The server had an error.', 'api_error'], + ])('maps a %s error in the middle of a chat stream', async (code, message, type) => { + const events = await collectSse(openaiChatStreamToAnthropic(makeStream([ + chatChunk({ delta: { content: 'Hi' }, finish_reason: null }), + `data: ${JSON.stringify({ error: { message, code } })}\n\n`, + ]), 'm')) + + expect(lastError(events)).toEqual({ type, message }) + }) +}) diff --git a/src/server/__tests__/proxy-transform.test.ts b/src/server/__tests__/proxy-transform.test.ts index 9ab5bfcf..cd6e3e11 100644 --- a/src/server/__tests__/proxy-transform.test.ts +++ b/src/server/__tests__/proxy-transform.test.ts @@ -261,6 +261,23 @@ describe('anthropicToOpenaiChat', () => { }) describe('openaiChatToAnthropic', () => { + test('reads usage that the provider puts inside the choice (Moonshot)', () => { + const res = { + id: 'chatcmpl-1', + object: 'chat.completion', + created: 0, + model: 'kimi-k2.6', + choices: [{ + index: 0, + message: { role: 'assistant', content: 'ok' }, + finish_reason: 'stop', + usage: { prompt_tokens: 1000, completion_tokens: 20, total_tokens: 1020 }, + }], + } as unknown as OpenAIChatResponse + + expect(openaiChatToAnthropic(res, 'kimi-k2.6').usage).toEqual({ input_tokens: 1000, output_tokens: 20, cache_read_input_tokens: 0 }) + }) + test('basic text response', () => { const res: OpenAIChatResponse = { id: 'chatcmpl-1', @@ -375,7 +392,9 @@ describe('openaiChatToAnthropic', () => { expect(openaiChatToAnthropic(make('stop'), 'gpt-4').stop_reason).toBe('end_turn') expect(openaiChatToAnthropic(make('length'), 'gpt-4').stop_reason).toBe('max_tokens') expect(openaiChatToAnthropic(make('tool_calls'), 'gpt-4').stop_reason).toBe('tool_use') - expect(openaiChatToAnthropic(make('content_filter'), 'gpt-4').stop_reason).toBe('end_turn') + expect(openaiChatToAnthropic(make('model_context_window_exceeded'), 'gpt-4').stop_reason).toBe('model_context_window_exceeded') + expect(() => openaiChatToAnthropic(make('content_filter'), 'gpt-4')) + .toThrow("The provider's content filter stopped the reply (finish_reason: content_filter).") }) test('empty choices', () => { @@ -667,6 +686,22 @@ describe('anthropicToOpenaiResponses', () => { }) describe('openaiResponsesToAnthropic', () => { + test.each([ + [{ reason: 'max_output_tokens' }, 'max_tokens'], + [undefined, 'max_tokens'], + ])('maps an incomplete reply with %o to %s', (details, stopReason) => { + const res = { id: 'r1', object: 'response', created_at: 0, model: 'gpt-4.1', status: 'incomplete', incomplete_details: details, output: [] } as OpenAIResponsesResponse + + expect(openaiResponsesToAnthropic(res, 'gpt-4.1').stop_reason).toBe(stopReason) + }) + + test('reports an incomplete reply stopped by the content filter as an error', () => { + const res = { id: 'r1', object: 'response', created_at: 0, model: 'gpt-4.1', status: 'incomplete', incomplete_details: { reason: 'content_filter' }, output: [] } as OpenAIResponsesResponse + + expect(() => openaiResponsesToAnthropic(res, 'gpt-4.1')) + .toThrow("The provider's content filter stopped the reply (incomplete: content_filter).") + }) + test('basic text response', () => { const res: OpenAIResponsesResponse = { id: 'resp_1', diff --git a/src/server/__tests__/stream-retraction.test.ts b/src/server/__tests__/stream-retraction.test.ts new file mode 100644 index 00000000..0f602cb0 --- /dev/null +++ b/src/server/__tests__/stream-retraction.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, test } from 'bun:test' +import { createStreamState, translateCliOutput } from '../session/translate.js' +import { createSessionState, reduceSessionState } from '../session/state/reducer.js' + +function streamEvent(event: Record) { + return { type: 'stream_event', event } +} + +describe('a stream the CLI retries', () => { + test('drops what the failed attempt streamed and tells the clients', () => { + const stream = createStreamState() + translateCliOutput(streamEvent({ type: 'message_start' }), stream) + translateCliOutput(streamEvent({ type: 'content_block_start', index: 0, content_block: { type: 'tool_use', id: 'toolu_1', name: 'Write' } }), stream) + translateCliOutput(streamEvent({ type: 'content_block_delta', index: 0, delta: { type: 'input_json_delta', partial_json: '{"file_pa' } }), stream) + + const retracted = translateCliOutput({ type: 'system', subtype: 'stream_retracted', uuid: 'u1' }, stream) + + expect(retracted.messages).toEqual([{ type: 'reply_retracted' }]) + expect(retracted.events).toEqual([{ type: 'turn_activity_reported', activity: 'thinking' }]) + expect(stream.activeBlockTypes.size).toBe(0) + expect(stream.activeToolBlocks.size).toBe(0) + expect(stream.streamedToolUseIds.size).toBe(0) + expect(stream.hasReceivedStreamEvents).toBe(false) + }) + + test('the turn goes back to thinking', () => { + const running = reduceSessionState(createSessionState('s1'), { type: 'turn_started', at: 1 }).state + const streaming = reduceSessionState(running, { type: 'turn_activity_reported', activity: 'streaming' }).state + + const event = translateCliOutput({ type: 'system', subtype: 'stream_retracted', uuid: 'u1' }, createStreamState()).events[0]! + + expect(reduceSessionState(streaming, event).state.turn.activity).toBe('thinking') + }) +}) diff --git a/src/server/__tests__/upstream-client.test.ts b/src/server/__tests__/upstream-client.test.ts index b8024e46..a62ac28e 100644 --- a/src/server/__tests__/upstream-client.test.ts +++ b/src/server/__tests__/upstream-client.test.ts @@ -194,12 +194,103 @@ describe('upstream client', () => { expect(text).toContain('message_stop') }) - test('reports upstream HTTP errors with status and body', async () => { + test('reports an upstream HTTP error with its status, an Anthropic error type and the plain body', async () => { mockFetch(() => new Response('model not found', { status: 404 })) const result = await sendToUpstream(target({ apiFormat: 'openai_chat' }), request) - expect(result).toEqual({ ok: false, httpStatus: 404, error: 'Upstream returned HTTP 404: model not found' }) + expect(result).toEqual({ ok: false, httpStatus: 404, errorType: 'not_found_error', error: 'model not found', headers: {} }) + }) + + test('reads the message out of a JSON error body and keeps the retry and request id headers', async () => { + mockFetch(() => Response.json( + { error: { message: 'Rate limit reached for gpt-4.1, please try again in 20s.', type: 'requests', code: 'rate_limit_exceeded' } }, + { status: 429, headers: { 'retry-after': '20', 'retry-after-ms': '20000', 'x-request-id': 'req_abc', 'set-cookie': 'a=b' } }, + )) + + const result = await sendToUpstream(target({ apiFormat: 'openai_chat' }), request) + + expect(result).toEqual({ + ok: false, + httpStatus: 429, + errorType: 'rate_limit_error', + error: 'Rate limit reached for gpt-4.1, please try again in 20s.', + headers: { 'retry-after': '20', 'retry-after-ms': '20000', 'x-request-id': 'req_abc', 'request-id': 'req_abc' }, + }) + }) + + test('keeps at most 500 characters of a long provider message', async () => { + mockFetch(() => Response.json({ error: { message: 'x'.repeat(5000) } }, { status: 500 })) + + const result = await sendToUpstream(target({ apiFormat: 'openai_chat' }), request) + + expect(result.ok === false && result.error).toBe('x'.repeat(500)) + }) + + test.each([ + ['OpenAI insufficient_quota', 429, { error: { message: 'You exceeded your current quota, please check your plan and billing details.', type: 'insufficient_quota', code: 'insufficient_quota' } }], + ['Moonshot exceeded_current_quota_error', 429, { error: { message: 'Your account has exceeded the current quota.', type: 'exceeded_current_quota_error' } }], + ['Zhipu 1113', 429, { error: { code: '1113', message: '余额不足或无可用资源包,请充值。' } }], + ['DeepSeek 402', 402, { error: { message: 'Insufficient Balance', type: 'unknown_error' } }], + ['OpenRouter 402', 402, { error: { message: 'Insufficient credits. Add more using https://openrouter.ai/settings/credits', code: 402 } }], + ])('reports %s as a billing error the CLI does not retry', async (_kind, status, body) => { + mockFetch(() => Response.json(body, { status })) + + const result = await sendToUpstream(target({ apiFormat: 'openai_chat' }), request) + + expect(result).toMatchObject({ ok: false, httpStatus: 402, errorType: 'billing_error', headers: { 'x-should-retry': 'false' } }) + }) + + test.each([ + [401, 'authentication_error'], + [403, 'permission_error'], + ] as const)('tells the CLI not to retry a %i from the provider', async (status, type) => { + mockFetch(() => Response.json({ error: { message: 'Incorrect API key provided', code: 'invalid_api_key' } }, { status })) + + const result = await sendToUpstream(target({ apiFormat: 'openai_chat' }), request) + + expect(result).toEqual({ + ok: false, + httpStatus: status, + errorType: type, + error: 'Incorrect API key provided', + headers: { 'x-should-retry': 'false' }, + }) + }) + + test.each([ + ['a quota error', { error: { message: 'You exceeded your current quota.', code: 'insufficient_quota' } }, 402, 'billing_error'], + ['an unknown error', { error: { message: 'something broke' } }, 502, 'api_error'], + ] as const)('maps %s inside a 2xx body by its content', async (_kind, body, status, type) => { + mockFetch(() => Response.json(body)) + + const result = await sendToUpstream(target({ apiFormat: 'openai_chat' }), request) + + expect(result).toMatchObject({ ok: false, httpStatus: status, errorType: type }) + }) + + test.each([ + [ + 'an OpenRouter message with token counts', + { error: { message: "This endpoint's maximum context length is 128000 tokens. However, you requested about 270016 tokens (270000 of text input, 16 in the output).", code: 400 } }, + "prompt is too long: 270016 tokens > 128000 maximum · This endpoint's maximum context length is 128000 tokens. However, you requested about 270016 tokens (270000 of text input, 16 in the output).", + ], + [ + 'an OpenAI error code', + { error: { message: 'Your input exceeds the context window of this model.', type: 'invalid_request_error', code: 'context_length_exceeded' } }, + 'prompt is too long · Your input exceeds the context window of this model.', + ], + [ + 'a Zhipu business code', + { error: { code: '1261', message: 'Prompt exceeds max length' } }, + 'prompt is too long · Prompt exceeds max length', + ], + ])('turns a context overflow reported as %s into the Anthropic prompt-too-long error', async (_kind, body, message) => { + mockFetch(() => Response.json(body, { status: 400 })) + + const result = await sendToUpstream(target({ apiFormat: 'openai_chat' }), request) + + expect(result).toEqual({ ok: false, httpStatus: 400, errorType: 'invalid_request_error', error: message, headers: {} }) }) test('rejects responses that do not match the declared format', async () => { @@ -214,6 +305,17 @@ describe('upstream client', () => { }) }) + test.each([ + ['content_filter', 400, 'invalid_request_error', "The provider's content filter stopped the reply (finish_reason: content_filter)."], + ['insufficient_system_resource', 502, 'api_error', 'The provider stopped the reply because of an error on its side (finish_reason: insufficient_system_resource).'], + ] as const)('reports a non-streaming reply that ended with %s as an error', async (reason, status, type, message) => { + mockFetch(() => Response.json({ ...openaiChatMessage, choices: [{ index: 0, message: { role: 'assistant', content: 'Hi' }, finish_reason: reason }] })) + + const result = await sendToUpstream(target({ apiFormat: 'openai_chat' }), request) + + expect(result).toEqual({ ok: false, httpStatus: status, errorType: type, error: message, headers: {} }) + }) + test('reports conversion failures as 502 instead of throwing', async () => { mockFetch(() => Response.json({ choices: [{}] })) @@ -431,7 +533,7 @@ describe('upstream client', () => { expect(upstreamEvents()).toHaveLength(1) expect(upstreamEvents()[0]!.severity).toBe('warn') - expect(upstreamEvents()[0]!.details).toMatchObject({ status: 429, outcome: 'error', error: 'Upstream returned HTTP 429: slow down' }) + expect(upstreamEvents()[0]!.details).toMatchObject({ status: 429, outcome: 'error', error: 'slow down' }) }) test('records an error reported in the middle of a stream when the stream ends', async () => { diff --git a/src/server/proxy/handler.ts b/src/server/proxy/handler.ts index 381c25b0..da4decc3 100644 --- a/src/server/proxy/handler.ts +++ b/src/server/proxy/handler.ts @@ -1,6 +1,7 @@ import { isClaudeModelName } from '../services/attributionHeaderPolicy.js' import { ProviderService } from '../services/providerService.js' import { sendToUpstream } from '../upstream/client.js' +import { anthropicErrorResponse } from '../upstream/upstreamError.js' import { ensureClaudeCodeAttribution } from './claudeCodeAttribution.js' import type { AnthropicRequest } from './transform/types.js' @@ -77,9 +78,9 @@ export async function handleProxyRequest(req: Request, url: URL, server: ProxySe }) if (!result.ok) { - return Response.json( - { type: 'error', error: { type: 'api_error', message: result.error } }, - { status: result.httpStatus ?? 502 }, + return anthropicErrorResponse( + { status: result.httpStatus ?? 502, type: result.errorType ?? 'api_error', message: result.error }, + result.headers, ) } diff --git a/src/server/proxy/streaming/openaiChatStreamToAnthropic.ts b/src/server/proxy/streaming/openaiChatStreamToAnthropic.ts index ed798203..e3fe4be3 100644 --- a/src/server/proxy/streaming/openaiChatStreamToAnthropic.ts +++ b/src/server/proxy/streaming/openaiChatStreamToAnthropic.ts @@ -1,6 +1,14 @@ -import type { OpenAIChatStreamChunk } from '../transform/types.js' +import type { OpenAIChatStreamChunk, OpenAIChatUsage } from '../transform/types.js' import { stringifyOpenAIToolArguments } from '../transform/toolArguments.js' -import { mapUsage } from '../transform/openaiChatToAnthropic.js' +import { chatUsage, mapChatFinishReason, mapUsage } from '../transform/openaiChatToAnthropic.js' +import { + anthropicErrorBody, + apiError, + readProviderError, + streamClosedEarlyError, + toAnthropicError, + type AnthropicError, +} from '../../upstream/upstreamError.js' import { parseSseJson, SseDecoder, type SseMessage } from './sse.js' type ContentBlockType = 'text' | 'thinking' | 'tool_use' @@ -35,7 +43,10 @@ type StreamState = { heldMessageDelta: SseEvent | null sawValidChunk: boolean - upstreamError: string | null + sawDone: boolean + sawUsage: boolean + stopReason: string | null + upstreamError: AnthropicError | null } function formatSse(event: string, data: unknown): string { @@ -58,6 +69,9 @@ function createState(model: string): StreamState { messageStopSent: false, heldMessageDelta: null, sawValidChunk: false, + sawDone: false, + sawUsage: false, + stopReason: null, upstreamError: null, } } @@ -119,6 +133,7 @@ function handleData( encoder: TextEncoder, ): void { if (data === '[DONE]') { + state.sawDone = true finalizeStream(state) flushQueue(state, controller, encoder) return @@ -136,6 +151,7 @@ function handleData( if (Array.isArray(chunk.choices)) state.sawValidChunk = true processChunk(chunk, state) + if (state.upstreamError) finalizeStream(state) flushQueue(state, controller, encoder) } @@ -259,70 +275,37 @@ function extractReasoning(delta: DeltaEx): { thinking: string; signature: string return null } -function detectBlockTransition( - delta: DeltaEx, - state: StreamState, -): { type: ContentBlockType; isNew: boolean } | null { - if (delta.tool_calls && delta.tool_calls.length > 0) { - const tc = delta.tool_calls[0] - const isNew = state.currentBlockType !== 'tool_use' || !!(tc.function?.name) - return { type: 'tool_use', isNew } - } - - if (delta.content != null && delta.content !== '') { - const isNew = state.currentBlockType !== 'text' || !state.blockStartSent - return { type: 'text', isNew } - } - - const reasoning = extractReasoning(delta) - if (reasoning) { - const isNew = state.currentBlockType !== 'thinking' || !state.blockStartSent - return { type: 'thinking', isNew } - } - - return null +function leaveBlockFor(type: ContentBlockType, state: StreamState): void { + if (!state.blockStartSent || state.blockStopSent || state.currentBlockType === type) return + if (state.currentBlockType === 'tool_use') closeAllToolBlocks(state) + else closeCurrentBlock(state) } function processChunk(chunk: OpenAIChatStreamChunk, state: StreamState): void { const choice = chunk.choices?.[0] - - if (!choice) { - if (chunk.usage && state.heldMessageDelta) { - mergeUsageIntoHeldDelta(state, chunk.usage) - } - return - } + const usage = chatUsage(chunk) + if (usage && state.heldMessageDelta) mergeUsageIntoHeldDelta(state, usage) + if (!choice) return ensureMessageStart(state, chunk.id) const delta = choice.delta as DeltaEx - const transition = detectBlockTransition(delta, state) - - if (transition) { - if (transition.isNew && state.blockStartSent && !state.blockStopSent) { - if (state.currentBlockType === 'tool_use' && transition.type !== 'tool_use') { - closeAllToolBlocks(state) - } else if (state.currentBlockType !== 'tool_use') { - closeCurrentBlock(state) - } - } - - switch (transition.type) { - case 'thinking': - handleThinking(delta, state) - break - case 'text': - handleText(delta, state) - break - case 'tool_use': - handleToolCalls(delta, state) - break - } + if (extractReasoning(delta)) { + leaveBlockFor('thinking', state) + handleThinking(delta, state) + } + if (delta.content != null && delta.content !== '') { + leaveBlockFor('text', state) + handleText(delta, state) + } + if (delta.tool_calls && delta.tool_calls.length > 0) { + leaveBlockFor('tool_use', state) + handleToolCalls(delta, state) } if (choice.finish_reason) { - handleFinishReason(choice.finish_reason, chunk, state) + handleFinishReason(choice.finish_reason, usage, state) } } @@ -423,29 +406,31 @@ function handleToolCalls(delta: DeltaEx, state: StreamState): void { function handleFinishReason( finishReason: string, - chunk: OpenAIChatStreamChunk, + usage: OpenAIChatUsage | undefined, state: StreamState, ): void { if (state.messageDeltaSent) return - closeAllOpenBlocks(state) + const finish = mapChatFinishReason(finishReason, state.sawToolUse) + if ('error' in finish) { + state.upstreamError = finish.error + return + } - let stopReason = mapFinishReason(finishReason) - // Gemini ends a tool-calling turn with finish_reason "stop"; clients that keep going only - // on stop_reason "tool_use" would otherwise not run the tools. - if (state.sawToolUse && stopReason === 'end_turn') stopReason = 'tool_use' - const usage = mapUsage(chunk.usage) + closeAllOpenBlocks(state) + state.stopReason = finish.stopReason const messageDelta: SseEvent = { event: 'message_delta', data: { type: 'message_delta', - delta: { stop_reason: stopReason, stop_sequence: null }, - usage, + delta: { stop_reason: finish.stopReason, stop_sequence: null }, + ...(usage && { usage: mapUsage(usage) }), }, } - if (chunk.usage) { + if (usage) { + state.sawUsage = true state.messageDeltaSent = true state.queue.push(messageDelta) } else { @@ -453,38 +438,42 @@ function handleFinishReason( } } -function mergeUsageIntoHeldDelta( - state: StreamState, - usage: NonNullable, -): void { +function mergeUsageIntoHeldDelta(state: StreamState, usage: OpenAIChatUsage): void { if (!state.heldMessageDelta) return const data = state.heldMessageDelta.data as Record data.usage = mapUsage(usage) + state.sawUsage = true state.messageDeltaSent = true state.queue.push(state.heldMessageDelta) state.heldMessageDelta = null } -function readUpstreamError(chunk: unknown): string | null { +function readUpstreamError(chunk: unknown): AnthropicError | null { const error = (chunk as { error?: unknown } | null)?.error if (!error || typeof error !== 'object') return null - const message = (error as { message?: unknown }).message - return typeof message === 'string' && message ? message : 'Upstream returned an error' + return toAnthropicError(undefined, readProviderError(chunk) ?? { message: 'Upstream returned an error' }) +} + +function endError(state: StreamState): AnthropicError | null { + if (state.upstreamError) return state.upstreamError + if (!state.sawValidChunk) return apiError('Upstream returned no valid Chat Completions chunks') + if (!state.sawDone && !state.messageDeltaSent && !state.heldMessageDelta) { + return streamClosedEarlyError() + } + if (state.nextContentIndex === 0 && !state.sawUsage && (state.stopReason ?? 'end_turn') === 'end_turn') { + return apiError('The provider returned an empty reply.') + } + return null } function finalizeStream(state: StreamState): void { if (state.messageStopSent) return state.messageStopSent = true - if (state.upstreamError || !state.sawValidChunk) { - enqueue(state, 'error', { - type: 'error', - error: { - type: 'api_error', - message: state.upstreamError ?? 'Upstream returned no valid Chat Completions chunks', - }, - }) + const error = endError(state) + if (error) { + enqueue(state, 'error', anthropicErrorBody(error)) return } @@ -502,20 +491,9 @@ function finalizeStream(state: StreamState): void { state.messageDeltaSent = true enqueue(state, 'message_delta', { type: 'message_delta', - delta: { stop_reason: 'end_turn', stop_sequence: null }, - usage: { output_tokens: 0 }, + delta: { stop_reason: state.sawToolUse ? 'tool_use' : 'end_turn', stop_sequence: null }, }) } enqueue(state, 'message_stop', { type: 'message_stop' }) } - -function mapFinishReason(reason: string): string { - switch (reason) { - case 'stop': return 'end_turn' - case 'tool_calls': return 'tool_use' - case 'length': return 'max_tokens' - case 'content_filter': return 'end_turn' - default: return 'end_turn' - } -} diff --git a/src/server/proxy/streaming/openaiResponsesStreamToAnthropic.ts b/src/server/proxy/streaming/openaiResponsesStreamToAnthropic.ts index 947a8852..9993bc5f 100644 --- a/src/server/proxy/streaming/openaiResponsesStreamToAnthropic.ts +++ b/src/server/proxy/streaming/openaiResponsesStreamToAnthropic.ts @@ -1,24 +1,26 @@ import { stringifyOpenAIToolArguments } from '../transform/toolArguments.js' -import { mapResponsesUsage } from '../transform/openaiResponsesToAnthropic.js' +import { mapResponsesStatus, mapResponsesUsage } from '../transform/openaiResponsesToAnthropic.js' import type { OpenAIResponsesUsage } from '../transform/types.js' +import { + anthropicErrorBody, + readProviderError, + streamClosedEarlyError, + toAnthropicError, + type AnthropicError, +} from '../../upstream/upstreamError.js' import { parseSseJson, SseDecoder, type SseMessage } from './sse.js' +type OpenBlock = { index: number; sent: boolean; closed: boolean } + type StreamState = { nextContentIndex: number - indexByKey: Map - toolIndexByItemId: Map - reasoningIndexByItemId: Map + texts: Map + tools: Map + reasoning: Map model: string messageStarted: boolean messageStopped: boolean -} - -export function upstreamErrorMessage(data: Record): string { - const response = data.response as Record | undefined - const error = (response?.error ?? data.error ?? data) as Record | undefined - return typeof error?.message === 'string' && error.message - ? error.message - : 'Upstream Responses stream failed' + emit: (event: string, data: unknown) => void } function formatSse(event: string, data: unknown): string { @@ -31,26 +33,25 @@ export function openaiResponsesStreamToAnthropic( ): ReadableStream { const encoder = new TextEncoder() const sse = new SseDecoder() - - const state: StreamState = { - nextContentIndex: 0, - indexByKey: new Map(), - toolIndexByItemId: new Map(), - reasoningIndexByItemId: new Map(), - model, - messageStarted: false, - messageStopped: false, - } - const reader = upstream.getReader() let cancelled = false return new ReadableStream({ async start(controller) { + const state: StreamState = { + nextContentIndex: 0, + texts: new Map(), + tools: new Map(), + reasoning: new Map(), + model, + messageStarted: false, + messageStopped: false, + emit: (event, data) => controller.enqueue(encoder.encode(formatSse(event, data))), + } const handle = (messages: SseMessage[]) => { for (const message of messages) { - if (message.type === 'event') handleEvent(message.event, message.data.trim(), state, controller, encoder) - else if (!state.messageStopped) controller.enqueue(encoder.encode(formatSse('ping', { type: 'ping' }))) + if (message.type === 'event') handleEvent(message.event, message.data.trim(), state) + else if (!state.messageStopped) state.emit('ping', { type: 'ping' }) } } @@ -62,8 +63,13 @@ export function openaiResponsesStreamToAnthropic( } handle(sse.flush()) - const body = sse.unframedBody() - if (body) emitUnframedError(body, controller, encoder) + if (!state.messageStopped) { + const unframed = sse.unframedBody() + const body = unframed ? parseSseJson(unframed) as Record | undefined : undefined + emitError(state, body?.error + ? errorFromEvent(body) + : streamClosedEarlyError()) + } } catch (err) { if (!cancelled) controller.error(err) return @@ -77,50 +83,34 @@ export function openaiResponsesStreamToAnthropic( }) } -function handleEvent( - event: string, - data: string, - state: StreamState, - controller: ReadableStreamDefaultController, - encoder: TextEncoder, -): void { +function handleEvent(event: string, data: string, state: StreamState): void { + if (state.messageStopped) return if (data === '[DONE]') { - if (!state.messageStopped) { - state.messageStopped = true - if (!state.messageStarted) { - emitMessageStart(state, controller, encoder) - } - controller.enqueue(encoder.encode(formatSse('message_stop', { type: 'message_stop' }))) - } + finishMessage(state, 'completed', undefined, undefined) return } const parsed = parseSseJson(data) - if (state.messageStopped || !parsed || typeof parsed !== 'object') return + if (!parsed || typeof parsed !== 'object') return const record = parsed as Record - processEvent(event || String(record.type ?? ''), record, state, controller, encoder) + processEvent(event || String(record.type ?? ''), record, state) } -function emitUnframedError( - body: string, - controller: ReadableStreamDefaultController, - encoder: TextEncoder, -): void { - const parsed = parseSseJson(body) as Record | undefined - if (!parsed || typeof parsed !== 'object' || !parsed.error) return - controller.enqueue(encoder.encode(formatSse('error', { - type: 'error', - error: { type: 'api_error', message: upstreamErrorMessage(parsed) }, - }))) +export function errorFromEvent(data: Record): AnthropicError { + const response = data.response as Record | undefined + const source = response?.error ? { error: response.error } : data.error || data.message ? data : undefined + return toAnthropicError(undefined, readProviderError(source) ?? { message: 'Upstream Responses stream failed' }) } -function emitMessageStart( - state: StreamState, - controller: ReadableStreamDefaultController, - encoder: TextEncoder, -): void { +function emitError(state: StreamState, error: AnthropicError): void { + state.messageStopped = true + state.emit('error', anthropicErrorBody(error)) +} + +function ensureMessageStart(state: StreamState): void { + if (state.messageStarted) return state.messageStarted = true - controller.enqueue(encoder.encode(formatSse('message_start', { + state.emit('message_start', { type: 'message_start', message: { id: `msg_${Date.now()}`, @@ -132,217 +122,199 @@ function emitMessageStart( stop_sequence: null, usage: { input_tokens: 0, output_tokens: 0 }, }, - }))) + }) +} + +function openBlock(state: StreamState, contentBlock: Record): OpenBlock { + ensureMessageStart(state) + const block = { index: state.nextContentIndex++, sent: false, closed: false } + state.emit('content_block_start', { type: 'content_block_start', index: block.index, content_block: contentBlock }) + return block +} + +function sendDelta(state: StreamState, block: OpenBlock, delta: Record): void { + block.sent = true + state.emit('content_block_delta', { type: 'content_block_delta', index: block.index, delta }) +} + +function closeBlock(state: StreamState, block: OpenBlock | undefined): void { + if (!block || block.closed) return + block.closed = true + state.emit('content_block_stop', { type: 'content_block_stop', index: block.index }) +} + +function textKey(data: Record, contentIndex = data.content_index): string { + return `${(data.output_index as number) ?? 0}:${(contentIndex as number) ?? 0}` +} + +function textBlock(state: StreamState, key: string): OpenBlock { + let block = state.texts.get(key) + if (!block) { + block = openBlock(state, { type: 'text', text: '' }) + state.texts.set(key, block) + } + return block +} + +function toolBlock(state: StreamState, item: Record): OpenBlock { + const itemId = (item.id as string) || (item.call_id as string) || '' + let block = state.tools.get(itemId) + if (!block) { + block = openBlock(state, { + type: 'tool_use', + id: (item.call_id as string) || itemId, + name: (item.name as string) || '', + input: {}, + }) + state.tools.set(itemId, block) + } + return block } -function processEvent( - event: string, - data: Record, +function completeText(state: StreamState, key: string, text: unknown): void { + const existing = state.texts.get(key) + if (!existing && !text) return + const block = existing ?? textBlock(state, key) + if (!block.sent && typeof text === 'string' && text) sendDelta(state, block, { type: 'text_delta', text }) + closeBlock(state, block) +} + +function completeTool(state: StreamState, block: OpenBlock, args: unknown): void { + const json = stringifyOpenAIToolArguments(args) + if (!block.sent && json) sendDelta(state, block, { type: 'input_json_delta', partial_json: json }) + closeBlock(state, block) +} + +function completeOutputItem(state: StreamState, data: Record): void { + const item = data.item as Record | undefined + if (item?.type === 'function_call') { + completeTool(state, toolBlock(state, item), item.arguments) + return + } + if (item?.type !== 'message' || !Array.isArray(item.content)) return + for (const [contentIndex, part] of (item.content as Array>).entries()) { + if (part.type === 'output_text' || part.type === 'refusal') { + completeText(state, textKey(data, contentIndex), part.text ?? part.refusal) + } + } +} + +function finishMessage( state: StreamState, - controller: ReadableStreamDefaultController, - encoder: TextEncoder, + status: string | undefined, + incompleteReason: string | undefined, + usage: OpenAIResponsesUsage | undefined, ): void { + const finish = mapResponsesStatus(status, incompleteReason, state.tools.size > 0) + if ('error' in finish) { + emitError(state, finish.error) + return + } + + ensureMessageStart(state) + const open = [...state.texts.values(), ...state.tools.values(), ...state.reasoning.values()] + .sort((a, b) => a.index - b.index) + for (const block of open) closeBlock(state, block) + + state.emit('message_delta', { + type: 'message_delta', + delta: { stop_reason: finish.stopReason, stop_sequence: null }, + ...(usage && { usage: mapResponsesUsage(usage) }), + }) + state.messageStopped = true + state.emit('message_stop', { type: 'message_stop' }) +} + +function processEvent(event: string, data: Record, state: StreamState): void { switch (event) { case 'response.created': { - emitMessageStart(state, controller, encoder) + ensureMessageStart(state) break } case 'response.output_item.added': { - if (!state.messageStarted) emitMessageStart(state, controller, encoder) + ensureMessageStart(state) const item = data.item as Record | undefined - if (!item) break - - if (item.type === 'function_call') { - const index = state.nextContentIndex++ - const callId = (item.call_id as string) || (item.id as string) || '' - const name = (item.name as string) || '' - state.toolIndexByItemId.set(item.id as string || callId, index) - - controller.enqueue(encoder.encode(formatSse('content_block_start', { - type: 'content_block_start', - index, - content_block: { - type: 'tool_use', - id: callId, - name, - input: {}, - }, - }))) - } + if (item?.type !== 'function_call') break + const block = toolBlock(state, item) + const json = stringifyOpenAIToolArguments(item.arguments) + if (json) sendDelta(state, block, { type: 'input_json_delta', partial_json: json }) break } case 'response.content_part.added': { - if (!state.messageStarted) emitMessageStart(state, controller, encoder) + ensureMessageStart(state) const part = data.part as Record | undefined - if (part?.type !== 'output_text' && part?.type !== 'refusal') break - - const contentIndex = (data.content_index as number) ?? 0 - const outputIndex = (data.output_index as number) ?? 0 - const key = `${outputIndex}:${contentIndex}` - const index = state.nextContentIndex++ - state.indexByKey.set(key, index) - - controller.enqueue(encoder.encode(formatSse('content_block_start', { - type: 'content_block_start', - index, - content_block: { type: 'text', text: '' }, - }))) - break - } - - case 'response.output_text.delta': { - const contentIndex = (data.content_index as number) ?? 0 - const outputIndex = (data.output_index as number) ?? 0 - const key = `${outputIndex}:${contentIndex}` - const index = state.indexByKey.get(key) - if (index === undefined) break - - const delta = (data.delta as string) || '' - controller.enqueue(encoder.encode(formatSse('content_block_delta', { - type: 'content_block_delta', - index, - delta: { type: 'text_delta', text: delta }, - }))) + if (part?.type === 'output_text' || part?.type === 'refusal') textBlock(state, textKey(data)) break } + case 'response.output_text.delta': case 'response.refusal.delta': { - const contentIndex = (data.content_index as number) ?? 0 - const outputIndex = (data.output_index as number) ?? 0 - const key = `${outputIndex}:${contentIndex}` - const index = state.indexByKey.get(key) - if (index === undefined) break - const delta = (data.delta as string) || '' - controller.enqueue(encoder.encode(formatSse('content_block_delta', { - type: 'content_block_delta', - index, - delta: { type: 'text_delta', text: delta }, - }))) + if (delta) sendDelta(state, textBlock(state, textKey(data)), { type: 'text_delta', text: delta }) break } case 'response.function_call_arguments.delta': { - const itemId = (data.item_id as string) || '' - const index = state.toolIndexByItemId.get(itemId) - if (index === undefined) break - + const block = state.tools.get((data.item_id as string) || '') const delta = stringifyOpenAIToolArguments(data.delta) - controller.enqueue(encoder.encode(formatSse('content_block_delta', { - type: 'content_block_delta', - index, - delta: { type: 'input_json_delta', partial_json: delta }, - }))) + if (block && delta) sendDelta(state, block, { type: 'input_json_delta', partial_json: delta }) break } case 'response.output_text.done': case 'response.refusal.done': { - const contentIndex = (data.content_index as number) ?? 0 - const outputIndex = (data.output_index as number) ?? 0 - const key = `${outputIndex}:${contentIndex}` - const index = state.indexByKey.get(key) - if (index === undefined) break - - controller.enqueue(encoder.encode(formatSse('content_block_stop', { - type: 'content_block_stop', - index, - }))) + completeText(state, textKey(data), data.text ?? data.refusal) break } case 'response.function_call_arguments.done': { - const itemId = (data.item_id as string) || '' - const index = state.toolIndexByItemId.get(itemId) - if (index === undefined) break + const block = state.tools.get((data.item_id as string) || '') + if (block) completeTool(state, block, data.arguments) + break + } - controller.enqueue(encoder.encode(formatSse('content_block_stop', { - type: 'content_block_stop', - index, - }))) + case 'response.output_item.done': { + completeOutputItem(state, data) break } case 'response.reasoning_summary_text.delta': case 'response.reasoning_text.delta': { const itemId = (data.item_id as string) || '' - let index = state.reasoningIndexByItemId.get(itemId) - if (index === undefined) { - index = state.nextContentIndex++ - state.reasoningIndexByItemId.set(itemId, index) - if (!state.messageStarted) emitMessageStart(state, controller, encoder) - controller.enqueue(encoder.encode(formatSse('content_block_start', { - type: 'content_block_start', - index, - content_block: { type: 'thinking', thinking: '' }, - }))) + let block = state.reasoning.get(itemId) + if (!block || block.closed) { + block = openBlock(state, { type: 'thinking', thinking: '' }) + state.reasoning.set(itemId, block) } - - const delta = (data.delta as string) || '' - controller.enqueue(encoder.encode(formatSse('content_block_delta', { - type: 'content_block_delta', - index, - delta: { type: 'thinking_delta', thinking: delta }, - }))) + sendDelta(state, block, { type: 'thinking_delta', thinking: (data.delta as string) || '' }) break } case 'response.reasoning_summary_text.done': case 'response.reasoning_text.done': case 'response.reasoning_summary_part.done': { - const itemId = (data.item_id as string) || '' - const index = state.reasoningIndexByItemId.get(itemId) - if (index === undefined) break - state.reasoningIndexByItemId.delete(itemId) - - controller.enqueue(encoder.encode(formatSse('content_block_stop', { - type: 'content_block_stop', - index, - }))) + closeBlock(state, state.reasoning.get((data.item_id as string) || '')) break } case 'response.failed': case 'error': { - const message = upstreamErrorMessage(data) - state.messageStopped = true - controller.enqueue(encoder.encode(formatSse('error', { - type: 'error', - error: { type: 'api_error', message }, - }))) + emitError(state, errorFromEvent(data)) break } case 'response.incomplete': case 'response.completed': { const response = data.response as Record | undefined - const status = (response?.status as string) || 'completed' - const usage = response?.usage as OpenAIResponsesUsage | undefined - const hasToolUse = state.toolIndexByItemId.size > 0 - - const stopReason = status === 'completed' - ? (hasToolUse ? 'tool_use' : 'end_turn') - : status === 'incomplete' ? 'max_tokens' : 'end_turn' - - for (const [, index] of state.reasoningIndexByItemId) { - controller.enqueue(encoder.encode(formatSse('content_block_stop', { - type: 'content_block_stop', - index, - }))) - } - state.reasoningIndexByItemId.clear() - - controller.enqueue(encoder.encode(formatSse('message_delta', { - type: 'message_delta', - delta: { stop_reason: stopReason, stop_sequence: null }, - usage: mapResponsesUsage(usage), - }))) - if (!state.messageStopped) { - state.messageStopped = true - controller.enqueue(encoder.encode(formatSse('message_stop', { type: 'message_stop' }))) - } + const details = response?.incomplete_details as { reason?: string } | undefined + finishMessage( + state, + (response?.status as string) || 'completed', + details?.reason, + response?.usage as OpenAIResponsesUsage | undefined, + ) break } } diff --git a/src/server/proxy/streaming/openaiResponsesStreamToAnthropicResponse.ts b/src/server/proxy/streaming/openaiResponsesStreamToAnthropicResponse.ts index bebba1ec..87df8d5c 100644 --- a/src/server/proxy/streaming/openaiResponsesStreamToAnthropicResponse.ts +++ b/src/server/proxy/streaming/openaiResponsesStreamToAnthropicResponse.ts @@ -1,5 +1,6 @@ import { openaiResponsesToAnthropic } from '../transform/openaiResponsesToAnthropic.js' -import { upstreamErrorMessage } from './openaiResponsesStreamToAnthropic.js' +import { UpstreamReplyError, type AnthropicError } from '../../upstream/upstreamError.js' +import { errorFromEvent } from './openaiResponsesStreamToAnthropic.js' import { parseSseJson, SseDecoder, type SseMessage } from './sse.js' import type { AnthropicResponse, @@ -14,8 +15,6 @@ type StreamFallbackState = { usage: OpenAIResponsesResponse['usage'] } -export class OpenAIResponsesStreamError extends Error {} - export async function openaiResponsesStreamToAnthropicResponse( upstream: ReadableStream, model: string, @@ -23,7 +22,7 @@ export async function openaiResponsesStreamToAnthropicResponse( const sse = new SseDecoder() const reader = upstream.getReader() let completedResponse: OpenAIResponsesResponse | null = null - let upstreamError: string | null = null + let upstreamError: AnthropicError | null = null const fallback: StreamFallbackState = { id: `resp_${Date.now()}`, @@ -43,7 +42,7 @@ export async function openaiResponsesStreamToAnthropicResponse( if (event === 'response.completed' || event === 'response.incomplete') { if (isOpenAIResponsesResponse(data.response)) completedResponse = data.response } else if (event === 'response.failed' || event === 'error') { - upstreamError = upstreamErrorMessage(data) + upstreamError = errorFromEvent(data) } else { updateFallbackState(event, data, fallback) } @@ -58,7 +57,7 @@ export async function openaiResponsesStreamToAnthropicResponse( handle(sse.flush()) if (!completedResponse && upstreamError !== null) { - throw new OpenAIResponsesStreamError(upstreamError) + throw new UpstreamReplyError(upstreamError) } return openaiResponsesToAnthropic( diff --git a/src/server/proxy/transform/openaiChatToAnthropic.ts b/src/server/proxy/transform/openaiChatToAnthropic.ts index 6335c90c..6a7e07f1 100644 --- a/src/server/proxy/transform/openaiChatToAnthropic.ts +++ b/src/server/proxy/transform/openaiChatToAnthropic.ts @@ -1,10 +1,19 @@ import type { OpenAIChatResponse, + OpenAIChatUsage, AnthropicResponse, AnthropicContentBlock, } from './types.js' +import { + contentFilterError, + providerSideError, + UpstreamReplyError, + type AnthropicError, +} from '../../upstream/upstreamError.js' import { parseOpenAIToolArguments } from './toolArguments.js' +export type ChatFinish = { stopReason: string } | { error: AnthropicError } + export function openaiChatToAnthropic(response: OpenAIChatResponse, model: string): AnthropicResponse { const choice = response.choices?.[0] if (!choice) { @@ -48,37 +57,53 @@ export function openaiChatToAnthropic(response: OpenAIChatResponse, model: strin content.push({ type: 'text', text: '' }) } + const finish = mapChatFinishReason(choice.finish_reason, content.some((block) => block.type === 'tool_use')) + if ('error' in finish) { + throw new UpstreamReplyError(finish.error) + } + return { id: response.id || `msg_${Date.now()}`, type: 'message', role: 'assistant', content, model, - stop_reason: mapStopReason(choice.finish_reason, content.some((block) => block.type === 'tool_use')), + stop_reason: finish.stopReason, stop_sequence: null, - usage: mapUsage(response.usage), + usage: mapUsage(chatUsage(response)), } } // Gemini ends a tool-calling turn with finish_reason "stop"; report tool_use whenever the // message carries tool_use blocks. -function mapStopReason(reason: string | null, hasToolUse: boolean): string { - const stopReason = mapFinishReason(reason) - return hasToolUse && stopReason === 'end_turn' ? 'tool_use' : stopReason -} - -function mapFinishReason(reason: string | null): string { +export function mapChatFinishReason(reason: string | null, hasToolUse: boolean): ChatFinish { switch (reason) { - case 'stop': return 'end_turn' - case 'tool_calls': return 'tool_use' - case 'length': return 'max_tokens' - case 'content_filter': return 'end_turn' - default: return 'end_turn' + case 'tool_calls': + case 'function_call': + return { stopReason: 'tool_use' } + case 'length': + return { stopReason: 'max_tokens' } + case 'model_context_window_exceeded': + return { stopReason: 'model_context_window_exceeded' } + case 'content_filter': + case 'sensitive': + return { error: contentFilterError(`finish_reason: ${reason}`) } + case 'network_error': + case 'insufficient_system_resource': + return { error: providerSideError(`finish_reason: ${reason}`) } + default: + return { stopReason: hasToolUse ? 'tool_use' : 'end_turn' } } } +export function chatUsage( + response: { usage?: OpenAIChatUsage; choices?: Array<{ usage?: OpenAIChatUsage }> }, +): OpenAIChatUsage | undefined { + return response.usage ?? response.choices?.[0]?.usage +} + // OpenAI counts cached tokens inside prompt_tokens; Anthropic's input_tokens excludes cache reads. -export function mapUsage(usage?: OpenAIChatResponse['usage']): AnthropicResponse['usage'] { +export function mapUsage(usage?: OpenAIChatUsage): AnthropicResponse['usage'] { if (!usage) { return { input_tokens: 0, output_tokens: 0 } } @@ -102,6 +127,6 @@ function createEmptyResponse(response: OpenAIChatResponse, model: string): Anthr model, stop_reason: 'end_turn', stop_sequence: null, - usage: mapUsage(response.usage), + usage: mapUsage(chatUsage(response)), } } diff --git a/src/server/proxy/transform/openaiResponsesToAnthropic.ts b/src/server/proxy/transform/openaiResponsesToAnthropic.ts index a8d1956f..138ffe74 100644 --- a/src/server/proxy/transform/openaiResponsesToAnthropic.ts +++ b/src/server/proxy/transform/openaiResponsesToAnthropic.ts @@ -5,6 +5,12 @@ import type { AnthropicResponse, AnthropicContentBlock, } from './types.js' +import { + apiError, + contentFilterError, + UpstreamReplyError, +} from '../../upstream/upstreamError.js' +import type { ChatFinish } from './openaiChatToAnthropic.js' import { parseOpenAIToolArguments } from './toolArguments.js' export function openaiResponsesToAnthropic(response: OpenAIResponsesResponse, model: string): AnthropicResponse { @@ -20,13 +26,18 @@ export function openaiResponsesToAnthropic(response: OpenAIResponsesResponse, mo content.push({ type: 'text', text: '' }) } + const finish = mapResponsesStatus(response.status, response.incomplete_details?.reason, hasToolUse) + if ('error' in finish) { + throw new UpstreamReplyError(finish.error) + } + return { id: response.id || `msg_${Date.now()}`, type: 'message', role: 'assistant', content, model, - stop_reason: mapStatus(response.status, hasToolUse), + stop_reason: finish.stopReason, stop_sequence: null, usage: mapResponsesUsage(response.usage), } @@ -80,12 +91,9 @@ function convertOutputItem(item: OpenAIResponsesOutputItem, content: AnthropicCo } } -function mapStatus(status: string, hasToolUse: boolean): string { - switch (status) { - case 'completed': return hasToolUse ? 'tool_use' : 'end_turn' - case 'failed': return 'end_turn' - case 'cancelled': return 'end_turn' - case 'incomplete': return 'max_tokens' - default: return 'end_turn' - } +export function mapResponsesStatus(status: string | undefined, incompleteReason: string | undefined, hasToolUse: boolean): ChatFinish { + if (status !== 'incomplete') return { stopReason: hasToolUse ? 'tool_use' : 'end_turn' } + if (!incompleteReason || incompleteReason === 'max_output_tokens') return { stopReason: 'max_tokens' } + if (incompleteReason === 'content_filter') return { error: contentFilterError(`incomplete: ${incompleteReason}`) } + return { error: apiError(`The provider ended the reply before it finished (incomplete: ${incompleteReason}).`) } } diff --git a/src/server/proxy/transform/toolArguments.ts b/src/server/proxy/transform/toolArguments.ts index 60c7ea8e..5b802eab 100644 --- a/src/server/proxy/transform/toolArguments.ts +++ b/src/server/proxy/transform/toolArguments.ts @@ -1,24 +1,17 @@ +import { parseSseJson } from '../streaming/sse.js' import type { AnthropicContentBlock } from './types.js' function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value) } -export function parseOpenAIToolArguments(value: unknown): Record { - if (value == null || value === '') return {} - - if (typeof value === 'string') { - try { - const parsed = JSON.parse(value) as unknown - return isRecord(parsed) ? parsed : { raw: parsed } - } catch { - return { raw: value } - } - } - +export function parseOpenAIToolArguments(value: unknown): Record | string { + if (value == null || (typeof value === 'string' && value.trim() === '')) return {} if (isRecord(value)) return value - - return { raw: value } + const raw = typeof value === 'string' ? value : JSON.stringify(value) + const parsed = parseSseJson(raw) + if (parsed === null) return {} + return isRecord(parsed) ? parsed : raw } export function stringifyOpenAIToolArguments(value: unknown): string { diff --git a/src/server/proxy/transform/types.ts b/src/server/proxy/transform/types.ts index 37558146..54fdc6c7 100644 --- a/src/server/proxy/transform/types.ts +++ b/src/server/proxy/transform/types.ts @@ -59,17 +59,20 @@ export type OpenAIChatResponse = { tool_calls?: OpenAIToolCall[] } finish_reason: string | null + usage?: OpenAIChatUsage }> - usage?: { - prompt_tokens: number - completion_tokens: number - total_tokens: number - prompt_tokens_details?: { - cached_tokens?: number - } - prompt_cache_hit_tokens?: number + usage?: OpenAIChatUsage +} + +export type OpenAIChatUsage = { + prompt_tokens: number + completion_tokens: number + total_tokens: number + prompt_tokens_details?: { cached_tokens?: number } + prompt_cache_hit_tokens?: number + cached_tokens?: number } export type OpenAIChatStreamChunk = { @@ -93,8 +96,9 @@ export type OpenAIChatStreamChunk = { }> } finish_reason: string | null + usage?: OpenAIChatUsage }> - usage?: OpenAIChatResponse['usage'] + usage?: OpenAIChatUsage } export type OpenAIResponsesContentPart = @@ -140,6 +144,7 @@ export type OpenAIResponsesResponse = { created_at: number model: string status: string + incomplete_details?: { reason?: string } | null output: OpenAIResponsesOutputItem[] usage?: OpenAIResponsesUsage } @@ -154,7 +159,7 @@ export type OpenAIResponsesUsage = { export type AnthropicContentBlock = | { type: 'text'; text: string; cache_control?: unknown } | { type: 'image'; source: { type: 'base64'; media_type: string; data: string }; cache_control?: unknown } - | { type: 'tool_use'; id: string; name: string; input: Record; cache_control?: unknown } + | { type: 'tool_use'; id: string; name: string; input: Record | string; cache_control?: unknown } | { type: 'tool_result'; tool_use_id: string; content: string | AnthropicContentBlock[]; is_error?: boolean; cache_control?: unknown } | { type: 'thinking'; thinking: string; signature?: string } diff --git a/src/server/services/networkSettings.ts b/src/server/services/networkSettings.ts index 4c354c81..ed9bd0a4 100644 --- a/src/server/services/networkSettings.ts +++ b/src/server/services/networkSettings.ts @@ -82,6 +82,7 @@ export function buildNetworkEnvironment( const timeout = String(settings.aiRequestTimeoutMs) return { API_TIMEOUT_MS: timeout, + CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK: '1', CLAUDE_ENABLE_STREAM_WATCHDOG: '1', CLAUDE_STREAM_IDLE_TIMEOUT_MS: timeout, ...buildProxyEnvironment(settings, inheritedEnv), diff --git a/src/server/services/providerService.ts b/src/server/services/providerService.ts index 37cc329d..f095b49f 100644 --- a/src/server/services/providerService.ts +++ b/src/server/services/providerService.ts @@ -18,6 +18,7 @@ import { } from './providerRuntimeEnv.js' import { loadNetworkSettings } from './networkSettings.js' import { fetchUpstreamModels, resolveUpstreamTarget, sendToUpstream, type UpstreamTarget } from '../upstream/client.js' +import { describeFailure } from '../upstream/upstreamError.js' import type { SavedProvider, ProvidersIndex, @@ -321,7 +322,7 @@ export class ProviderService { const result = await fetchUpstreamModels(target, presetId) if (!result.ok) { const upstreamRejected = result.httpStatus !== undefined && result.httpStatus >= 400 && result.httpStatus < 500 - throw new ApiError(upstreamRejected ? 400 : 502, result.error, 'MODEL_DISCOVERY_FAILED') + throw new ApiError(upstreamRejected ? 400 : 502, describeFailure(result), 'MODEL_DISCOVERY_FAILED') } return result.models } @@ -368,7 +369,7 @@ export class ProviderService { connectivity: { success: false, latencyMs: Date.now() - start, - error: result.error, + error: describeFailure(result), modelUsed: modelId, httpStatus: result.httpStatus, }, diff --git a/src/server/services/titleService.ts b/src/server/services/titleService.ts index 1c7cf91e..d3596e63 100644 --- a/src/server/services/titleService.ts +++ b/src/server/services/titleService.ts @@ -3,6 +3,7 @@ import { SettingsService } from './settingsService.js' import { sessionService } from './sessionService.js' import { cleanSessionTitleSource, hasSessionTitleMarkup } from '../../utils/sessionTitleText.js' import { resolveUpstreamTarget, sendToUpstream } from '../upstream/client.js' +import { describeFailure } from '../upstream/upstreamError.js' const TITLE_MAX_LEN = 50 const TITLE_MAX_OUTPUT_TOKENS = 100 @@ -89,7 +90,7 @@ export async function generateTitle( }, { timeoutMs: 15_000 }) if (!result.ok) { - return fail(excerptForLog(result.error)) + return fail(excerptForLog(describeFailure(result))) } if (!('message' in result)) return fail('upstream returned a stream') const block = result.message.content.find((b) => b.type === 'text') diff --git a/src/server/session/translate.ts b/src/server/session/translate.ts index 2ea19d54..97a7af7a 100644 --- a/src/server/session/translate.ts +++ b/src/server/session/translate.ts @@ -463,6 +463,15 @@ function translateMessages( case 'system': { const subtype = cliMsg.subtype + if (subtype === 'stream_retracted') { + streamState.hasReceivedStreamEvents = false + streamState.activeBlockTypes.clear() + streamState.activeToolBlocks.clear() + streamState.pendingToolBlocks.clear() + streamState.streamedToolUseIds.clear() + events.push({ type: 'turn_activity_reported', activity: 'thinking' }) + return [{ type: 'reply_retracted' }] + } if (subtype === 'api_retry') { const apiRetryMessage = toApiRetryServerMessage(cliMsg) return apiRetryMessage ? [apiRetryMessage] : [] diff --git a/src/server/upstream/anthropicStream.ts b/src/server/upstream/anthropicStream.ts index b1fc377a..16cd4364 100644 --- a/src/server/upstream/anthropicStream.ts +++ b/src/server/upstream/anthropicStream.ts @@ -1,5 +1,6 @@ import { parseSseJson, SseDecoder, type SseMessage } from '../proxy/streaming/sse.js' import type { UpstreamOutcome } from './requestLog.js' +import { anthropicErrorBody, apiError } from './upstreamError.js' export function superviseAnthropicStream( stream: ReadableStream, @@ -39,10 +40,7 @@ export function superviseAnthropicStream( timer = setTimeout(() => { const message = `No response from the provider for ${Math.round(idleTimeoutMs / 1000)}s` if (!finish({ outcome: 'idle_timeout', error: message })) return - controller.enqueue(encoder.encode(`event: error\ndata: ${JSON.stringify({ - type: 'error', - error: { type: 'api_error', message }, - })}\n\n`)) + controller.enqueue(encoder.encode(`event: error\ndata: ${JSON.stringify(anthropicErrorBody(apiError(message)))}\n\n`)) controller.close() reader.cancel(new DOMException('Stream idle timeout', 'TimeoutError')).catch((err: unknown) => { console.error('[Upstream] Cancelling the provider stream after an idle timeout failed:', err) diff --git a/src/server/upstream/client.ts b/src/server/upstream/client.ts index b138e4f6..e91178a6 100644 --- a/src/server/upstream/client.ts +++ b/src/server/upstream/client.ts @@ -21,6 +21,13 @@ import { } from '../services/providerRuntimeEnv.js' import type { ApiFormat, ProviderAuthStrategy, DiscoveredProviderModel, SavedProvider } from '../types/provider.js' import { normalizeUpstreamBaseUrl, openaiUpstreamBaseUrl } from './baseUrl.js' +import { + toFailure, + upstreamBodyFailure, + upstreamHttpFailure, + UpstreamReplyError, + type UpstreamFailure, +} from './upstreamError.js' import { fillContextWindows } from './contextWindows/index.js' import { superviseAnthropicStream } from './anthropicStream.js' import { @@ -43,11 +50,11 @@ export type UpstreamTarget = { export type UpstreamResult = | { ok: true; httpStatus: number; message: AnthropicResponse } | { ok: true; httpStatus: number; stream: ReadableStream } - | { ok: false; httpStatus?: number; error: string } + | UpstreamFailure export type UpstreamModelsResult = | { ok: true; httpStatus: number; models: DiscoveredProviderModel[] } - | { ok: false; httpStatus?: number; error: string } + | UpstreamFailure type UpstreamRequest = { url: string @@ -146,12 +153,7 @@ async function requestUpstream( responseInfo.headersMs = Date.now() - startedAt if (!response.ok) { - const errText = await response.text().catch(() => '') - return { - ok: false, - httpStatus: response.status, - error: `Upstream returned HTTP ${response.status}: ${errText.slice(0, 500)}`, - } + return upstreamHttpFailure(response.status, await response.text().catch(() => ''), response.headers) } if (isStream) { @@ -166,6 +168,7 @@ async function requestUpstream( } const body = await response.json().catch(() => null) as Record | null + if (body?.error && typeof body.error === 'object') return upstreamBodyFailure(body) const invalid = validateResponseBody(body, target.apiFormat) if (invalid) { return { ok: false, httpStatus: 502, error: invalid } @@ -177,6 +180,7 @@ async function requestUpstream( message: toAnthropicMessage(target.apiFormat, body!, anthropicRequest.model), } } catch (err) { + if (err instanceof UpstreamReplyError) return toFailure(err.error, {}) return { ok: false, httpStatus: 502, error: err instanceof Error ? err.message : String(err) } } } @@ -221,12 +225,7 @@ export async function fetchUpstreamModels(target: UpstreamTarget, presetId?: str } if (!response.ok) { - const errText = await response.text().catch(() => '') - return { - ok: false, - httpStatus: response.status, - error: `Upstream returned HTTP ${response.status}: ${errText.slice(0, 500)}`, - } + return upstreamHttpFailure(response.status, await response.text().catch(() => ''), response.headers) } const body = await response.json().catch(() => null) as Record | null @@ -390,9 +389,6 @@ function toAnthropicMessage( function validateResponseBody(body: Record | null, format: ApiFormat): string | null { if (!body) return 'Empty response — not a valid API endpoint' - if (body.error && typeof body.error === 'object') { - return ((body.error as Record).message as string) || 'Error in response body' - } if (format === 'openai_chat') { return Array.isArray(body.choices) && body.choices.length > 0 ? null diff --git a/src/server/upstream/upstreamError.ts b/src/server/upstream/upstreamError.ts new file mode 100644 index 00000000..33792c92 --- /dev/null +++ b/src/server/upstream/upstreamError.ts @@ -0,0 +1,150 @@ +import { classifyProviderError, toPromptTooLongMessage } from '../../utils/providerErrors.js' +import { parseSseJson } from '../proxy/streaming/sse.js' + +export type AnthropicErrorType = + | 'invalid_request_error' + | 'authentication_error' + | 'billing_error' + | 'permission_error' + | 'not_found_error' + | 'request_too_large' + | 'rate_limit_error' + | 'api_error' + | 'overloaded_error' + +export type UpstreamFailure = { + ok: false + httpStatus?: number + error: string + errorType?: AnthropicErrorType + headers?: Record +} + +type ProviderError = { message: string; code?: string } + +export type AnthropicError = { status: number; type: AnthropicErrorType; message: string; retryable?: false } + +export class UpstreamReplyError extends Error { + constructor(readonly error: AnthropicError) { + super(error.message) + } +} + +export function anthropicErrorBody(error: { type: AnthropicErrorType; message: string }) { + return { type: 'error', error: { type: error.type, message: error.message } } +} + +export function anthropicErrorResponse(error: AnthropicError, headers?: Record): Response { + return Response.json(anthropicErrorBody(error), { status: error.status, headers }) +} + +const MAX_PROVIDER_MESSAGE_CHARS = 500 + +export function apiError(message: string): AnthropicError { + return { status: 502, type: 'api_error', message } +} + +export function streamClosedEarlyError(): AnthropicError { + return apiError('The provider closed the stream before the reply finished.') +} + +export function contentFilterError(detail: string): AnthropicError { + return { status: 400, type: 'invalid_request_error', message: `The provider's content filter stopped the reply (${detail}).` } +} + +export function providerSideError(detail: string): AnthropicError { + return apiError(`The provider stopped the reply because of an error on its side (${detail}).`) +} + +const PASSED_HEADERS = ['retry-after', 'retry-after-ms', 'x-request-id'] + +export function readProviderError(body: unknown): ProviderError | undefined { + if (!body || typeof body !== 'object') return undefined + const record = body as Record + const nested = record.error + const source = nested && typeof nested === 'object' ? nested as Record : record + const message = [source.message, typeof nested === 'string' ? nested : undefined, record.message, record.msg] + .find((value): value is string => typeof value === 'string' && value.length > 0) + const code = [source.code, source.type, record.code].find( + (value): value is string | number => typeof value === 'string' || typeof value === 'number', + ) + if (!message && code === undefined) return undefined + return { + message: (message ?? String(code)).slice(0, MAX_PROVIDER_MESSAGE_CHARS), + ...(code !== undefined && { code: String(code) }), + } +} + +export function toAnthropicError(status: number | undefined, error: ProviderError): AnthropicError { + const { message } = error + switch (classifyProviderError({ status, ...error })) { + case 'context_overflow': + return { status: 400, type: 'invalid_request_error', message: toPromptTooLongMessage(message) } + case 'quota': + return { status: 402, type: 'billing_error', message, retryable: false } + case 'auth': + return status === 403 + ? { status, type: 'permission_error', message, retryable: false } + : { status: 401, type: 'authentication_error', message, retryable: false } + case 'rate_limit': + return { status: 429, type: 'rate_limit_error', message } + } + return status === undefined + ? apiError(message) + : { status, type: errorTypeForStatus(status), message } +} + +export function upstreamHttpFailure(status: number, text: string, headers: Headers): UpstreamFailure { + const error = readProviderError(parseSseJson(text)) ?? { message: text.trim().slice(0, MAX_PROVIDER_MESSAGE_CHARS) || `HTTP ${status}` } + return toFailure(toAnthropicError(status, error), passedHeaders(headers)) +} + +export function upstreamBodyFailure(body: unknown): UpstreamFailure { + const error = readProviderError(body) ?? { message: 'Error in response body' } + return toFailure(toAnthropicError(undefined, error), {}) +} + +export function describeFailure(failure: UpstreamFailure): string { + const status = failure.httpStatus ? `HTTP ${failure.httpStatus}` : '' + if (!status || failure.error === status) return failure.error || status + return `${status}: ${failure.error}` +} + +export function toFailure(error: AnthropicError, headers: Record): UpstreamFailure { + return { + ok: false, + httpStatus: error.status, + errorType: error.type, + error: error.message, + headers: error.retryable === false ? { ...headers, 'x-should-retry': 'false' } : headers, + } +} + +function errorTypeForStatus(status: number): AnthropicErrorType { + switch (status) { + case 400: + case 422: + return 'invalid_request_error' + case 404: + return 'not_found_error' + case 413: + return 'request_too_large' + case 429: + return 'rate_limit_error' + case 529: + return 'overloaded_error' + default: + return 'api_error' + } +} + +function passedHeaders(headers: Headers): Record { + const passed: Record = {} + for (const name of PASSED_HEADERS) { + const value = headers.get(name) + if (value) passed[name] = value + } + const requestId = passed['x-request-id'] ?? headers.get('request-id') + if (requestId) passed['request-id'] = requestId + return passed +} diff --git a/src/server/ws/protocol.ts b/src/server/ws/protocol.ts index b75755ee..7167395b 100644 --- a/src/server/ws/protocol.ts +++ b/src/server/ws/protocol.ts @@ -108,6 +108,7 @@ export type ServerMessage = | { type: 'tool_result'; toolUseId: string; content: unknown; isError: boolean; parentToolUseId?: string } | { type: 'permission_resolved'; requestId: string; allowed: boolean; by: ClientKind } | { type: 'message_complete'; usage: TokenUsage } + | { type: 'reply_retracted' } | { type: 'thinking'; text: string } | { type: 'api_retry' diff --git a/src/services/api/__tests__/contextOverflow.test.ts b/src/services/api/__tests__/contextOverflow.test.ts new file mode 100644 index 00000000..fa4047de --- /dev/null +++ b/src/services/api/__tests__/contextOverflow.test.ts @@ -0,0 +1,70 @@ +import { afterAll, beforeAll, describe, expect, test } from 'bun:test' +import { APIError } from '@anthropic-ai/sdk' +import { truncateHeadForPTLRetry } from '../../compact/compact.js' +import { createAssistantMessage, createUserMessage } from '../../../utils/messages.js' +import type { Message } from '../../../types/message.js' +import { classifyAPIError, getAssistantMessageFromError, isPromptTooLongMessage } from '../errors.js' + +const openRouterOverflow = + "This endpoint's maximum context length is 128000 tokens. However, you requested about 270016 tokens (270000 of text input, 16 in the output). Please reduce the length of the input, or use the context-compression plugin to compress your prompt automatically." + +function apiError(status: number | undefined, message: string): APIError { + const body = { type: 'error', error: { type: 'invalid_request_error', message } } + return new APIError(status, body, undefined, new Headers()) +} + +function conversation(): Message[] { + const turn = 'x'.repeat(80_000) + const messages: Message[] = [createUserMessage({ content: turn })] + for (let i = 0; i < 10; i += 1) { + messages.push(createAssistantMessage({ content: `step ${i}` })) + messages.push(createUserMessage({ content: turn })) + } + return messages +} + +describe('context overflow from third-party providers', () => { + let originalApiKey: string | undefined + + beforeAll(() => { + originalApiKey = process.env.ANTHROPIC_API_KEY + process.env.ANTHROPIC_API_KEY = 'sk-test' + }) + + afterAll(() => { + if (originalApiKey === undefined) delete process.env.ANTHROPIC_API_KEY + else process.env.ANTHROPIC_API_KEY = originalApiKey + }) + + test.each([ + ['the proxy', 400, 'prompt is too long: 270016 tokens > 128000 maximum · ' + openRouterOverflow], + ['a direct provider', 400, openRouterOverflow], + ['a direct provider stream', undefined, openRouterOverflow], + ])('an overflow reported by %s drops enough old rounds to fit on the compact retry', (_source, status, message) => { + const error = apiError(status, message) + + const reply = getAssistantMessageFromError(error, 'gpt-4o-mini') + const truncated = truncateHeadForPTLRetry(conversation(), reply) + + expect(isPromptTooLongMessage(reply)).toBe(true) + expect(classifyAPIError(error)).toBe('prompt_too_long') + expect(truncated).toHaveLength(7) + }) + + test.each([ + 'Your input exceeds the context window of this model. Please adjust your input and try again.', + 'Range of input length should be [1, 129024]', + 'The input token count (1200000) exceeds the maximum number of tokens allowed (1048576).', + 'Invalid request: Your request exceeded model token limit: 131072', + 'Invalid request: total message token length 140000 exceed model limit 131072', + 'Prompt exceeds max length', + ])('recognizes "%s" as a context overflow', (message) => { + expect(isPromptTooLongMessage(getAssistantMessageFromError(apiError(400, message), 'model'))).toBe(true) + }) + + test('does not treat an unrelated 400 as a context overflow', () => { + const reply = getAssistantMessageFromError(apiError(400, 'max_tokens must be at most 8192'), 'model') + + expect(isPromptTooLongMessage(reply)).toBe(false) + }) +}) diff --git a/src/services/api/__tests__/providerRetry.test.ts b/src/services/api/__tests__/providerRetry.test.ts new file mode 100644 index 00000000..c110538a --- /dev/null +++ b/src/services/api/__tests__/providerRetry.test.ts @@ -0,0 +1,95 @@ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test' +import Anthropic, { APIError } from '@anthropic-ai/sdk' +import { CannotRetryError, withRetry } from '../withRetry.js' + +const ENV_KEYS = ['ANTHROPIC_API_KEY', 'ANTHROPIC_BASE_URL'] as const +let savedEnv: Partial> + +function errorBody(message: string, type = 'api_error', code?: string) { + return { type: 'error', error: { type, message, ...(code && { code }) } } +} + +async function firstOutcome(error: APIError): Promise<{ retried: boolean; retryInMs?: number; attempts: number }> { + let attempts = 0 + const generator = withRetry( + async () => ({}) as Anthropic, + async () => { + attempts += 1 + throw error + }, + { model: 'model-main', thinkingConfig: { type: 'disabled' }, maxRetries: 3 }, + ) + try { + const step = await generator.next() + const message = step.value as { retryInMs?: number } + await generator.return(undefined as never) + return { retried: true, retryInMs: message.retryInMs, attempts } + } catch (err) { + expect(err).toBeInstanceOf(CannotRetryError) + return { retried: false, attempts } + } +} + +describe('retrying provider errors', () => { + beforeEach(() => { + savedEnv = Object.fromEntries(ENV_KEYS.map((key) => [key, process.env[key]])) + process.env.ANTHROPIC_API_KEY = 'sk-test' + delete process.env.ANTHROPIC_BASE_URL + }) + + afterEach(() => { + for (const key of ENV_KEYS) { + if (savedEnv[key] === undefined) delete process.env[key] + else process.env[key] = savedEnv[key] + } + }) + + test('does not retry a quota error the proxy reports as 402', async () => { + process.env.ANTHROPIC_BASE_URL = 'http://127.0.0.1:3456/proxy' + const error = APIError.generate( + 402, + errorBody('You exceeded your current quota, please check your plan and billing details.', 'billing_error'), + undefined, + new Headers({ 'x-should-retry': 'false' }), + ) + + expect(await firstOutcome(error)).toEqual({ retried: false, attempts: 1 }) + }) + + test('does not retry a quota error a direct third-party provider reports as 429', async () => { + process.env.ANTHROPIC_BASE_URL = 'https://api.moonshot.cn/anthropic' + const error = APIError.generate( + 429, + errorBody('Your account has exceeded the current quota.', 'exceeded_current_quota_error'), + undefined, + new Headers(), + ) + + expect(await firstOutcome(error)).toEqual({ retried: false, attempts: 1 }) + }) + + test('does not retry a 401 from a direct third-party provider', async () => { + process.env.ANTHROPIC_BASE_URL = 'https://open.bigmodel.cn/api/anthropic' + const error = APIError.generate(401, errorBody('invalid api key', 'authentication_error'), undefined, new Headers()) + + expect(await firstOutcome(error)).toEqual({ retried: false, attempts: 1 }) + }) + + test('keeps retrying a 401 on the Anthropic API', async () => { + const error = APIError.generate(401, errorBody('invalid x-api-key', 'authentication_error'), undefined, new Headers()) + + expect(await firstOutcome(error)).toMatchObject({ retried: true, attempts: 1 }) + }) + + test('waits as long as the retry-after header the proxy passes on', async () => { + process.env.ANTHROPIC_BASE_URL = 'http://127.0.0.1:3456/proxy' + const error = APIError.generate( + 429, + errorBody('Rate limit reached, please try again in 20s.', 'rate_limit_error'), + undefined, + new Headers({ 'retry-after': '20' }), + ) + + expect(await firstOutcome(error)).toEqual({ retried: true, retryInMs: 20_000, attempts: 1 }) + }) +}) diff --git a/src/services/api/__tests__/streamRetry.test.ts b/src/services/api/__tests__/streamRetry.test.ts new file mode 100644 index 00000000..7128a3ca --- /dev/null +++ b/src/services/api/__tests__/streamRetry.test.ts @@ -0,0 +1,218 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { getEmptyToolPermissionContext } from '../../../Tool.js' +import * as sessionActivity from '../../../utils/sessionActivity.js' +import { createUserMessage } from '../../../utils/messages.js' +import { asSystemPrompt } from '../../../utils/systemPromptType.js' +import { queryModelWithStreaming } from '../claude.js' + +const ENV_KEYS = [ + 'ANTHROPIC_API_KEY', + 'ANTHROPIC_BASE_URL', + 'CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK', + 'CLAUDE_CODE_MAX_RETRIES', + 'CLAUDE_CODE_TEST_FIXTURES_ROOT', + 'VCR_RECORD', +] as const +let savedEnv: Partial> +let fixturesDir: string +let starts: ReturnType +let stops: ReturnType +let server: ReturnType | undefined + +function sse(event: string, data: Record): string { + return `event: ${event}\ndata: ${JSON.stringify({ type: event, ...data })}\n\n` +} + +const messageStart = sse('message_start', { + message: { + id: 'msg_1', type: 'message', role: 'assistant', content: [], model: 'claude-sonnet-4-5', + stop_reason: null, stop_sequence: null, usage: { input_tokens: 10, output_tokens: 0 }, + }, +}) + +function textReply(text: string): string { + return messageStart + + sse('content_block_start', { index: 0, content_block: { type: 'text', text: '' } }) + + sse('content_block_delta', { index: 0, delta: { type: 'text_delta', text } }) + + sse('content_block_stop', { index: 0 }) + + sse('message_delta', { delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 5 } }) + + sse('message_stop', {}) +} + +function brokenReply(errorType: string): string { + return messageStart + + sse('content_block_start', { index: 0, content_block: { type: 'text', text: '' } }) + + sse('content_block_delta', { index: 0, delta: { type: 'text_delta', text: 'Half an ans' } }) + + sse('content_block_stop', { index: 0 }) + + `event: error\ndata: ${JSON.stringify({ type: 'error', error: { type: errorType, message: 'the stream broke' } })}\n\n` +} + +const overloaded = { status: 500, body: JSON.stringify({ type: 'error', error: { type: 'api_error', message: 'try again' } }) } + +type Reply = string | { status: number; body: string } | { openEnded: string } + +function reply(body: Reply): Response { + if (typeof body === 'string') return new Response(body, { headers: { 'content-type': 'text/event-stream' } }) + if ('openEnded' in body) { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(body.openEnded)) + }, + }) + return new Response(stream, { headers: { 'content-type': 'text/event-stream' } }) + } + return new Response(body.body, { status: body.status, headers: { 'content-type': 'application/json' } }) +} + +function serve(replies: Reply[]): { requests: () => number } { + let count = 0 + server = Bun.serve({ + port: 0, + fetch() { + const next = replies[Math.min(count, replies.length - 1)]! + count += 1 + return reply(next) + }, + }) + process.env.ANTHROPIC_BASE_URL = `http://127.0.0.1:${server.port}` + return { requests: () => count } +} + +function abortAfterFirstChunk(controller: AbortController): typeof fetch { + return (async (input: RequestInfo | URL, init?: RequestInit) => { + const response = await fetch(input, init) + const reader = response.body!.getReader() + let reads = 0 + const body = new ReadableStream({ + async pull(stream) { + reads += 1 + if (reads > 1) controller.abort() + try { + const { done, value } = await reader.read() + if (done) stream.close() + else stream.enqueue(value) + } catch (error) { + stream.error(error) + } + }, + }, { highWaterMark: 0 }) + return new Response(body, { status: response.status, headers: response.headers }) + }) as typeof fetch +} + +async function ask(controller = new AbortController(), fetchOverride?: typeof fetch) { + let fallbacks = 0 + const yielded = [] + for await (const message of queryModelWithStreaming({ + messages: [createUserMessage({ content: 'hello' })], + systemPrompt: asSystemPrompt(['You are a test.']), + thinkingConfig: { type: 'disabled' }, + tools: [], + signal: controller.signal, + options: { + getToolPermissionContext: async () => getEmptyToolPermissionContext(), + model: 'claude-sonnet-4-5', + isNonInteractiveSession: true, + querySource: 'sdk', + agents: [], + hasAppendSystemPrompt: false, + mcpTools: [], + onStreamingFallback: () => { + fallbacks += 1 + }, + ...(fetchOverride && { fetchOverride }), + }, + })) { + yielded.push(message) + } + const assistants = yielded.filter((m) => m.type === 'assistant') as Array<{ isApiErrorMessage?: boolean; message: { content: Array<{ type: string; text?: string }> } }> + const restarts = yielded.filter((m) => m.type === 'system' && (m as { streamRestart?: boolean }).streamRestart === true) + return { fallbacks, assistants, restarts } +} + +describe('retrying a stream that breaks in the middle', () => { + beforeEach(async () => { + savedEnv = Object.fromEntries(ENV_KEYS.map((key) => [key, process.env[key]])) + fixturesDir = await mkdtemp(join(tmpdir(), 'stream-retry-test-')) + process.env.ANTHROPIC_API_KEY = 'sk-test' + process.env.CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK = '1' + process.env.CLAUDE_CODE_TEST_FIXTURES_ROOT = fixturesDir + process.env.VCR_RECORD = '1' + delete process.env.CLAUDE_CODE_MAX_RETRIES + starts = spyOn(sessionActivity, 'startSessionActivity') + stops = spyOn(sessionActivity, 'stopSessionActivity') + }) + + afterEach(async () => { + starts.mockRestore() + stops.mockRestore() + server?.stop(true) + server = undefined + for (const key of ENV_KEYS) { + if (savedEnv[key] === undefined) delete process.env[key] + else process.env[key] = savedEnv[key] + } + await rm(fixturesDir, { recursive: true, force: true }) + }) + + test('opens a new stream and returns the complete reply', async () => { + const endpoint = serve([brokenReply('api_error'), textReply('The full answer.')]) + + const { fallbacks, assistants, restarts } = await ask() + + expect(endpoint.requests()).toBe(2) + expect(fallbacks).toBe(1) + expect(restarts).toHaveLength(1) + const last = assistants.at(-1)! + expect(last.isApiErrorMessage).toBeFalsy() + expect(last.message.content).toEqual([{ type: 'text', text: 'The full answer.' }]) + expect(starts.mock.calls.length).toBe(stops.mock.calls.length) + }) + + test('treats the user stopping the reply as a stop, not a broken stream to retry', async () => { + const partial = messageStart + + sse('content_block_start', { index: 0, content_block: { type: 'text', text: '' } }) + + sse('content_block_delta', { index: 0, delta: { type: 'text_delta', text: 'Half an ans' } }) + const controller = new AbortController() + const endpoint = serve([{ openEnded: partial }, textReply('never sent')]) + + const { fallbacks, restarts } = await ask(controller, abortAfterFirstChunk(controller)) + + expect(endpoint.requests()).toBe(1) + expect(fallbacks).toBe(0) + expect(restarts).toHaveLength(0) + }) + + test('does not retry an error the provider will repeat', async () => { + const endpoint = serve([brokenReply('invalid_request_error'), textReply('never sent')]) + + const { fallbacks, assistants } = await ask() + + expect(endpoint.requests()).toBe(1) + expect(fallbacks).toBe(0) + expect(assistants.at(-1)!.isApiErrorMessage).toBe(true) + }) + + test('counts stream retries and request retries against one budget', async () => { + process.env.CLAUDE_CODE_MAX_RETRIES = '2' + const endpoint = serve([brokenReply('api_error')]) + + const { assistants } = await ask() + + expect(endpoint.requests()).toBe(3) + expect(assistants.at(-1)!.isApiErrorMessage).toBe(true) + }) + + test('leaves request retries only what the broken streams did not use', async () => { + process.env.CLAUDE_CODE_MAX_RETRIES = '2' + const endpoint = serve([brokenReply('api_error'), brokenReply('api_error'), overloaded]) + + const { assistants } = await ask() + + expect(endpoint.requests()).toBe(3) + expect(assistants.at(-1)!.isApiErrorMessage).toBe(true) + }) +}) diff --git a/src/services/api/claude.ts b/src/services/api/claude.ts index 1b80331a..bd3cca8a 100644 --- a/src/services/api/claude.ts +++ b/src/services/api/claude.ts @@ -72,8 +72,10 @@ import { isEnvTruthy } from "../../utils/envUtils.js"; import { errorMessage } from "../../utils/errors.js"; import { computeFingerprintFromMessages } from "../../utils/fingerprint.js"; import { captureAPIRequest, logError } from "../../utils/log.js"; +import { sleep } from "../../utils/sleep.js"; import { createAssistantAPIErrorMessage, + createSystemAPIErrorMessage, createUserMessage, ensureToolResultPairing, normalizeContentFromAPI, @@ -108,6 +110,7 @@ const autoModeStateModule = feature("TRANSCRIPT_CLASSIFIER") import type { ClientOptions } from "@anthropic-ai/sdk"; import { + APIConnectionError, APIConnectionTimeoutError, APIError, APIUserAbortError, @@ -250,7 +253,11 @@ import { import { CannotRetryError, FallbackTriggeredError, + getDefaultMaxRetries, + getRetryAfter, + getRetryDelay, is529Error, + isRetryableStreamError, type RetryContext, withRetry, } from "./withRetry.js"; @@ -1823,711 +1830,780 @@ async function* queryModel( let isFastModeRequest = isFastMode; // Keep separate state as it may change if falling back let isAdvisorInProgress = false; + const maxStreamRequests = getDefaultMaxRetries() + 1; + let streamRequests = 0; + try { queryCheckpoint("query_client_creation_start"); - const generator = withRetry( - () => - getAnthropicClient({ - maxRetries: 0, // Disabled auto-retry in favor of manual implementation - model: options.model, - fetchOverride: options.fetchOverride, - source: options.querySource, - }), - async (anthropic, attempt, context) => { - attemptNumber = attempt; - isFastModeRequest = context.fastMode ?? false; - start = Date.now(); - attemptStartTimes.push(start); - // Client has been created by withRetry's getClient() call. This fires - // once per attempt; on retries the client is usually cached (withRetry - // only calls getClient() again after auth errors), so the delta from - // client_creation_start is meaningful on attempt 1. - queryCheckpoint("query_client_creation_end"); - - const params = paramsFromContext(context); - captureAPIRequest(params, options.querySource); // Capture for bug reports - - maxOutputTokens = params.max_tokens; - - // Fire immediately before the fetch is dispatched. .withResponse() below - // awaits until response headers arrive, so this MUST be before the await - // or the "Network TTFB" phase measurement is wrong. - queryCheckpoint("query_api_request_sent"); - if (!options.agentId) { - headlessProfilerCheckpoint("api_request_sent"); - } - - // Generate and track client request ID so timeouts (which return no - // server request ID) can still be correlated with server logs. - // First-party only — 3P providers don't log it (inc-4029 class). - clientRequestId = - getAPIProvider() === "firstParty" && isFirstPartyAnthropicBaseUrl() - ? randomUUID() - : undefined; - - // Use raw stream instead of BetaMessageStream to avoid O(n²) partial JSON parsing - // BetaMessageStream calls partialParse() on every input_json_delta, which we don't need - // since we handle tool input accumulation ourselves - // biome-ignore lint/plugin: main conversation loop handles attribution separately - const result = await anthropic.beta.messages - .create( - { ...params, stream: true }, - { - signal, - ...(clientRequestId && { - headers: { [CLIENT_REQUEST_ID_HEADER]: clientRequestId }, - }), - }, - ) - .withResponse(); - queryCheckpoint("query_response_headers_received"); - streamRequestId = result.request_id; - streamResponse = result.response; - return result.data; - }, - { - model: options.model, - fallbackModel: options.fallbackModel, - thinkingConfig, - ...(isFastModeEnabled() ? { fastMode: isFastMode } : false), - signal, - querySource: options.querySource, - }, - ); - - let e; - do { - e = await generator.next(); + startSessionActivity("api_call"); + streamAttempts: for (;;) { + const generator = withRetry( + () => + getAnthropicClient({ + maxRetries: 0, // Disabled auto-retry in favor of manual implementation + model: options.model, + fetchOverride: options.fetchOverride, + source: options.querySource, + }), + async (anthropic, attempt, context) => { + attemptNumber = attempt; + streamRequests += 1; + isFastModeRequest = context.fastMode ?? false; + start = Date.now(); + attemptStartTimes.push(start); + // Client has been created by withRetry's getClient() call. This fires + // once per attempt; on retries the client is usually cached (withRetry + // only calls getClient() again after auth errors), so the delta from + // client_creation_start is meaningful on attempt 1. + queryCheckpoint("query_client_creation_end"); + + const params = paramsFromContext(context); + captureAPIRequest(params, options.querySource); // Capture for bug reports + + maxOutputTokens = params.max_tokens; + + // Fire immediately before the fetch is dispatched. .withResponse() below + // awaits until response headers arrive, so this MUST be before the await + // or the "Network TTFB" phase measurement is wrong. + queryCheckpoint("query_api_request_sent"); + if (!options.agentId) { + headlessProfilerCheckpoint("api_request_sent"); + } - // yield API error messages (the stream has a 'controller' property, error messages don't) - if (!("controller" in e.value)) { - yield e.value; - } - } while (!e.done); - stream = e.value as Stream; - - // reset state - newMessages.length = 0; - ttftMs = 0; - partialMessage = undefined; - contentBlocks.length = 0; - usage = EMPTY_USAGE; - stopReason = null; - isAdvisorInProgress = false; - - // Streaming idle timeout watchdog: abort the stream if no chunks arrive - // for STREAM_IDLE_TIMEOUT_MS. Unlike the stall detection below (which only - // fires when the *next* chunk arrives), this uses setTimeout to actively - // kill hung streams. Without this, a silently dropped connection can hang - // the session indefinitely since the SDK's request timeout only covers the - // initial fetch(), not the streaming body. - const streamWatchdogEnabled = isEnvTruthy( - process.env.CLAUDE_ENABLE_STREAM_WATCHDOG, - ); - const STREAM_IDLE_TIMEOUT_MS = - parseInt(process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS || "", 10) || 90_000; - const STREAM_IDLE_WARNING_MS = STREAM_IDLE_TIMEOUT_MS / 2; - let streamIdleAborted = false; - // performance.now() snapshot when watchdog fires, for measuring abort propagation delay - let streamWatchdogFiredAt: number | null = null; - let streamIdleWarningTimer: ReturnType | null = null; - let streamIdleTimer: ReturnType | null = null; - function clearStreamIdleTimers(): void { - if (streamIdleWarningTimer !== null) { - clearTimeout(streamIdleWarningTimer); - streamIdleWarningTimer = null; - } - if (streamIdleTimer !== null) { - clearTimeout(streamIdleTimer); - streamIdleTimer = null; - } - } - function resetStreamIdleTimer(): void { - clearStreamIdleTimers(); - if (!streamWatchdogEnabled) { - return; - } - streamIdleWarningTimer = setTimeout( - (warnMs) => { - logForDebugging( - `Streaming idle warning: no chunks received for ${warnMs / 1000}s`, - { level: "warn" }, - ); - logForDiagnosticsNoPII("warn", "cli_streaming_idle_warning"); + // Generate and track client request ID so timeouts (which return no + // server request ID) can still be correlated with server logs. + // First-party only — 3P providers don't log it (inc-4029 class). + clientRequestId = + getAPIProvider() === "firstParty" && isFirstPartyAnthropicBaseUrl() + ? randomUUID() + : undefined; + + // Use raw stream instead of BetaMessageStream to avoid O(n²) partial JSON parsing + // BetaMessageStream calls partialParse() on every input_json_delta, which we don't need + // since we handle tool input accumulation ourselves + // biome-ignore lint/plugin: main conversation loop handles attribution separately + const result = await anthropic.beta.messages + .create( + { ...params, stream: true }, + { + signal, + ...(clientRequestId && { + headers: { [CLIENT_REQUEST_ID_HEADER]: clientRequestId }, + }), + }, + ) + .withResponse(); + queryCheckpoint("query_response_headers_received"); + streamRequestId = result.request_id; + streamResponse = result.response; + return result.data; + }, + { + model: options.model, + fallbackModel: options.fallbackModel, + thinkingConfig, + ...(isFastModeEnabled() ? { fastMode: isFastMode } : false), + signal, + querySource: options.querySource, + maxRetries: maxStreamRequests - streamRequests - 1, }, - STREAM_IDLE_WARNING_MS, - STREAM_IDLE_WARNING_MS, ); - streamIdleTimer = setTimeout(() => { - streamIdleAborted = true; - streamWatchdogFiredAt = performance.now(); - logForDebugging( - `Streaming idle timeout: no chunks received for ${STREAM_IDLE_TIMEOUT_MS / 1000}s, aborting stream`, - { level: "error" }, - ); - logForDiagnosticsNoPII("error", "cli_streaming_idle_timeout"); - logEvent("tengu_streaming_idle_timeout", { - model: - options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - request_id: (streamRequestId ?? - "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - timeout_ms: STREAM_IDLE_TIMEOUT_MS, - }); - releaseStreamResources(); - }, STREAM_IDLE_TIMEOUT_MS); - } - resetStreamIdleTimer(); - startSessionActivity("api_call"); - try { - // stream in and accumulate state - let isFirstChunk = true; - let lastEventTime: number | null = null; // Set after first chunk to avoid measuring TTFB as a stall - const STALL_THRESHOLD_MS = 30_000; // 30 seconds - let totalStallTime = 0; - let stallCount = 0; - - for await (const part of stream) { - resetStreamIdleTimer(); - const now = Date.now(); - - // Detect and log streaming stalls (only after first event to avoid counting TTFB) - if (lastEventTime !== null) { - const timeSinceLastEvent = now - lastEventTime; - if (timeSinceLastEvent > STALL_THRESHOLD_MS) { - stallCount++; - totalStallTime += timeSinceLastEvent; + let e; + do { + e = await generator.next(); + + // yield API error messages (the stream has a 'controller' property, error messages don't) + if (!("controller" in e.value)) { + yield e.value; + } + } while (!e.done); + stream = e.value as Stream; + + // reset state + newMessages.length = 0; + ttftMs = 0; + partialMessage = undefined; + contentBlocks.length = 0; + usage = EMPTY_USAGE; + stopReason = null; + isAdvisorInProgress = false; + + // Streaming idle timeout watchdog: abort the stream if no chunks arrive + // for STREAM_IDLE_TIMEOUT_MS. Unlike the stall detection below (which only + // fires when the *next* chunk arrives), this uses setTimeout to actively + // kill hung streams. Without this, a silently dropped connection can hang + // the session indefinitely since the SDK's request timeout only covers the + // initial fetch(), not the streaming body. + const streamWatchdogEnabled = isEnvTruthy( + process.env.CLAUDE_ENABLE_STREAM_WATCHDOG, + ); + const STREAM_IDLE_TIMEOUT_MS = + parseInt(process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS || "", 10) || 90_000; + const STREAM_IDLE_WARNING_MS = STREAM_IDLE_TIMEOUT_MS / 2; + let streamIdleAborted = false; + // performance.now() snapshot when watchdog fires, for measuring abort propagation delay + let streamWatchdogFiredAt: number | null = null; + let streamIdleWarningTimer: ReturnType | null = null; + let streamIdleTimer: ReturnType | null = null; + function clearStreamIdleTimers(): void { + if (streamIdleWarningTimer !== null) { + clearTimeout(streamIdleWarningTimer); + streamIdleWarningTimer = null; + } + if (streamIdleTimer !== null) { + clearTimeout(streamIdleTimer); + streamIdleTimer = null; + } + } + function resetStreamIdleTimer(): void { + clearStreamIdleTimers(); + if (!streamWatchdogEnabled) { + return; + } + streamIdleWarningTimer = setTimeout( + (warnMs) => { logForDebugging( - `Streaming stall detected: ${(timeSinceLastEvent / 1000).toFixed(1)}s gap between events (stall #${stallCount})`, + `Streaming idle warning: no chunks received for ${warnMs / 1000}s`, { level: "warn" }, ); - logEvent("tengu_streaming_stall", { - stall_duration_ms: timeSinceLastEvent, - stall_count: stallCount, - total_stall_time_ms: totalStallTime, - event_type: - part.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - model: - options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - request_id: (streamRequestId ?? - "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - }); - } - } - lastEventTime = now; + logForDiagnosticsNoPII("warn", "cli_streaming_idle_warning"); + }, + STREAM_IDLE_WARNING_MS, + STREAM_IDLE_WARNING_MS, + ); + streamIdleTimer = setTimeout(() => { + streamIdleAborted = true; + streamWatchdogFiredAt = performance.now(); + logForDebugging( + `Streaming idle timeout: no chunks received for ${STREAM_IDLE_TIMEOUT_MS / 1000}s, aborting stream`, + { level: "error" }, + ); + logForDiagnosticsNoPII("error", "cli_streaming_idle_timeout"); + logEvent("tengu_streaming_idle_timeout", { + model: + options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + request_id: (streamRequestId ?? + "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + timeout_ms: STREAM_IDLE_TIMEOUT_MS, + }); + releaseStreamResources(); + }, STREAM_IDLE_TIMEOUT_MS); + } + resetStreamIdleTimer(); - if (isFirstChunk) { - logForDebugging("Stream started - received first chunk"); - queryCheckpoint("query_first_chunk_received"); - if (!options.agentId) { - headlessProfilerCheckpoint("first_chunk"); + try { + // stream in and accumulate state + let isFirstChunk = true; + let lastEventTime: number | null = null; // Set after first chunk to avoid measuring TTFB as a stall + const STALL_THRESHOLD_MS = 30_000; // 30 seconds + let totalStallTime = 0; + let stallCount = 0; + + for await (const part of stream) { + resetStreamIdleTimer(); + const now = Date.now(); + + // Detect and log streaming stalls (only after first event to avoid counting TTFB) + if (lastEventTime !== null) { + const timeSinceLastEvent = now - lastEventTime; + if (timeSinceLastEvent > STALL_THRESHOLD_MS) { + stallCount++; + totalStallTime += timeSinceLastEvent; + logForDebugging( + `Streaming stall detected: ${(timeSinceLastEvent / 1000).toFixed(1)}s gap between events (stall #${stallCount})`, + { level: "warn" }, + ); + logEvent("tengu_streaming_stall", { + stall_duration_ms: timeSinceLastEvent, + stall_count: stallCount, + total_stall_time_ms: totalStallTime, + event_type: + part.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + model: + options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + request_id: (streamRequestId ?? + "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + }); + } } - endQueryProfile(); - isFirstChunk = false; - } + lastEventTime = now; - switch (part.type) { - case "message_start": { - partialMessage = part.message; - ttftMs = Date.now() - start; - usage = updateUsage(usage, part.message?.usage); - // Capture research from message_start if available (internal only). - // Always overwrite with the latest value. - if ( - process.env.USER_TYPE === "ant" && - "research" in (part.message as unknown as Record) - ) { - research = (part.message as unknown as Record) - .research; + if (isFirstChunk) { + logForDebugging("Stream started - received first chunk"); + queryCheckpoint("query_first_chunk_received"); + if (!options.agentId) { + headlessProfilerCheckpoint("first_chunk"); } - break; + endQueryProfile(); + isFirstChunk = false; } - case "content_block_start": - switch (part.content_block.type) { - case "tool_use": - contentBlocks[part.index] = { - ...part.content_block, - input: "", - }; - break; - case "server_tool_use": - contentBlocks[part.index] = { - ...part.content_block, - input: "" as unknown as { [key: string]: unknown }, - }; - if ((part.content_block.name as string) === "advisor") { - isAdvisorInProgress = true; - logForDebugging(`[AdvisorTool] Advisor tool called`); - logEvent("tengu_advisor_tool_call", { - model: - options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - advisor_model: (advisorModel ?? - "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - }); - } - break; - case "text": - contentBlocks[part.index] = { - ...part.content_block, - // awkwardly, the sdk sometimes returns text as part of a - // content_block_start message, then returns the same text - // again in a content_block_delta message. we ignore it here - // since there doesn't seem to be a way to detect when a - // content_block_delta message duplicates the text. - text: "", - }; - break; - case "thinking": - contentBlocks[part.index] = { - ...part.content_block, - // also awkward - thinking: "", - // initialize signature to ensure field exists even if signature_delta never arrives - signature: "", - }; - break; - default: - // even more awkwardly, the sdk mutates the contents of text blocks - // as it works. we want the blocks to be immutable, so that we can - // accumulate state ourselves. - contentBlocks[part.index] = { ...part.content_block }; - if ( - (part.content_block.type as string) === "advisor_tool_result" - ) { - isAdvisorInProgress = false; - logForDebugging(`[AdvisorTool] Advisor tool result received`); - } - break; - } - break; - case "content_block_delta": { - const contentBlock = contentBlocks[part.index]; - const delta = part.delta as typeof part.delta | ConnectorTextDelta; - if (!contentBlock) { - logEvent("tengu_streaming_error", { - error_type: - "content_block_not_found_delta" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - part_type: - part.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - part_index: part.index, - }); - throw new RangeError("Content block not found"); - } - if ( - feature("CONNECTOR_TEXT") && - delta.type === "connector_text_delta" - ) { - if (contentBlock.type !== "connector_text") { - logEvent("tengu_streaming_error", { - error_type: - "content_block_type_mismatch_connector_text" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - expected_type: - "connector_text" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - actual_type: - contentBlock.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - }); - throw new Error("Content block is not a connector_text block"); + + switch (part.type) { + case "message_start": { + partialMessage = part.message; + ttftMs = Date.now() - start; + usage = updateUsage(usage, part.message?.usage); + // Capture research from message_start if available (internal only). + // Always overwrite with the latest value. + if ( + process.env.USER_TYPE === "ant" && + "research" in (part.message as unknown as Record) + ) { + research = (part.message as unknown as Record) + .research; } - contentBlock.connector_text += delta.connector_text; - } else { - switch (delta.type) { - case "citations_delta": - // TODO: handle citations + break; + } + case "content_block_start": + switch (part.content_block.type) { + case "tool_use": + contentBlocks[part.index] = { + ...part.content_block, + input: "", + }; break; - case "input_json_delta": - if ( - contentBlock.type !== "tool_use" && - contentBlock.type !== "server_tool_use" - ) { - logEvent("tengu_streaming_error", { - error_type: - "content_block_type_mismatch_input_json" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - expected_type: - "tool_use" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - actual_type: - contentBlock.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + case "server_tool_use": + contentBlocks[part.index] = { + ...part.content_block, + input: "" as unknown as { [key: string]: unknown }, + }; + if ((part.content_block.name as string) === "advisor") { + isAdvisorInProgress = true; + logForDebugging(`[AdvisorTool] Advisor tool called`); + logEvent("tengu_advisor_tool_call", { + model: + options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + advisor_model: (advisorModel ?? + "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, }); - throw new Error("Content block is not a input_json block"); } - if (typeof contentBlock.input !== "string") { - logEvent("tengu_streaming_error", { - error_type: - "content_block_input_not_string" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - input_type: - typeof contentBlock.input as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - }); - throw new Error("Content block input is not a string"); - } - contentBlock.input += delta.partial_json; break; - case "text_delta": - if (contentBlock.type !== "text") { - logEvent("tengu_streaming_error", { - error_type: - "content_block_type_mismatch_text" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - expected_type: - "text" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - actual_type: - contentBlock.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - }); - throw new Error("Content block is not a text block"); - } - contentBlock.text += delta.text; + case "text": + contentBlocks[part.index] = { + ...part.content_block, + // awkwardly, the sdk sometimes returns text as part of a + // content_block_start message, then returns the same text + // again in a content_block_delta message. we ignore it here + // since there doesn't seem to be a way to detect when a + // content_block_delta message duplicates the text. + text: "", + }; break; - case "signature_delta": + case "thinking": + contentBlocks[part.index] = { + ...part.content_block, + // also awkward + thinking: "", + // initialize signature to ensure field exists even if signature_delta never arrives + signature: "", + }; + break; + default: + // even more awkwardly, the sdk mutates the contents of text blocks + // as it works. we want the blocks to be immutable, so that we can + // accumulate state ourselves. + contentBlocks[part.index] = { ...part.content_block }; if ( - feature("CONNECTOR_TEXT") && - contentBlock.type === "connector_text" + (part.content_block.type as string) === "advisor_tool_result" ) { - contentBlock.signature = delta.signature; - break; - } - if (contentBlock.type !== "thinking") { - logEvent("tengu_streaming_error", { - error_type: - "content_block_type_mismatch_thinking_signature" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - expected_type: - "thinking" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - actual_type: - contentBlock.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - }); - throw new Error("Content block is not a thinking block"); - } - contentBlock.signature = delta.signature; - break; - case "thinking_delta": - if (contentBlock.type !== "thinking") { - logEvent("tengu_streaming_error", { - error_type: - "content_block_type_mismatch_thinking_delta" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - expected_type: - "thinking" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - actual_type: - contentBlock.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - }); - throw new Error("Content block is not a thinking block"); + isAdvisorInProgress = false; + logForDebugging(`[AdvisorTool] Advisor tool result received`); } - contentBlock.thinking += delta.thinking; break; } - } - // Capture research from content_block_delta if available (internal only). - // Always overwrite with the latest value. - if (process.env.USER_TYPE === "ant" && "research" in part) { - research = (part as { research: unknown }).research; - } - break; - } - case "content_block_stop": { - const contentBlock = contentBlocks[part.index]; - if (!contentBlock) { - logEvent("tengu_streaming_error", { - error_type: - "content_block_not_found_stop" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - part_type: - part.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - part_index: part.index, - }); - throw new RangeError("Content block not found"); - } - if (!partialMessage) { - logEvent("tengu_streaming_error", { - error_type: - "partial_message_not_found" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - part_type: - part.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - }); - throw new Error("Message not found"); - } - const m: AssistantMessage = { - message: { - ...partialMessage, - content: normalizeContentFromAPI( - [contentBlock] as BetaContentBlock[], - tools, - options.agentId, - ), - }, - requestId: streamRequestId ?? undefined, - type: "assistant", - uuid: randomUUID(), - timestamp: new Date().toISOString(), - ...(process.env.USER_TYPE === "ant" && - research !== undefined && { research }), - ...(advisorModel && { advisorModel }), - }; - newMessages.push(m); - yield m; - break; - } - case "message_delta": { - usage = updateUsage(usage, part.usage); - // Capture research from message_delta if available (internal only). - // Always overwrite with the latest value. Also write back to - // already-yielded messages since message_delta arrives after - // content_block_stop. - if ( - process.env.USER_TYPE === "ant" && - "research" in (part as unknown as Record) - ) { - research = (part as unknown as Record).research; - for (const msg of newMessages) { - msg.research = research; + break; + case "content_block_delta": { + const contentBlock = contentBlocks[part.index]; + const delta = part.delta as typeof part.delta | ConnectorTextDelta; + if (!contentBlock) { + logEvent("tengu_streaming_error", { + error_type: + "content_block_not_found_delta" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + part_type: + part.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + part_index: part.index, + }); + throw new RangeError("Content block not found"); + } + if ( + feature("CONNECTOR_TEXT") && + delta.type === "connector_text_delta" + ) { + if (contentBlock.type !== "connector_text") { + logEvent("tengu_streaming_error", { + error_type: + "content_block_type_mismatch_connector_text" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + expected_type: + "connector_text" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + actual_type: + contentBlock.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + }); + throw new Error("Content block is not a connector_text block"); + } + contentBlock.connector_text += delta.connector_text; + } else { + switch (delta.type) { + case "citations_delta": + // TODO: handle citations + break; + case "input_json_delta": + if ( + contentBlock.type !== "tool_use" && + contentBlock.type !== "server_tool_use" + ) { + logEvent("tengu_streaming_error", { + error_type: + "content_block_type_mismatch_input_json" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + expected_type: + "tool_use" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + actual_type: + contentBlock.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + }); + throw new Error("Content block is not a input_json block"); + } + if (typeof contentBlock.input !== "string") { + logEvent("tengu_streaming_error", { + error_type: + "content_block_input_not_string" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + input_type: + typeof contentBlock.input as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + }); + throw new Error("Content block input is not a string"); + } + contentBlock.input += delta.partial_json; + break; + case "text_delta": + if (contentBlock.type !== "text") { + logEvent("tengu_streaming_error", { + error_type: + "content_block_type_mismatch_text" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + expected_type: + "text" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + actual_type: + contentBlock.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + }); + throw new Error("Content block is not a text block"); + } + contentBlock.text += delta.text; + break; + case "signature_delta": + if ( + feature("CONNECTOR_TEXT") && + contentBlock.type === "connector_text" + ) { + contentBlock.signature = delta.signature; + break; + } + if (contentBlock.type !== "thinking") { + logEvent("tengu_streaming_error", { + error_type: + "content_block_type_mismatch_thinking_signature" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + expected_type: + "thinking" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + actual_type: + contentBlock.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + }); + throw new Error("Content block is not a thinking block"); + } + contentBlock.signature = delta.signature; + break; + case "thinking_delta": + if (contentBlock.type !== "thinking") { + logEvent("tengu_streaming_error", { + error_type: + "content_block_type_mismatch_thinking_delta" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + expected_type: + "thinking" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + actual_type: + contentBlock.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + }); + throw new Error("Content block is not a thinking block"); + } + contentBlock.thinking += delta.thinking; + break; + } + } + // Capture research from content_block_delta if available (internal only). + // Always overwrite with the latest value. + if (process.env.USER_TYPE === "ant" && "research" in part) { + research = (part as { research: unknown }).research; } + break; } - - // Write final usage and stop_reason back to the last yielded - // message. Messages are created at content_block_stop from - // partialMessage, which was set at message_start before any tokens - // were generated (output_tokens: 0, stop_reason: null). - // message_delta arrives after content_block_stop with the real - // values. - // - // IMPORTANT: Use direct property mutation, not object replacement. - // The transcript write queue holds a reference to message.message - // and serializes it lazily (100ms flush interval). Object - // replacement ({ ...lastMsg.message, usage }) would disconnect - // the queued reference; direct mutation ensures the transcript - // captures the final values. - stopReason = part.delta.stop_reason; - - const lastMsg = newMessages.at(-1); - if (lastMsg) { - lastMsg.message.usage = usage; - lastMsg.message.stop_reason = stopReason; + case "content_block_stop": { + const contentBlock = contentBlocks[part.index]; + if (!contentBlock) { + logEvent("tengu_streaming_error", { + error_type: + "content_block_not_found_stop" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + part_type: + part.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + part_index: part.index, + }); + throw new RangeError("Content block not found"); + } + if (!partialMessage) { + logEvent("tengu_streaming_error", { + error_type: + "partial_message_not_found" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + part_type: + part.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + }); + throw new Error("Message not found"); + } + const m: AssistantMessage = { + message: { + ...partialMessage, + content: normalizeContentFromAPI( + [contentBlock] as BetaContentBlock[], + tools, + options.agentId, + ), + }, + requestId: streamRequestId ?? undefined, + type: "assistant", + uuid: randomUUID(), + timestamp: new Date().toISOString(), + ...(process.env.USER_TYPE === "ant" && + research !== undefined && { research }), + ...(advisorModel && { advisorModel }), + }; + newMessages.push(m); + yield m; + break; } + case "message_delta": { + usage = updateUsage(usage, part.usage); + // Capture research from message_delta if available (internal only). + // Always overwrite with the latest value. Also write back to + // already-yielded messages since message_delta arrives after + // content_block_stop. + if ( + process.env.USER_TYPE === "ant" && + "research" in (part as unknown as Record) + ) { + research = (part as unknown as Record).research; + for (const msg of newMessages) { + msg.research = research; + } + } - // Update cost - const costUSDForPart = calculateUSDCost(resolvedModel, usage); - costUSD += addToTotalSessionCost( - costUSDForPart, - usage, - options.model, - ); + // Write final usage and stop_reason back to the last yielded + // message. Messages are created at content_block_stop from + // partialMessage, which was set at message_start before any tokens + // were generated (output_tokens: 0, stop_reason: null). + // message_delta arrives after content_block_stop with the real + // values. + // + // IMPORTANT: Use direct property mutation, not object replacement. + // The transcript write queue holds a reference to message.message + // and serializes it lazily (100ms flush interval). Object + // replacement ({ ...lastMsg.message, usage }) would disconnect + // the queued reference; direct mutation ensures the transcript + // captures the final values. + stopReason = part.delta.stop_reason; + + const lastMsg = newMessages.at(-1); + if (lastMsg) { + lastMsg.message.usage = usage; + lastMsg.message.stop_reason = stopReason; + } - const refusalMessage = getErrorMessageIfRefusal( - part.delta.stop_reason, - options.model, - ); - if (refusalMessage) { - yield refusalMessage; - } + // Update cost + const costUSDForPart = calculateUSDCost(resolvedModel, usage); + costUSD += addToTotalSessionCost( + costUSDForPart, + usage, + options.model, + ); + + const refusalMessage = getErrorMessageIfRefusal( + part.delta.stop_reason, + options.model, + ); + if (refusalMessage) { + yield refusalMessage; + } - if (stopReason === "max_tokens") { - logEvent("tengu_max_tokens_reached", { - max_tokens: maxOutputTokens, - }); - yield createAssistantAPIErrorMessage({ - content: `${API_ERROR_MESSAGE_PREFIX}: Claude's response exceeded the ${ - maxOutputTokens - } output token maximum. To configure this behavior, set the CLAUDE_CODE_MAX_OUTPUT_TOKENS environment variable.`, - apiError: "max_output_tokens", - error: "max_output_tokens", - }); - } + if (stopReason === "max_tokens") { + logEvent("tengu_max_tokens_reached", { + max_tokens: maxOutputTokens, + }); + yield createAssistantAPIErrorMessage({ + content: `${API_ERROR_MESSAGE_PREFIX}: Claude's response exceeded the ${ + maxOutputTokens + } output token maximum. To configure this behavior, set the CLAUDE_CODE_MAX_OUTPUT_TOKENS environment variable.`, + apiError: "max_output_tokens", + error: "max_output_tokens", + }); + } - if (stopReason === "model_context_window_exceeded") { - logEvent("tengu_context_window_exceeded", { - max_tokens: maxOutputTokens, - output_tokens: usage.output_tokens, - }); - // Reuse the max_output_tokens recovery path — from the model's - // perspective, both mean "response was cut off, continue from - // where you left off." - yield createAssistantAPIErrorMessage({ - content: `${API_ERROR_MESSAGE_PREFIX}: The model has reached its context window limit.`, - apiError: "max_output_tokens", - error: "max_output_tokens", - }); + if (stopReason === "model_context_window_exceeded") { + logEvent("tengu_context_window_exceeded", { + max_tokens: maxOutputTokens, + output_tokens: usage.output_tokens, + }); + // Reuse the max_output_tokens recovery path — from the model's + // perspective, both mean "response was cut off, continue from + // where you left off." + yield createAssistantAPIErrorMessage({ + content: `${API_ERROR_MESSAGE_PREFIX}: The model has reached its context window limit.`, + apiError: "max_output_tokens", + error: "max_output_tokens", + }); + } + break; } - break; + case "message_stop": + break; } - case "message_stop": - break; + + yield { + type: "stream_event", + event: part, + ...(part.type === "message_start" ? { ttftMs } : undefined), + }; + } + // Clear the idle timeout watchdog now that the stream loop has exited + clearStreamIdleTimers(); + + // If the stream was aborted by our idle timeout watchdog, fall back to + // non-streaming retry rather than treating it as a completed stream. + if (streamIdleAborted) { + // Instrumentation: proves the for-await exited after the watchdog fired + // (vs. hung forever). exit_delay_ms measures abort propagation latency: + // 0-10ms = abort worked; >>1000ms = something else woke the loop. + const exitDelayMs = + streamWatchdogFiredAt !== null + ? Math.round(performance.now() - streamWatchdogFiredAt) + : -1; + logForDiagnosticsNoPII( + "info", + "cli_stream_loop_exited_after_watchdog_clean", + ); + logEvent("tengu_stream_loop_exited_after_watchdog", { + request_id: (streamRequestId ?? + "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + exit_delay_ms: exitDelayMs, + exit_path: + "clean" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + model: + options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + }); + // Prevent double-emit: this throw lands in the catch block below, + // whose exit_path='error' probe guards on streamWatchdogFiredAt. + streamWatchdogFiredAt = null; + throw new Error("Stream idle timeout - no chunks received"); } - yield { - type: "stream_event", - event: part, - ...(part.type === "message_start" ? { ttftMs } : undefined), - }; - } - // Clear the idle timeout watchdog now that the stream loop has exited - clearStreamIdleTimers(); - - // If the stream was aborted by our idle timeout watchdog, fall back to - // non-streaming retry rather than treating it as a completed stream. - if (streamIdleAborted) { - // Instrumentation: proves the for-await exited after the watchdog fired - // (vs. hung forever). exit_delay_ms measures abort propagation latency: - // 0-10ms = abort worked; >>1000ms = something else woke the loop. - const exitDelayMs = - streamWatchdogFiredAt !== null - ? Math.round(performance.now() - streamWatchdogFiredAt) - : -1; - logForDiagnosticsNoPII( - "info", - "cli_stream_loop_exited_after_watchdog_clean", - ); - logEvent("tengu_stream_loop_exited_after_watchdog", { - request_id: (streamRequestId ?? - "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - exit_delay_ms: exitDelayMs, - exit_path: - "clean" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - model: - options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - }); - // Prevent double-emit: this throw lands in the catch block below, - // whose exit_path='error' probe guards on streamWatchdogFiredAt. - streamWatchdogFiredAt = null; - throw new Error("Stream idle timeout - no chunks received"); - } + // Detect when the stream completed without producing any assistant messages. + // This covers two proxy failure modes: + // 1. No events at all (!partialMessage): proxy returned 200 with non-SSE body + // 2. Partial events (partialMessage set but no content blocks completed AND + // no stop_reason received): proxy returned message_start but stream ended + // before content_block_stop and before message_delta with stop_reason + // BetaMessageStream had the first check in _endRequest() but the raw Stream + // does not - without it the generator silently returns no assistant messages, + // causing "Execution error" in -p mode. + // Note: We must check stopReason to avoid false positives. For example, with + // structured output (--json-schema), the model calls a StructuredOutput tool + // on turn 1, then on turn 2 responds with end_turn and no content blocks. + // That's a legitimate empty response, not an incomplete stream. + if (!partialMessage || (newMessages.length === 0 && !stopReason)) { + logForDebugging( + !partialMessage + ? "Stream completed without receiving message_start event - triggering non-streaming fallback" + : "Stream completed with message_start but no content blocks completed - triggering non-streaming fallback", + { level: "error" }, + ); + logEvent("tengu_stream_no_events", { + model: + options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + request_id: (streamRequestId ?? + "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + }); + throw new Error("Stream ended without receiving any events"); + } - // Detect when the stream completed without producing any assistant messages. - // This covers two proxy failure modes: - // 1. No events at all (!partialMessage): proxy returned 200 with non-SSE body - // 2. Partial events (partialMessage set but no content blocks completed AND - // no stop_reason received): proxy returned message_start but stream ended - // before content_block_stop and before message_delta with stop_reason - // BetaMessageStream had the first check in _endRequest() but the raw Stream - // does not - without it the generator silently returns no assistant messages, - // causing "Execution error" in -p mode. - // Note: We must check stopReason to avoid false positives. For example, with - // structured output (--json-schema), the model calls a StructuredOutput tool - // on turn 1, then on turn 2 responds with end_turn and no content blocks. - // That's a legitimate empty response, not an incomplete stream. - if (!partialMessage || (newMessages.length === 0 && !stopReason)) { - logForDebugging( - !partialMessage - ? "Stream completed without receiving message_start event - triggering non-streaming fallback" - : "Stream completed with message_start but no content blocks completed - triggering non-streaming fallback", - { level: "error" }, - ); - logEvent("tengu_stream_no_events", { - model: - options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - request_id: (streamRequestId ?? - "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - }); - throw new Error("Stream ended without receiving any events"); - } + // Log summary if any stalls occurred during streaming + if (stallCount > 0) { + logForDebugging( + `Streaming completed with ${stallCount} stall(s), total stall time: ${(totalStallTime / 1000).toFixed(1)}s`, + { level: "warn" }, + ); + logEvent("tengu_streaming_stall_summary", { + stall_count: stallCount, + total_stall_time_ms: totalStallTime, + model: + options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + request_id: (streamRequestId ?? + "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + }); + } - // Log summary if any stalls occurred during streaming - if (stallCount > 0) { - logForDebugging( - `Streaming completed with ${stallCount} stall(s), total stall time: ${(totalStallTime / 1000).toFixed(1)}s`, - { level: "warn" }, - ); - logEvent("tengu_streaming_stall_summary", { - stall_count: stallCount, - total_stall_time_ms: totalStallTime, - model: - options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - request_id: (streamRequestId ?? - "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - }); - } + // Check if the cache actually broke based on response tokens + if (feature("PROMPT_CACHE_BREAK_DETECTION")) { + void checkResponseForCacheBreak( + options.querySource, + usage.cache_read_input_tokens, + usage.cache_creation_input_tokens, + messages, + options.agentId, + streamRequestId, + ); + } - // Check if the cache actually broke based on response tokens - if (feature("PROMPT_CACHE_BREAK_DETECTION")) { - void checkResponseForCacheBreak( - options.querySource, - usage.cache_read_input_tokens, - usage.cache_creation_input_tokens, - messages, - options.agentId, - streamRequestId, - ); - } + // Process fallback percentage header and quota status if available + // streamResponse is set when the stream is created in the withRetry callback above + // TypeScript's control flow analysis can't track that streamResponse is set in the callback + // eslint-disable-next-line eslint-plugin-n/no-unsupported-features/node-builtins + const resp = streamResponse as unknown as Response | undefined; + if (resp) { + extractQuotaStatusFromHeaders(resp.headers); + // Store headers for gateway detection + responseHeaders = resp.headers; + } + } catch (streamingError) { + // Clear the idle timeout watchdog on error path too + clearStreamIdleTimers(); + + // Instrumentation: if the watchdog had already fired and the for-await + // threw (rather than exiting cleanly), record that the loop DID exit and + // how long after the watchdog. Distinguishes true hangs from error exits. + if (streamIdleAborted && streamWatchdogFiredAt !== null) { + const exitDelayMs = Math.round( + performance.now() - streamWatchdogFiredAt, + ); + logForDiagnosticsNoPII( + "info", + "cli_stream_loop_exited_after_watchdog_error", + ); + logEvent("tengu_stream_loop_exited_after_watchdog", { + request_id: (streamRequestId ?? + "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + exit_delay_ms: exitDelayMs, + exit_path: + "error" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + error_name: + streamingError instanceof Error + ? (streamingError.name as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS) + : ("unknown" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS), + model: + options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + }); + } - // Process fallback percentage header and quota status if available - // streamResponse is set when the stream is created in the withRetry callback above - // TypeScript's control flow analysis can't track that streamResponse is set in the callback - // eslint-disable-next-line eslint-plugin-n/no-unsupported-features/node-builtins - const resp = streamResponse as unknown as Response | undefined; - if (resp) { - extractQuotaStatusFromHeaders(resp.headers); - // Store headers for gateway detection - responseHeaders = resp.headers; - } - } catch (streamingError) { - // Clear the idle timeout watchdog on error path too - clearStreamIdleTimers(); - - // Instrumentation: if the watchdog had already fired and the for-await - // threw (rather than exiting cleanly), record that the loop DID exit and - // how long after the watchdog. Distinguishes true hangs from error exits. - if (streamIdleAborted && streamWatchdogFiredAt !== null) { - const exitDelayMs = Math.round( - performance.now() - streamWatchdogFiredAt, - ); - logForDiagnosticsNoPII( - "info", - "cli_stream_loop_exited_after_watchdog_error", - ); - logEvent("tengu_stream_loop_exited_after_watchdog", { - request_id: (streamRequestId ?? - "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - exit_delay_ms: exitDelayMs, - exit_path: - "error" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - error_name: - streamingError instanceof Error - ? (streamingError.name as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS) - : ("unknown" as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS), - model: - options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - }); - } + if (streamingError instanceof APIUserAbortError) { + // Check if the abort signal was triggered by the user (ESC key) + // If the signal is aborted, it's a user-initiated abort + // If not, it's likely a timeout from the SDK + if (signal.aborted) { + // This is a real user abort (ESC key was pressed) + logForDebugging( + `Streaming aborted by user: ${errorMessage(streamingError)}`, + ); + if (isAdvisorInProgress) { + logEvent("tengu_advisor_tool_interrupted", { + model: + options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + advisor_model: (advisorModel ?? + "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + }); + } + throw streamingError; + } else { + // The SDK threw APIUserAbortError but our signal wasn't aborted + // This means it's a timeout from the SDK's internal timeout + logForDebugging( + `Streaming timeout (SDK abort): ${streamingError.message}`, + { level: "error" }, + ); + // Throw a more specific error for timeout + throw new APIConnectionTimeoutError({ message: "Request timed out" }); + } + } - if (streamingError instanceof APIUserAbortError) { - // Check if the abort signal was triggered by the user (ESC key) - // If the signal is aborted, it's a user-initiated abort - // If not, it's likely a timeout from the SDK - if (signal.aborted) { - // This is a real user abort (ESC key was pressed) + // When the flag is enabled, skip the non-streaming fallback and let the + // error propagate to withRetry. The mid-stream fallback causes double tool + // execution when streaming tool execution is active: the partial stream + // starts a tool, then the non-streaming retry produces the same tool_use + // and runs it again. See inc-4258. + const disableFallback = + isEnvTruthy(process.env.CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK) || + getFeatureValue_CACHED_MAY_BE_STALE( + "tengu_disable_streaming_to_non_streaming_fallback", + false, + ); + + if (disableFallback) { logForDebugging( - `Streaming aborted by user: ${errorMessage(streamingError)}`, + `Error streaming (non-streaming fallback disabled): ${errorMessage(streamingError)}`, + { level: "error" }, ); - if (isAdvisorInProgress) { - logEvent("tengu_advisor_tool_interrupted", { - model: - options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - advisor_model: (advisorModel ?? - "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + logEvent("tengu_streaming_fallback_to_non_streaming", { + model: + options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + error: + streamingError instanceof Error + ? (streamingError.name as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS) + : (String( + streamingError, + ) as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS), + attemptNumber, + maxOutputTokens, + thinkingType: + thinkingConfig.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + fallback_disabled: true, + request_id: (streamRequestId ?? + "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + fallback_cause: (streamIdleAborted + ? "watchdog" + : "other") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + }); + if ( + !signal.aborted && + streamRequests < maxStreamRequests && + isRetryableStreamError(streamingError) + ) { + const retryError = + streamingError instanceof APIError + ? streamingError + : new APIConnectionError({ + message: errorMessage(streamingError), + }); + const delayMs = getRetryDelay( + streamRequests, + getRetryAfter(streamingError), + ); + releaseStreamResources(); + options.onStreamingFallback?.(); + yield { + ...createSystemAPIErrorMessage( + retryError, + delayMs, + streamRequests, + maxStreamRequests - 1, + ), + streamRestart: true, + }; + await sleep(delayMs, signal, { + abortError: () => new APIUserAbortError(), }); + continue streamAttempts; } throw streamingError; - } else { - // The SDK threw APIUserAbortError but our signal wasn't aborted - // This means it's a timeout from the SDK's internal timeout - logForDebugging( - `Streaming timeout (SDK abort): ${streamingError.message}`, - { level: "error" }, - ); - // Throw a more specific error for timeout - throw new APIConnectionTimeoutError({ message: "Request timed out" }); } - } - // When the flag is enabled, skip the non-streaming fallback and let the - // error propagate to withRetry. The mid-stream fallback causes double tool - // execution when streaming tool execution is active: the partial stream - // starts a tool, then the non-streaming retry produces the same tool_use - // and runs it again. See inc-4258. - const disableFallback = - isEnvTruthy(process.env.CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK) || - getFeatureValue_CACHED_MAY_BE_STALE( - "tengu_disable_streaming_to_non_streaming_fallback", - false, - ); - - if (disableFallback) { logForDebugging( - `Error streaming (non-streaming fallback disabled): ${errorMessage(streamingError)}`, + `Error streaming, falling back to non-streaming mode: ${errorMessage(streamingError)}`, { level: "error" }, ); + didFallBackToNonStreaming = true; + if (options.onStreamingFallback) { + options.onStreamingFallback(); + } + logEvent("tengu_streaming_fallback_to_non_streaming", { model: options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, @@ -2541,110 +2617,80 @@ async function* queryModel( maxOutputTokens, thinkingType: thinkingConfig.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - fallback_disabled: true, + fallback_disabled: false, request_id: (streamRequestId ?? "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, fallback_cause: (streamIdleAborted ? "watchdog" : "other") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, }); - throw streamingError; - } - - logForDebugging( - `Error streaming, falling back to non-streaming mode: ${errorMessage(streamingError)}`, - { level: "error" }, - ); - didFallBackToNonStreaming = true; - if (options.onStreamingFallback) { - options.onStreamingFallback(); - } - logEvent("tengu_streaming_fallback_to_non_streaming", { - model: - options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - error: - streamingError instanceof Error - ? (streamingError.name as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS) - : (String( - streamingError, - ) as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS), - attemptNumber, - maxOutputTokens, - thinkingType: - thinkingConfig.type as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - fallback_disabled: false, - request_id: (streamRequestId ?? - "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - fallback_cause: (streamIdleAborted - ? "watchdog" - : "other") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - }); - - // Fall back to non-streaming mode with retries. - // If the streaming failure was itself a 529, count it toward the - // consecutive-529 budget so total 529s-before-model-fallback is the - // same whether the overload was hit in streaming or non-streaming mode. - // This is a speculative fix for https://github.com/anthropics/claude-code/issues/1513 - // Instrumentation: proves executeNonStreamingRequest was entered (vs. the - // fallback event firing but the call itself hanging at dispatch). - logForDiagnosticsNoPII("info", "cli_nonstreaming_fallback_started"); - logEvent("tengu_nonstreaming_fallback_started", { - request_id: (streamRequestId ?? - "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - model: - options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - fallback_cause: (streamIdleAborted - ? "watchdog" - : "other") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - }); - const result = yield* executeNonStreamingRequest( - { model: options.model, source: options.querySource }, - { - model: options.model, - fallbackModel: options.fallbackModel, - thinkingConfig, - ...(isFastModeEnabled() && { fastMode: isFastMode }), - signal, - initialConsecutive529Errors: is529Error(streamingError) ? 1 : 0, - querySource: options.querySource, - }, - paramsFromContext, - (attempt, _startTime, tokens) => { - attemptNumber = attempt; - maxOutputTokens = tokens; - }, - (params) => captureAPIRequest(params, options.querySource), - streamRequestId, - ); + // Fall back to non-streaming mode with retries. + // If the streaming failure was itself a 529, count it toward the + // consecutive-529 budget so total 529s-before-model-fallback is the + // same whether the overload was hit in streaming or non-streaming mode. + // This is a speculative fix for https://github.com/anthropics/claude-code/issues/1513 + // Instrumentation: proves executeNonStreamingRequest was entered (vs. the + // fallback event firing but the call itself hanging at dispatch). + logForDiagnosticsNoPII("info", "cli_nonstreaming_fallback_started"); + logEvent("tengu_nonstreaming_fallback_started", { + request_id: (streamRequestId ?? + "unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + model: + options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + fallback_cause: (streamIdleAborted + ? "watchdog" + : "other") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + }); + const result = yield* executeNonStreamingRequest( + { model: options.model, source: options.querySource }, + { + model: options.model, + fallbackModel: options.fallbackModel, + thinkingConfig, + ...(isFastModeEnabled() && { fastMode: isFastMode }), + signal, + initialConsecutive529Errors: is529Error(streamingError) ? 1 : 0, + querySource: options.querySource, + }, + paramsFromContext, + (attempt, _startTime, tokens) => { + attemptNumber = attempt; + maxOutputTokens = tokens; + }, + (params) => captureAPIRequest(params, options.querySource), + streamRequestId, + ); - const m: AssistantMessage = { - message: { - ...result, - usage: normalizeUsage(result.usage), - content: normalizeContentFromAPI( - result.content, - tools, - options.agentId, - ), - }, - requestId: streamRequestId ?? undefined, - type: "assistant", - uuid: randomUUID(), - timestamp: new Date().toISOString(), - ...(process.env.USER_TYPE === "ant" && - research !== undefined && { - research, + const m: AssistantMessage = { + message: { + ...result, + usage: normalizeUsage(result.usage), + content: normalizeContentFromAPI( + result.content, + tools, + options.agentId, + ), + }, + requestId: streamRequestId ?? undefined, + type: "assistant", + uuid: randomUUID(), + timestamp: new Date().toISOString(), + ...(process.env.USER_TYPE === "ant" && + research !== undefined && { + research, + }), + ...(advisorModel && { + advisorModel, }), - ...(advisorModel && { - advisorModel, - }), - }; - newMessages.push(m); - fallbackMessage = m; - yield m; - } finally { - clearStreamIdleTimers(); + }; + newMessages.push(m); + fallbackMessage = m; + yield m; + } finally { + clearStreamIdleTimers(); + } + break; } } catch (errorFromRetry) { // FallbackTriggeredError must propagate to query.ts, which performs the diff --git a/src/services/api/errors.ts b/src/services/api/errors.ts index c9237ca0..08bb50d9 100644 --- a/src/services/api/errors.ts +++ b/src/services/api/errors.ts @@ -50,6 +50,7 @@ import { } from '../claudeAiLimits.js' import { shouldProcessRateLimits } from '../rateLimitMocking.js' // Used for /mock-limits command import { extractConnectionErrorDetails, formatAPIError } from './errorUtils.js' +import { isContextOverflowText, parseContextOverflowTokens } from '../../utils/providerErrors.js' export const API_ERROR_MESSAGE_PREFIX = 'API Error' @@ -86,13 +87,8 @@ export function parsePromptTooLongTokenCounts(rawMessage: string): { actualTokens: number | undefined limitTokens: number | undefined } { - const match = rawMessage.match( - /prompt is too long[^0-9]*(\d+)\s*tokens?\s*>\s*(\d+)/i, - ) - return { - actualTokens: match ? parseInt(match[1]!, 10) : undefined, - limitTokens: match ? parseInt(match[2]!, 10) : undefined, - } + const { actualTokens, limitTokens } = parseContextOverflowTokens(rawMessage) + return { actualTokens, limitTokens } } /** @@ -559,10 +555,7 @@ export function getAssistantMessageFromError( // Handle prompt too long errors (Vertex returns 413, direct API returns 400) // Use case-insensitive check since Vertex returns "Prompt is too long" (capitalized) - if ( - error instanceof Error && - error.message.toLowerCase().includes('prompt is too long') - ) { + if (error instanceof Error && isContextOverflowText(error.message)) { // Content stays generic (UI matches on exact string). The raw error with // token counts goes into errorDetails — reactive compact's retry loop // parses the gap from there via getPromptTooLongTokenGap. @@ -1008,12 +1001,7 @@ export function classifyAPIError(error: unknown): string { } // Prompt/content size errors - if ( - error instanceof Error && - error.message - .toLowerCase() - .includes(PROMPT_TOO_LONG_ERROR_MESSAGE.toLowerCase()) - ) { + if (error instanceof Error && isContextOverflowText(error.message)) { return 'prompt_too_long' } diff --git a/src/services/api/withRetry.ts b/src/services/api/withRetry.ts index 5ec9ad08..6c22517a 100644 --- a/src/services/api/withRetry.ts +++ b/src/services/api/withRetry.ts @@ -11,17 +11,22 @@ import { isAwsCredentialsProviderError } from 'src/utils/aws.js' import { logForDebugging } from 'src/utils/debug.js' import { logError } from 'src/utils/log.js' import { createSystemAPIErrorMessage } from 'src/utils/messages.js' -import { getAPIProviderForStatsig } from 'src/utils/model/providers.js' +import { + getAPIProviderForStatsig, + isFirstPartyAnthropicBaseUrl, +} from 'src/utils/model/providers.js' import { clearApiKeyHelperCache, clearAwsCredentialsCache, clearGcpCredentialsCache, getClaudeAIOAuthTokens, + getConfiguredApiKeyHelper, handleOAuth401Error, isClaudeAISubscriber, isEnterpriseSubscriber, } from '../../utils/auth.js' import { isEnvTruthy } from '../../utils/envUtils.js' +import { isQuotaErrorText } from '../../utils/providerErrors.js' import { errorMessage } from '../../utils/errors.js' import { type CooldownReason, @@ -36,6 +41,7 @@ import { disableKeepAlive } from '../../utils/proxy.js' import { sleep } from '../../utils/sleep.js' import type { ThinkingConfig } from '../../utils/thinking.js' import { getFeatureValue_CACHED_MAY_BE_STALE } from '../analytics/growthbook.js' +import { shouldUseOpenAICodexAuth } from '../openaiAuth/fetch.js' import { type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, logEvent, @@ -516,7 +522,23 @@ export async function* withRetry( throw new CannotRetryError(lastError, retryContext) } -function getRetryAfter(error: unknown): string | null { +const NON_RETRYABLE_STREAM_ERROR_TYPES = new Set([ + 'invalid_request_error', + 'authentication_error', + 'permission_error', + 'billing_error', + 'not_found_error', + 'request_too_large', +]) + +export function isRetryableStreamError(error: unknown): boolean { + if (error instanceof APIUserAbortError) return false + if (!(error instanceof APIError)) return true + if (error.status !== undefined) return shouldRetry(error) + return !NON_RETRYABLE_STREAM_ERROR_TYPES.has(String(error.type)) +} + +export function getRetryAfter(error: unknown): string | null { return ( ((error as { headers?: { 'retry-after'?: string } }).headers?.[ 'retry-after' @@ -699,6 +721,10 @@ function shouldRetry(error: APIError): boolean { return false } + if (!isFirstPartyAnthropicBaseUrl() && isQuotaErrorText(error.message)) { + return false + } + // Persistent mode: 429/529 always retryable, bypass subscriber gates and // x-should-retry header. if (isPersistentRetryEnabled() && isTransientCapacityError(error)) { @@ -772,7 +798,7 @@ function shouldRetry(error: APIError): boolean { // OAuth token handling is done in the main retry loop via handleOAuth401Error. if (error.status === 401) { clearApiKeyHelperCache() - return true + return canRefreshCredentials() } // Retry on 403 "token revoked" (same refresh logic as 401, see above) @@ -786,6 +812,15 @@ function shouldRetry(error: APIError): boolean { return false } +function canRefreshCredentials(): boolean { + return ( + isFirstPartyAnthropicBaseUrl() || + !!getConfiguredApiKeyHelper() || + !!getClaudeAIOAuthTokens()?.accessToken || + shouldUseOpenAICodexAuth() + ) +} + export function getDefaultMaxRetries(): number { if (process.env.CLAUDE_CODE_MAX_RETRIES) { return parseInt(process.env.CLAUDE_CODE_MAX_RETRIES, 10) diff --git a/src/services/compact/autoCompact.test.ts b/src/services/compact/autoCompact.test.ts index 79436480..8a3d834b 100644 --- a/src/services/compact/autoCompact.test.ts +++ b/src/services/compact/autoCompact.test.ts @@ -3,7 +3,8 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { getAutoCompactThreshold, getEffectiveContextWindowSize } from './autoCompact.js' +import { getAutoCompactThreshold, getEffectiveContextWindowSize, shouldAutoCompact } from './autoCompact.js' +import { createAssistantMessage, createUserMessage } from '../../utils/messages.js' import { getContextWindowForModel } from '../../utils/context.js' import { MODEL_CONTEXT_WINDOWS_ENV_KEY } from '../../utils/model/modelContextWindows.js' import { resetSettingsCache } from '../../utils/settings/settingsCache.js' @@ -94,3 +95,40 @@ describe('auto-compact threshold', () => { expect(getAutoCompactThreshold('deepseek-v4-pro')).toBe(490_000) }) }) + +describe('auto compact without usage from the provider', () => { + const zeroUsage = { + input_tokens: 0, + output_tokens: 0, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + } + + function reply(content: string, usage: typeof zeroUsage) { + const message = createAssistantMessage({ content, usage: usage as never }) + message.message.model = 'kimi-k2.6' + return message + } + + test('falls back to the local estimate when the last reply reports zero usage', async () => { + const messages = [ + createUserMessage({ content: 'x'.repeat(1_200_000) }), + reply('done', zeroUsage), + createUserMessage({ content: 'next' }), + ] + + expect(await shouldAutoCompact(messages, 'kimi-k2.6')).toBe(true) + }) + + test('uses the last reply that does report usage', async () => { + const messages = [ + createUserMessage({ content: 'hello' }), + reply('first', { ...zeroUsage, input_tokens: 250_000 }), + createUserMessage({ content: 'more' }), + reply('second', zeroUsage), + createUserMessage({ content: 'next' }), + ] + + expect(await shouldAutoCompact(messages, 'kimi-k2.6')).toBe(true) + }) +}) diff --git a/src/services/openaiAuth/fetch.test.ts b/src/services/openaiAuth/fetch.test.ts index ff28642e..eea21e4c 100644 --- a/src/services/openaiAuth/fetch.test.ts +++ b/src/services/openaiAuth/fetch.test.ts @@ -157,4 +157,37 @@ describe('buildOpenAICodexFetch', () => { error: { type: 'api_error', message: 'model overloaded' }, }) }) + + test.each([ + ['a JSON reply', () => Response.json({ + id: 'resp_1', + object: 'response', + created_at: 0, + model: 'gpt-5.5', + status: 'incomplete', + incomplete_details: { reason: 'content_filter' }, + output: [], + })], + ['a streamed reply', () => new Response([ + 'event: response.incomplete', + 'data: {"type":"response.incomplete","response":{"id":"resp_1","status":"incomplete","incomplete_details":{"reason":"content_filter"},"output":[]}}', + '', + ].join('\n'), { headers: { 'Content-Type': 'text/event-stream' } })], + ])('answers a content-filtered %s for a non-streaming caller with an Anthropic error', async (_kind, reply) => { + const openAIFetch = buildOpenAICodexFetch(async () => reply(), 'test') + const response = await openAIFetch('https://api.anthropic.com/v1/messages', { + method: 'POST', + body: JSON.stringify({ + model: 'gpt-5.5', + max_tokens: 64, + messages: [{ role: 'user', content: 'Say ok' }], + }), + }) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + type: 'error', + error: { type: 'invalid_request_error', message: "The provider's content filter stopped the reply (incomplete: content_filter)." }, + }) + }) }) diff --git a/src/services/openaiAuth/fetch.ts b/src/services/openaiAuth/fetch.ts index 81be5320..c1df0e42 100644 --- a/src/services/openaiAuth/fetch.ts +++ b/src/services/openaiAuth/fetch.ts @@ -7,10 +7,8 @@ import { getOpenAIOAuthTokens } from './storage.js' import { anthropicToOpenaiResponses } from '../../server/proxy/transform/anthropicToOpenaiResponses.js' import { openaiResponsesToAnthropic } from '../../server/proxy/transform/openaiResponsesToAnthropic.js' import { openaiResponsesStreamToAnthropic } from '../../server/proxy/streaming/openaiResponsesStreamToAnthropic.js' -import { - OpenAIResponsesStreamError, - openaiResponsesStreamToAnthropicResponse, -} from '../../server/proxy/streaming/openaiResponsesStreamToAnthropicResponse.js' +import { openaiResponsesStreamToAnthropicResponse } from '../../server/proxy/streaming/openaiResponsesStreamToAnthropicResponse.js' +import { anthropicErrorResponse, UpstreamReplyError } from '../../server/upstream/upstreamError.js' import type { AnthropicRequest } from '../../server/proxy/transform/types.js' import { logForDebugging } from '../../utils/debug.js' @@ -111,29 +109,15 @@ export function buildOpenAICodexFetch( ) } - if (upstream.body && isEventStreamResponse(upstream)) { - try { - const responseBody = await openaiResponsesStreamToAnthropicResponse( - upstream.body, - mappedModel, - ) - return Response.json(responseBody) - } catch (error) { - if (!(error instanceof OpenAIResponsesStreamError)) throw error - return Response.json( - { - type: 'error', - error: { type: 'api_error', message: error.message }, - }, - { status: 502 }, - ) - } + try { + const responseBody = upstream.body && isEventStreamResponse(upstream) + ? await openaiResponsesStreamToAnthropicResponse(upstream.body, mappedModel) + : openaiResponsesToAnthropic(await upstream.json(), mappedModel) + return Response.json(responseBody) + } catch (error) { + if (!(error instanceof UpstreamReplyError)) throw error + return anthropicErrorResponse(error.error) } - - const responseBody = await upstream.json() - return Response.json( - openaiResponsesToAnthropic(responseBody, mappedModel), - ) } } diff --git a/src/services/tools/__tests__/truncatedToolArguments.test.ts b/src/services/tools/__tests__/truncatedToolArguments.test.ts new file mode 100644 index 00000000..e9da2a3c --- /dev/null +++ b/src/services/tools/__tests__/truncatedToolArguments.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, test } from 'bun:test' +import { openaiChatToAnthropic } from '../../../server/proxy/transform/openaiChatToAnthropic.js' +import { parseOpenAIToolArguments } from '../../../server/proxy/transform/toolArguments.js' +import type { OpenAIChatResponse } from '../../../server/proxy/transform/types.js' +import type { Tool, ToolUseContext, Tools } from '../../../Tool.js' +import type { AssistantMessage } from '../../../types/message.js' +import { createAssistantMessage, normalizeContentFromAPI } from '../../../utils/messages.js' +import { runToolUse } from '../toolExecution.js' + +const truncated = '{"file_path":"/tmp/a.ts","content":"export const a = 1\\nexport const b' + +function writeTool(calls: unknown[]): Tool { + return { + name: 'Write', + call: async (input: unknown) => { + calls.push(input) + return { data: 'written' } + }, + } as unknown as Tool +} + +function normalizedInput(input: unknown, tools: Tools): unknown { + const [block] = normalizeContentFromAPI( + [{ type: 'tool_use', id: 'toolu_1', name: 'Write', input } as never], + tools, + ) + return (block as { input: unknown }).input +} + +async function run(input: unknown, tools: Tools) { + const assistant = createAssistantMessage({ + content: [{ type: 'tool_use', id: 'toolu_1', name: 'Write', input } as never], + }) as AssistantMessage + const context = { + options: { tools, mcpClients: [] }, + abortController: new AbortController(), + } as unknown as ToolUseContext + const updates = [] + for await (const update of runToolUse( + { type: 'tool_use', id: 'toolu_1', name: 'Write', input } as never, + assistant, + async () => ({ behavior: 'allow' }) as never, + context, + )) { + updates.push(update) + } + return updates +} + +describe('truncated tool arguments', () => { + test('the non-streaming proxy passes unparseable arguments on as the raw string', () => { + const response = { + id: 'c1', + object: 'chat.completion', + created: 0, + model: 'deepseek-chat', + choices: [{ + index: 0, + message: { role: 'assistant', content: null, tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'Write', arguments: truncated } }] }, + finish_reason: 'length', + }], + } as OpenAIChatResponse + + const block = openaiChatToAnthropic(response, 'deepseek-chat').content[0] + + expect(block).toMatchObject({ type: 'tool_use', input: truncated }) + }) + + test.each([ + ['a streamed tool call cut off by the output limit', truncated], + ['arguments that are JSON but not an object', '["a.ts"]'], + ])('marks %s instead of turning it into an empty object', (_kind, input) => { + const marked = normalizedInput(input, [writeTool([])]) + + expect(marked).toEqual({ + __dreamcoder_invalid_arguments: { receivedChars: input.length, parseError: expect.any(String) }, + }) + }) + + test.each(['', ' ', 'null'])('still treats the argument string %p as no arguments', (input) => { + expect(normalizedInput(input, [writeTool([])])).toEqual({}) + }) + + test.each(['null', ' '])('the non-streaming proxy turns the argument string %p into no arguments', (args) => { + expect(parseOpenAIToolArguments(args)).toEqual({}) + }) + + test('does not run the tool and tells the model the arguments were cut off', async () => { + const calls: unknown[] = [] + const tools = [writeTool(calls)] + + const updates = await run(normalizedInput(truncated, tools), tools) + + expect(calls).toHaveLength(0) + expect(updates).toHaveLength(1) + const [result] = (updates[0]!.message as { message: { content: Array> } }).message.content + expect(result).toMatchObject({ type: 'tool_result', tool_use_id: 'toolu_1', is_error: true }) + expect(String(result!.content)).toContain(`received ${truncated.length} characters`) + expect(String(result!.content)).toContain('cut off by the output token limit') + }) +}) diff --git a/src/services/tools/toolExecution.ts b/src/services/tools/toolExecution.ts index 4e024010..22d107a1 100644 --- a/src/services/tools/toolExecution.ts +++ b/src/services/tools/toolExecution.ts @@ -69,6 +69,10 @@ import { TelemetrySafeError_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, } from '../../utils/errors.js' import { executePermissionDeniedHooks } from '../../utils/hooks.js' +import { + invalidToolArgumentsMessage, + readInvalidToolArguments, +} from '../../utils/invalidToolArguments.js' import { logError } from '../../utils/log.js' import { CANCEL_MESSAGE, @@ -411,6 +415,26 @@ export async function* runToolUse( return } + const invalidArguments = readInvalidToolArguments(toolUse.input) + if (invalidArguments) { + const message = invalidToolArgumentsMessage(invalidArguments) + yield { + message: createUserMessage({ + content: [ + { + type: 'tool_result', + content: `${message}`, + is_error: true, + tool_use_id: toolUse.id, + }, + ], + toolUseResult: message, + sourceToolAssistantUUID: assistantMessage.uuid, + }), + } + return + } + const toolInput = toolUse.input as { [key: string]: string } try { if (toolUseContext.abortController.signal.aborted) { diff --git a/src/utils/invalidToolArguments.ts b/src/utils/invalidToolArguments.ts new file mode 100644 index 00000000..bc00bcd7 --- /dev/null +++ b/src/utils/invalidToolArguments.ts @@ -0,0 +1,32 @@ +const INVALID_TOOL_ARGUMENTS_KEY = '__dreamcoder_invalid_arguments' + +export type InvalidToolArguments = { receivedChars: number; parseError: string } + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +export function parseToolArguments(raw: string): Record { + if (raw.trim().length === 0) return {} + let parseError: string + try { + const parsed: unknown = JSON.parse(raw) + if (parsed === null) return {} + if (isRecord(parsed)) return parsed + parseError = 'the arguments are not a JSON object' + } catch (error) { + parseError = error instanceof Error ? error.message : String(error) + } + const invalid: InvalidToolArguments = { receivedChars: raw.length, parseError } + return { [INVALID_TOOL_ARGUMENTS_KEY]: invalid } +} + +export function readInvalidToolArguments(input: unknown): InvalidToolArguments | undefined { + if (!isRecord(input)) return undefined + const invalid = input[INVALID_TOOL_ARGUMENTS_KEY] + return isRecord(invalid) ? invalid as InvalidToolArguments : undefined +} + +export function invalidToolArgumentsMessage({ receivedChars, parseError }: InvalidToolArguments): string { + return `The arguments of this tool call are not a complete JSON object (received ${receivedChars} characters: ${parseError}). They were most likely cut off by the output token limit, so the tool was not run. Call the tool again with shorter arguments, for example by writing a large file in several smaller parts.` +} diff --git a/src/utils/messages.ts b/src/utils/messages.ts index 29e0c35f..647c56e8 100644 --- a/src/utils/messages.ts +++ b/src/utils/messages.ts @@ -156,6 +156,10 @@ import { stripIdeContextTags } from './displayTags.js' import { hasEmbeddedSearchTools } from './embeddedTools.js' import { formatFileSize } from './format.js' import { validateImagesForAPI } from './imageValidation.js' +import { + parseToolArguments, + readInvalidToolArguments, +} from './invalidToolArguments.js' import { safeParseJSON } from './json.js' import { logError, logMCPDebug } from './log.js' import { normalizeLegacyToolName } from './permissions/permissionRuleParser.js' @@ -2768,12 +2772,8 @@ export function normalizeContentFromAPI( // TODO: This needs patching as recursive fields can still be stringified let normalizedInput: unknown if (typeof contentBlock.input === 'string') { - const parsed = safeParseJSON(contentBlock.input) - if (parsed === null && contentBlock.input.length > 0) { - // TET/FC-v3 diagnostic: the streamed tool input JSON failed to - // parse. We fall back to {} which means downstream validation - // sees empty input. The raw prefix goes to debug log only — no - // PII-tagged proto column exists for it yet. + normalizedInput = parseToolArguments(contentBlock.input) + if (readInvalidToolArguments(normalizedInput)) { logEvent('tengu_tool_input_json_parse_fail', { toolName: sanitizeToolNameForAnalytics(contentBlock.name), inputLen: contentBlock.input.length, @@ -2785,13 +2785,16 @@ export function normalizeContentFromAPI( ) } } - normalizedInput = parsed ?? {} } else { normalizedInput = contentBlock.input } // Then apply tool-specific corrections - if (typeof normalizedInput === 'object' && normalizedInput !== null) { + if ( + typeof normalizedInput === 'object' && + normalizedInput !== null && + !readInvalidToolArguments(normalizedInput) + ) { const tool = findToolByName(tools, contentBlock.name) if (tool) { try { diff --git a/src/utils/providerErrors.ts b/src/utils/providerErrors.ts new file mode 100644 index 00000000..b0d74537 --- /dev/null +++ b/src/utils/providerErrors.ts @@ -0,0 +1,88 @@ +export type ProviderErrorKind = 'context_overflow' | 'quota' | 'auth' | 'rate_limit' + +const OVERFLOW_CODES = new Set([ + 'context_length_exceeded', + 'context_window_exceeded', + 'model_context_window_exceeded', + '1261', +]) + +const OVERFLOW_PATTERNS = [ + /prompt is too long/i, + /maximum context length/i, + /context[ _](?:length|window)[ _](?:is )?exceeded/i, + /exceeds? the (?:model's )?context window/i, + /range of input length should be/i, + /input token count.*exceeds? the maximum number of tokens/i, + /exceeded model token limit/i, + /message token length.*exceeds? model limit/i, + /prompt exceeds max length/i, +] + +const QUOTA_CODES = new Set([ + 'insufficient_quota', + 'exceeded_current_quota_error', + 'billing_hard_limit_reached', + 'insufficient_balance', + '1113', +]) + +const QUOTA_PATTERNS = [ + /insufficient[ _](?:quota|balance|credits)/i, + /exceeded (?:your |the )?current quota/i, + /exceeded_current_quota_error/i, + /billing_hard_limit_reached/i, + /"code"\s*:\s*"?1113\b/, + /余额不足/, +] + +const AUTH_CODES = new Set(['invalid_api_key', 'invalid_authentication', 'authentication_error', 'unauthorized']) + +const RATE_LIMIT_CODES = new Set(['rate_limit_exceeded', 'rate_limit_error', '1302', '1303', '1305']) + +const TOKEN_COUNT_PATTERNS: Array<{ pattern: RegExp; actual: number; limit: number }> = [ + { pattern: /prompt is too long[^0-9]*(\d+)\s*tokens?\s*>\s*(\d+)/i, actual: 1, limit: 2 }, + { pattern: /maximum context length is (\d+) tokens.*?(?:resulted in|requested about) (\d+) tokens/is, actual: 2, limit: 1 }, + { pattern: /input token count \((\d+)\) exceeds? the maximum number of tokens allowed \((\d+)\)/i, actual: 1, limit: 2 }, + { pattern: /message token length (\d+) exceeds? model limit (\d+)/i, actual: 1, limit: 2 }, +] + +function hasCode(codes: Set, code: unknown): boolean { + return (typeof code === 'string' || typeof code === 'number') && codes.has(String(code).toLowerCase()) +} + +function isContextOverflowCode(code: unknown): boolean { + return hasCode(OVERFLOW_CODES, code) +} + +export function isQuotaErrorText(text: string): boolean { + return QUOTA_PATTERNS.some((pattern) => pattern.test(text)) +} + +export function classifyProviderError(error: { status?: number; code?: string; message: string }): ProviderErrorKind | undefined { + if (isContextOverflowCode(error.code) || isContextOverflowText(error.message)) return 'context_overflow' + if (error.status === 402 || hasCode(QUOTA_CODES, error.code) || isQuotaErrorText(error.message)) return 'quota' + if (error.status === 401 || error.status === 403 || hasCode(AUTH_CODES, error.code)) return 'auth' + if (error.status === 429 || hasCode(RATE_LIMIT_CODES, error.code)) return 'rate_limit' + return undefined +} + +export function isContextOverflowText(text: string): boolean { + return OVERFLOW_PATTERNS.some((pattern) => pattern.test(text)) +} + +export function parseContextOverflowTokens(text: string): { actualTokens?: number; limitTokens?: number } { + for (const { pattern, actual, limit } of TOKEN_COUNT_PATTERNS) { + const match = text.match(pattern) + if (match) return { actualTokens: Number(match[actual]), limitTokens: Number(match[limit]) } + } + return {} +} + +export function toPromptTooLongMessage(text: string): string { + const { actualTokens, limitTokens } = parseContextOverflowTokens(text) + const counts = actualTokens !== undefined && limitTokens !== undefined + ? `: ${actualTokens} tokens > ${limitTokens} maximum` + : '' + return `prompt is too long${counts} · ${text}` +} diff --git a/src/utils/tokens.ts b/src/utils/tokens.ts index 3c3034bb..54bf4428 100644 --- a/src/utils/tokens.ts +++ b/src/utils/tokens.ts @@ -12,7 +12,8 @@ export function getTokenUsage(message: Message): Usage | undefined { message.message.content[0]?.type === 'text' && SYNTHETIC_MESSAGES.has(message.message.content[0].text) ) && - message.message.model !== SYNTHETIC_MODEL + message.message.model !== SYNTHETIC_MODEL && + getTokenCountFromUsage(message.message.usage) > 0 ) { return message.message.usage }