From 6f4a0fcb3664b095c5b0a650e9b6ac173150d204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A2=A8=E8=8F=8A?= <277378677+maiphucgiang@users.noreply.github.com> Date: Thu, 24 Sep 2026 05:57:24 +0800 Subject: [PATCH 1/3] Enable opt-in realtime streaming across client protocols --- .env.example | 3 + app/adapters/anthropic_adapter.py | 211 +++++- app/adapters/responses_adapter.py | 300 ++++++-- app/audit_store.py | 2 + app/observability.py | 7 + app/settings.py | 2 + app/upstream_io.py | 208 +++++- converter.py | 371 +++++++--- docker-compose.yml | 1 + docs/advanced.md | 51 +- docs/advanced.zh-CN.md | 51 +- docs/clients.md | 4 +- docs/clients.zh-CN.md | 4 +- tests/test_environment_config.py | 22 +- tests/test_realtime_streaming.py | 1134 +++++++++++++++++++++++++++++ tests/test_webui_integration.py | 14 + 16 files changed, 2172 insertions(+), 213 deletions(-) create mode 100644 tests/test_realtime_streaming.py diff --git a/.env.example b/.env.example index 33534d6..153de6b 100644 --- a/.env.example +++ b/.env.example @@ -46,6 +46,9 @@ CODEBUDDY2API_MAX_CONCURRENT=64 # CODEBUDDY2API_MAX_INFLIGHT_PER_ACCOUNT=0 # Optional session/attempt tracing; legacy preserves existing upstream header behavior. # CODEBUDDY2API_REQUEST_CONTEXT_MODE=legacy +# compatible preserves aggregation; realtime forwards all protocol output incrementally +# and does not regenerate malformed tool arguments. +# CODEBUDDY2API_STREAM_MODE=compatible # Disable only declared model-capability preflight; other safeguards and Intl image merging remain enabled. # CODEBUDDY2API_MODEL_CAPABILITY_GUARD=true diff --git a/app/adapters/anthropic_adapter.py b/app/adapters/anthropic_adapter.py index 3dcf71d..1be6885 100644 --- a/app/adapters/anthropic_adapter.py +++ b/app/adapters/anthropic_adapter.py @@ -7,6 +7,9 @@ import time from typing import Any +from app.upstream_io import (StreamOutputBudget, merge_tool_call_delta, new_tool_state, + seal_tool_identities, tool_identity_complete) + # --------------------------------------------------------------------------- # ID generation # --------------------------------------------------------------------------- @@ -236,9 +239,17 @@ def _convert_anthropic_tools(tools: list) -> list: class AnthropicStreamConverter: """Convert Chat SSE increments to Anthropic Messages events.""" - def __init__(self, model: str = "unknown"): + def __init__(self, model: str = "unknown", *, realtime: bool = False, + budget: StreamOutputBudget | None = None, tool_states: dict | None = None, + declared_names=None): self.msg_id = _rand_id("msg_") self.model = model + self._realtime = bool(realtime) + self._budget = budget if budget is not None else StreamOutputBudget(0) + self._tool_states = tool_states + self._local_tool_states: dict[int, dict] = {} + self._declared_names = frozenset( + value for value in (declared_names or ()) if isinstance(value, str) and value) self.created_at = int(time.time()) # Stream state @@ -282,6 +293,14 @@ def feed_line(self, line: str) -> str: def finish(self) -> str: """Emit final events and close the message.""" events: list[str] = [] + if self._realtime: + if (self._tool_uses and self._finish_reason not in + ("tool_calls", "length", "content_filter", "content-filter", "refusal") + and not self._content_filter): + raise ValueError("tool calls require a tool_calls finish reason") + seal_tool_identities(self._tool_states if self._tool_states is not None + else self._local_tool_states, self._declared_names) + self._flush_ready_tools(events) # Close thinking blocks. if self._thinking_block_open: @@ -304,6 +323,7 @@ def finish(self) -> str: "content_block_stop", {"index": tc["block_idx"]} )) tc["open"] = False + tc["terminal_closed"] = True # Map the finish reason. sr = self._finish_reason or "stop" @@ -326,6 +346,30 @@ def finish(self) -> str: return "".join(events) + def mark_content_filter(self) -> None: + """Keep a detector-confirmed refusal from becoming a tool-choice error.""" + self._content_filter = True + + def set_validated_tools(self, tool_calls) -> None: + """Accept terminal metadata already validated by the shared Chat accumulator.""" + if not self._realtime: + return + expected = [self._tool_uses[index] for index in sorted(self._tool_uses)] + if len(expected) != len(tool_calls or []): + raise ValueError("validated tool calls do not match stream items") + for slot, call in zip(expected, tool_calls or []): + function = call.get("function") or {} + state = slot.get("state") + values = ((slot.get("id"), call.get("id")), + (slot.get("name"), function.get("name"))) + if state is not None: + values += ((state.get("id"), call.get("id")), + (state.get("name"), function.get("name")), + (state.get("arguments"), function.get("arguments"))) + if any(left != right for left, right in values): + raise ValueError("validated tool metadata does not match stream items") + slot["validated"] = True + def get_nonstream_response(self) -> dict: """Return the complete non-streaming Message response.""" content = self._build_content_blocks() @@ -377,13 +421,24 @@ def _process_chunk(self, chunk: dict) -> str: for choice in chunk.get("choices", []): delta = choice.get("delta", {}) - finish = choice.get("finish_reason") + finish = choice.get("finish_reason") or None + if self._finish_reason is not None: + if (any(delta.get(key) for key in ("content", "reasoning_content", "refusal")) + or bool(delta.get("tool_calls")) or bool(delta.get("function_call"))): + raise ValueError("output after finish_reason") + if finish is not None and finish != self._finish_reason: + raise ValueError("changed finish_reason") # Emit reasoning before text. thinking = delta.get("reasoning_content") if thinking: + self._budget.charge_text(thinking) self._thinking_content += thinking if not self._thinking_block_open: + if self._text_block_open: + events.append(self._evt("content_block_stop", { + "index": self._text_block_idx})) + self._text_block_open = False self._thinking_block_idx = self._next_block_idx self._next_block_idx += 1 events.append(self._evt("content_block_start", { @@ -405,6 +460,7 @@ def _process_chunk(self, chunk: dict) -> str: "index": self._thinking_block_idx })) self._thinking_block_open = False + self._budget.charge_text(content) self._text_content += content if not self._text_block_open: self._text_block_idx = self._next_block_idx @@ -423,44 +479,48 @@ def _process_chunk(self, chunk: dict) -> str: for tc in delta.get("tool_calls", []): idx = tc.get("index", 0) if idx not in self._tool_uses: - block_idx = self._next_block_idx - self._next_block_idx += 1 + block_idx = None if self._realtime else self._next_block_idx + if not self._realtime: + self._next_block_idx += 1 self._tool_uses[idx] = { - "id": tc.get("id", ""), - "name": "", - "args": "", - "block_idx": block_idx, - "open": False, + "id": "", "name": "", "args": "", "block_idx": block_idx, + "open": False, "emitted_args_length": 0, } slot = self._tool_uses[idx] - if tc.get("id"): - slot["id"] = tc["id"] - fn = tc.get("function", {}) - if fn.get("name"): - slot["name"] = fn["name"] - - if not slot["open"]: - # Close thinking before a tool block begins. - if self._thinking_block_open: - events.append(self._evt("content_block_stop", { - "index": self._thinking_block_idx + if self._realtime: + state = self._sync_tool_state(idx, tc) + slot["state"] = state + else: + if tc.get("id"): + slot["id"] = tc["id"] + fn = tc.get("function", {}) + if fn.get("name"): + slot["name"] = fn["name"] + slot["_pending_args"] = fn.get("arguments") or "" + + if self._realtime: + events.extend(self._flush_tool_slot(idx, slot)) + else: + if not slot["open"]: + self._close_content_blocks(events) + events.append(self._evt("content_block_start", { + "index": slot["block_idx"], + "content_block": {"type": "tool_use", "id": slot["id"], "name": slot["name"], "input": {}}, + })) + slot["open"] = True + arguments = self._new_tool_arguments(slot) + if arguments: + events.append(self._evt("content_block_delta", { + "index": slot["block_idx"], + "delta": {"type": "input_json_delta", "partial_json": arguments}, })) - self._thinking_block_open = False - events.append(self._evt("content_block_start", { - "index": slot["block_idx"], - "content_block": {"type": "tool_use", "id": slot["id"], "name": slot["name"], "input": {}}, - })) - slot["open"] = True - - if fn.get("arguments"): - slot["args"] += fn["arguments"] - events.append(self._evt("content_block_delta", { - "index": slot["block_idx"], - "delta": {"type": "input_json_delta", "partial_json": fn["arguments"]}, - })) if finish: self._finish_reason = finish + if self._realtime: + seal_tool_identities(self._tool_states if self._tool_states is not None + else self._local_tool_states, self._declared_names) + self._flush_ready_tools(events) # Close open blocks when the upstream finishes. if self._thinking_block_open: @@ -481,9 +541,90 @@ def _process_chunk(self, chunk: dict) -> str: "index": tc["block_idx"] })) tc["open"] = False + tc["terminal_closed"] = True return "".join(events) + def _close_content_blocks(self, events: list[str]) -> None: + if self._thinking_block_open: + events.append(self._evt("content_block_stop", {"index": self._thinking_block_idx})) + self._thinking_block_open = False + if self._realtime and self._text_block_open: + events.append(self._evt("content_block_stop", {"index": self._text_block_idx})) + self._text_block_open = False + + def _sync_tool_state(self, index: int, tool: dict) -> dict: + state = ((self._tool_states or {}).get(index) + if self._tool_states is not None else self._local_tool_states.get(index)) + if state is None: + if self._tool_states is not None: + raise ValueError("tool state missing from realtime accumulator") + state = new_tool_state() + self._local_tool_states[index] = state + if self._tool_states is None: + merge_tool_call_delta(state, tool, declared_names=self._declared_names, + charge=self._budget.charge_text) + else: + state["identity_complete"] = tool_identity_complete( + state, self._declared_names, terminal=bool(state.get("_terminal"))) + slot = self._tool_uses[index] + slot["id"] = state.get("id") or "" + slot["name"] = state.get("name") or "" + return state + + def _flush_tool_slot(self, index: int, slot: dict) -> list[str]: + """Open one ready tool and flush its buffered arguments exactly once.""" + if not self._realtime or slot.get("terminal_closed"): + return [] + state = slot.get("state") + if state is None or not state.get("identity_complete"): + return [] + events: list[str] = [] + if not slot.get("open"): + if slot.get("block_idx") is None: + slot["block_idx"] = self._next_block_idx + self._next_block_idx += 1 + self._close_content_blocks(events) + events.append(self._evt("content_block_start", { + "index": slot["block_idx"], + "content_block": {"type": "tool_use", "id": state["id"], + "name": state["name"], "input": {}}, + })) + slot["open"] = True + state["identity_emitted"] = True + arguments = self._new_tool_arguments(slot) + if arguments: + events.append(self._evt("content_block_delta", { + "index": slot["block_idx"], + "delta": {"type": "input_json_delta", "partial_json": arguments}, + })) + return events + + def _flush_ready_tools(self, events: list[str]) -> None: + for index in sorted(self._tool_uses): + events.extend(self._flush_tool_slot(index, self._tool_uses[index])) + + def _tool_arguments(self, slot: dict) -> str: + if self._realtime and slot.get("state") is not None: + return slot["state"].get("arguments") or "" + return slot["args"] + + def _new_tool_arguments(self, slot: dict) -> str: + if not self._realtime: + piece = slot.get("_pending_args", "") + slot["args"] += piece + return piece + if not slot.get("open"): + return "" + arguments = self._tool_arguments(slot) + emitted = slot.get("emitted_args_length", 0) + if len(arguments) < emitted or not arguments.startswith(slot.get("_emitted_prefix", "")): + raise ValueError("non-append-only tool arguments") + piece = arguments[emitted:] + slot["emitted_args_length"] = len(arguments) + slot["_emitted_prefix"] = arguments + return piece + def _evt(self, event_type: str, data: dict) -> str: """Format an Anthropic SSE event with its event name.""" payload = {"type": event_type, **data} @@ -511,9 +652,9 @@ def _build_content_blocks(self) -> list[dict]: } # Parse tool arguments as a JSON object. try: - block["input"] = json.loads(tc["args"]) + block["input"] = json.loads(self._tool_arguments(tc)) except (json.JSONDecodeError, ValueError): - block["input"] = tc["args"] + block["input"] = self._tool_arguments(tc) blocks.append(block) return blocks diff --git a/app/adapters/responses_adapter.py b/app/adapters/responses_adapter.py index 171c671..844746b 100644 --- a/app/adapters/responses_adapter.py +++ b/app/adapters/responses_adapter.py @@ -7,6 +7,9 @@ import time from typing import Any +from app.upstream_io import (StreamOutputBudget, merge_tool_call_delta, new_tool_state, + seal_tool_identities, tool_identity_complete) + # --------------------------------------------------------------------------- # ID generation # --------------------------------------------------------------------------- @@ -267,11 +270,19 @@ def _convert_tools_for_chat(tools: list) -> list: class ResponsesStreamConverter: """Convert Chat SSE increments into Responses events.""" - def __init__(self, model: str = "unknown", parallel_tool_calls: bool = True): + def __init__(self, model: str = "unknown", parallel_tool_calls: bool = True, *, + realtime: bool = False, budget: StreamOutputBudget | None = None, + tool_states: dict | None = None, declared_names=None): self.resp_id = _rand_id("resp_") self.msg_id = _rand_id("msg_") self.model = model self._parallel_tool_calls = bool(parallel_tool_calls) + self._realtime = bool(realtime) + self._budget = budget if budget is not None else StreamOutputBudget(0) + self._tool_states = tool_states + self._local_tool_states: dict[int, dict] = {} + self._declared_names = frozenset( + value for value in (declared_names or ()) if isinstance(value, str) and value) self.created_at = int(time.time()) # Stream state @@ -285,9 +296,14 @@ def __init__(self, model: str = "unknown", parallel_tool_calls: bool = True): self._reasoning = "" self._reasoning_item_id = _rand_id("rs_") self._emitted_reasoning_item = False - self._tool_calls: dict[int, dict] = {} # index → {id, name, args, fc_id, output_idx, emitted} + self._reasoning_output_idx: int | None = None + self._message_output_idx: int | None = None + self._tool_calls: dict[int, dict] = {} # index → stable output item state + self._output_order: list[tuple[str, int | None]] = [] + self._next_output_idx = 0 self._finish_reason: str | None = None self._usage: dict | None = None + self._content_filter = False self._seq = 0 # Monotonic emitted-event sequence # Public methods @@ -307,7 +323,19 @@ def feed_line(self, line: str) -> str: def finish(self) -> str: """Close output items and emit the terminal response status.""" + if self._realtime: + if (self._tool_calls and self._finish_reason not in + ("tool_calls", "length", "content_filter", "content-filter", "refusal") + and not self._content_filter): + raise ValueError("tool calls require a tool_calls finish reason") + seal_tool_identities(self._tool_states if self._tool_states is not None + else self._local_tool_states, self._declared_names) + pending = self._flush_ready_tools() + else: + pending = [] status, reason = self._final_status() + if self._realtime: + return "".join(pending) + self._finish_realtime(status) events: list[str] = [] # Close reasoning items. @@ -356,6 +384,64 @@ def finish(self) -> str: })) return "".join(events) + def mark_content_filter(self) -> None: + """Keep a detector-confirmed refusal from becoming a successful terminal.""" + self._content_filter = True + + def set_validated_tools(self, tool_calls) -> None: + """Accept terminal tool metadata already validated by the shared Chat accumulator.""" + if not self._realtime: + return + expected = [self._tool_calls[index] for index in sorted(self._tool_calls)] + if len(expected) != len(tool_calls or []): + raise ValueError("validated tool calls do not match stream items") + for slot, call in zip(expected, tool_calls or []): + function = call.get("function") or {} + state = slot.get("state") + values = ((slot.get("id"), call.get("id")), + (slot.get("name"), function.get("name"))) + if state is not None: + values += ((state.get("id"), call.get("id")), + (state.get("name"), function.get("name")), + (state.get("arguments"), function.get("arguments"))) + if any(left != right for left, right in values): + raise ValueError("validated tool metadata does not match stream items") + slot["validated"] = True + + def _finish_realtime(self, status: str) -> str: + events: list[str] = [] + for kind, index in self._output_order: + if kind == "reasoning" and self._emitted_reasoning_item: + output_index = self._reasoning_output_idx + events.append(self._evt("response.reasoning_summary_text.done", { + "output_index": output_index, "summary_index": 0, "text": self._reasoning, + "item_id": self._reasoning_item_id})) + events.append(self._evt("response.output_item.done", { + "output_index": output_index, "item": self._reasoning_item(status)})) + elif kind == "message" and self._emitted_msg_item: + output_index = self._message_output_idx + events.append(self._evt("response.output_text.done", { + "output_index": output_index, "content_index": 0, "text": self._content, + "item_id": self.msg_id})) + events.append(self._evt("response.content_part.done", { + "output_index": output_index, "content_index": 0, + "part": {"type": "output_text", "text": self._content, "annotations": []}, + "item_id": self.msg_id})) + events.append(self._evt("response.output_item.done", { + "output_index": output_index, "item": self._msg_item(status)})) + elif kind == "tool": + slot = self._tool_calls[index] + if slot.get("emitted"): + output_index = slot["output_idx"] + events.append(self._evt("response.function_call_arguments.done", { + "output_index": output_index, "arguments": self._tool_arguments(slot), + "item_id": slot["fc_id"]})) + events.append(self._evt("response.output_item.done", { + "output_index": output_index, "item": self._fc_item(slot, status)})) + events.append(self._evt(f"response.{status}", { + "response": self._response_obj(status, incomplete_reason=self._final_status()[1])})) + return "".join(events) + def get_nonstream_response(self) -> dict: """Return the complete non-streaming Response object.""" status, reason = self._final_status() @@ -364,6 +450,8 @@ def get_nonstream_response(self) -> dict: def _final_status(self) -> tuple[str, str | None]: """Map finish reasons to response status without hiding truncation or filtering.""" fr = self._finish_reason + if self._content_filter and fr in (None, "stop", "tool_calls"): + return "incomplete", "content_filter" if fr in (None, "stop", "tool_calls"): return "completed", None if fr == "length": @@ -393,21 +481,29 @@ def _process_chunk(self, chunk: dict) -> str: for choice in chunk.get("choices", []): delta = choice.get("delta", {}) - finish = choice.get("finish_reason") + finish = choice.get("finish_reason") or None + if self._finish_reason is not None: + if (any(delta.get(key) for key in ("content", "reasoning_content", "refusal")) + or bool(delta.get("tool_calls")) or bool(delta.get("function_call"))): + raise ValueError("output after finish_reason") + if finish is not None and finish != self._finish_reason: + raise ValueError("changed finish_reason") # Emit reasoning before message content. reasoning = delta.get("reasoning_content") if reasoning: if not self._emitted_reasoning_item: + self._reasoning_output_idx = self._claim_output("reasoning") events.append(self._evt("response.output_item.added", { - "output_index": 0, + "output_index": self._reasoning_output_idx, "item": {"type": "reasoning", "id": self._reasoning_item_id, "summary": [], "status": "in_progress"} })) self._emitted_reasoning_item = True + self._budget.charge_text(reasoning) self._reasoning += reasoning events.append(self._evt("response.reasoning_summary_text.delta", { - "output_index": 0, "summary_index": 0, "delta": reasoning, + "output_index": self._reasoning_output_idx, "summary_index": 0, "delta": reasoning, "item_id": self._reasoning_item_id })) @@ -415,8 +511,9 @@ def _process_chunk(self, chunk: dict) -> str: content = (delta.get("content") or "") + (delta.get("refusal") or "") if content: if not self._emitted_msg_item: + self._message_output_idx = self._claim_output("message") events.append(self._evt("response.output_item.added", { - "output_index": self._msg_idx(), + "output_index": self._message_output_idx, "item": self._msg_item("in_progress", empty=True) })) self._emitted_msg_item = True @@ -429,6 +526,7 @@ def _process_chunk(self, chunk: dict) -> str: })) self._emitted_content_part = True + self._budget.charge_text(content) self._content += content events.append(self._evt("response.output_text.delta", { "output_index": self._msg_idx(), "content_index": 0, "delta": content, @@ -439,44 +537,47 @@ def _process_chunk(self, chunk: dict) -> str: for tc in delta.get("tool_calls", []): idx = tc.get("index", 0) if idx not in self._tool_calls: - # Place function calls after reasoning and message items. - base = (1 if self._emitted_reasoning_item else 0) + \ - (1 if (self._emitted_msg_item or self._content) else 0) - oi = base + len(self._tool_calls) + output_idx = None if self._realtime else self._claim_output("tool", idx) self._tool_calls[idx] = { - "id": tc.get("id", ""), - "name": "", - "args": "", - "fc_id": _rand_id("fc_"), - "output_idx": oi, - "emitted": False, + "id": tc.get("id", ""), "name": "", "args": "", + "fc_id": _rand_id("fc_"), "output_idx": output_idx, + "emitted": False, "emitted_args_length": 0, } slot = self._tool_calls[idx] - if tc.get("id"): - slot["id"] = tc["id"] - fn = tc.get("function", {}) - if fn.get("name"): - slot["name"] = fn["name"] - - if not slot["emitted"]: - # Ensure the message item exists even when empty. - if not self._emitted_msg_item and (self._content or not self._tool_calls): - pass - events.append(self._evt("response.output_item.added", { - "output_index": slot["output_idx"], - "item": self._fc_item(slot, "in_progress") - })) - slot["emitted"] = True - - if fn.get("arguments"): - slot["args"] += fn["arguments"] - events.append(self._evt("response.function_call_arguments.delta", { - "output_index": slot["output_idx"], - "delta": fn["arguments"], "item_id": slot["fc_id"] - })) + if self._realtime: + state = self._sync_tool_state(idx, tc) + slot["state"] = state + else: + if tc.get("id"): + slot["id"] = tc["id"] + fn = tc.get("function", {}) + if fn.get("name"): + slot["name"] = fn["name"] + slot["_pending_args"] = fn.get("arguments") or "" + + if self._realtime: + events.extend(self._flush_tool_slot(idx, slot)) + else: + if not slot["emitted"]: + events.append(self._evt("response.output_item.added", { + "output_index": slot["output_idx"], + "item": self._fc_item(slot, "in_progress") + })) + slot["emitted"] = True + arguments = self._new_tool_arguments(slot) + if arguments: + events.append(self._evt("response.function_call_arguments.delta", { + "output_index": slot["output_idx"], + "delta": arguments, "item_id": slot["fc_id"] + })) if finish: self._finish_reason = finish + if self._realtime: + seal_tool_identities(self._tool_states if self._tool_states is not None + else self._local_tool_states, self._declared_names) + for pending_idx in sorted(self._tool_calls): + events.extend(self._flush_tool_slot(pending_idx, self._tool_calls[pending_idx])) return "".join(events) @@ -486,9 +587,95 @@ def _evt(self, event_type: str, data: dict) -> str: payload = {"type": event_type, **data, "sequence_number": self._seq} return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" + def _claim_output(self, kind: str, index: int | None = None) -> int: + if self._realtime: + output_index = self._next_output_idx + self._next_output_idx += 1 + self._output_order.append((kind, index)) + return output_index + if kind == "reasoning": + return 0 + if kind == "message": + return 1 if self._emitted_reasoning_item else 0 + return ((1 if self._emitted_reasoning_item else 0) + + (1 if self._emitted_msg_item else 0) + len(self._tool_calls)) + + def _sync_tool_state(self, index: int, tool: dict) -> dict: + state = ((self._tool_states or {}).get(index) + if self._tool_states is not None else self._local_tool_states.get(index)) + if state is None: + if self._tool_states is not None: + raise ValueError("tool state missing from realtime accumulator") + state = new_tool_state() + self._local_tool_states[index] = state + if self._tool_states is None: + merge_tool_call_delta(state, tool, declared_names=self._declared_names, + charge=self._budget.charge_text) + else: + state["identity_complete"] = tool_identity_complete( + state, self._declared_names, terminal=bool(state.get("_terminal"))) + slot = self._tool_calls[index] + slot["id"] = state.get("id") or "" + slot["name"] = state.get("name") or "" + return state + + def _flush_tool_slot(self, index: int, slot: dict) -> list[str]: + """Start one ready tool and flush its buffered arguments exactly once.""" + if not self._realtime: + return [] + state = slot.get("state") + if state is None or not state.get("identity_complete"): + return [] + if not slot.get("emitted"): + if slot.get("output_idx") is None: + slot["output_idx"] = self._claim_output("tool", index) + events = [self._evt("response.output_item.added", { + "output_index": slot["output_idx"], + "item": self._fc_item(slot, "in_progress", include_arguments=False) + })] + slot["emitted"] = True + state["identity_emitted"] = True + else: + events = [] + arguments = self._new_tool_arguments(slot) + if arguments: + events.append(self._evt("response.function_call_arguments.delta", { + "output_index": slot["output_idx"], + "delta": arguments, "item_id": slot["fc_id"] + })) + return events + + def _flush_ready_tools(self) -> list[str]: + events: list[str] = [] + for index in sorted(self._tool_calls): + events.extend(self._flush_tool_slot(index, self._tool_calls[index])) + return events + + def _tool_arguments(self, slot: dict) -> str: + if self._realtime and slot.get("state") is not None: + return slot["state"].get("arguments") or "" + return slot["args"] + + def _new_tool_arguments(self, slot: dict) -> str: + if not self._realtime: + piece = slot.get("_pending_args", "") + slot["args"] += piece + return piece + if not slot.get("emitted"): + return "" + arguments = self._tool_arguments(slot) + emitted = slot.get("emitted_args_length", 0) + if len(arguments) < emitted or not arguments.startswith(slot.get("_emitted_prefix", "")): + raise ValueError("non-append-only tool arguments") + piece = arguments[emitted:] + slot["emitted_args_length"] = len(arguments) + slot["_emitted_prefix"] = arguments + return piece + def _msg_idx(self) -> int: - """Place the message at index one when reasoning occupies index zero.""" - return 1 if self._emitted_reasoning_item else 0 + """Return the stable message index in realtime mode or the canonical placement.""" + return (self._message_output_idx if self._realtime else + 1 if self._emitted_reasoning_item else 0) def _reasoning_item(self, status: str) -> dict: """Build a reasoning item with its text in the first summary block.""" @@ -507,26 +694,35 @@ def _msg_item(self, status: str = "in_progress", empty: bool = False) -> dict: "content": content, } - def _fc_item(self, tc: dict, status: str) -> dict: + def _fc_item(self, tc: dict, status: str, *, include_arguments: bool = True) -> dict: return { "type": "function_call", "id": tc["fc_id"], - "call_id": tc["id"], - "name": tc["name"], - "arguments": tc["args"], + "call_id": tc["id"] or (tc.get("state", {}).get("id") or ""), + "name": tc["name"] or (tc.get("state", {}).get("name") or ""), + "arguments": self._tool_arguments(tc) if include_arguments else "", "status": status, } def _response_obj(self, status: str, incomplete_reason: str | None = None) -> dict: output = [] - if self._emitted_reasoning_item: - output.append(self._reasoning_item(status)) - if self._emitted_msg_item or self._content: - output.append(self._msg_item(status)) - for idx in sorted(self._tool_calls): - tc = self._tool_calls[idx] - if tc.get("emitted"): - output.append(self._fc_item(tc, status)) + if self._realtime: + for kind, index in self._output_order: + if kind == "reasoning" and self._emitted_reasoning_item: + output.append(self._reasoning_item(status)) + elif kind == "message" and self._emitted_msg_item: + output.append(self._msg_item(status)) + elif kind == "tool" and self._tool_calls[index].get("emitted"): + output.append(self._fc_item(self._tool_calls[index], status)) + else: + if self._emitted_reasoning_item: + output.append(self._reasoning_item(status)) + if self._emitted_msg_item or self._content: + output.append(self._msg_item(status)) + for idx in sorted(self._tool_calls): + tc = self._tool_calls[idx] + if tc.get("emitted"): + output.append(self._fc_item(tc, status)) usage = None if self._usage: diff --git a/app/audit_store.py b/app/audit_store.py index 7a608a9..6436fc4 100644 --- a/app/audit_store.py +++ b/app/audit_store.py @@ -265,6 +265,8 @@ def _aggregate(self, record): def _sanitize_record(self, source): result = {key: safe_label(source.get(key)) for key in ("upstream_model", "profile", "credential", "protocol", "error_code", "usage_source")} + result["stream_mode"] = (source.get("stream_mode") + if source.get("stream_mode") in ("compatible", "realtime") else None) result["public_model"] = safe_label(source.get("public_model", source.get("model"))) result["model"] = result["public_model"] result["id"] = safe_label(source.get("id", source.get("event_id"))) or uuid.uuid4().hex diff --git a/app/observability.py b/app/observability.py index b0e75a2..81df0a8 100644 --- a/app/observability.py +++ b/app/observability.py @@ -130,6 +130,13 @@ def payload(self, value, source): self.output() +def observe_stream_mode(mode): + """Attach one allowlisted request marker without consuming bounded attempt diagnostics.""" + observation = _current.get() + if observation is not None and mode in ("compatible", "realtime"): + observation.record["stream_mode"] = mode + + def observe_route(public_model, upstream_model, profile, credential): observation = _current.get() if observation is not None: diff --git a/app/settings.py b/app/settings.py index 4fbbf0d..4277280 100644 --- a/app/settings.py +++ b/app/settings.py @@ -87,6 +87,8 @@ def _item(default, type_, label, *, mode="hot", env=None, minimum=None, maximum= env="CODEBUDDY2API_MAX_INFLIGHT_PER_ACCOUNT", minimum=0, maximum=10000), "request_context_mode": _item("legacy", "string", "请求上下文模式", env="CODEBUDDY2API_REQUEST_CONTEXT_MODE", choices=["legacy", "scoped"]), + "stream_mode": _item("compatible", "string", "流式模式（实时模式不重生成工具参数）", + env="CODEBUDDY2API_STREAM_MODE", choices=["compatible", "realtime"]), "audit_max_bytes": _item(256 * 1024 * 1024, "integer", "审计明细预算", minimum=1024**2, maximum=1024**4), "audit_retention_days": _item(30, "integer", "审计明细保留天数", minimum=1, maximum=36500), "audit_diagnostic_bytes": _item(8192, "integer", "失败诊断最大字节", minimum=0, maximum=8192), diff --git a/app/upstream_io.py b/app/upstream_io.py index a9d4c4d..4d7e7d0 100644 --- a/app/upstream_io.py +++ b/app/upstream_io.py @@ -32,6 +32,118 @@ def __init__(self, status, raw, *, retry_after=None): MAX_RETRY_AFTER = 86400 +MAX_TOOL_CALLS = 256 + + +class StreamOutputBudget: + """Account for each logical output fragment once across realtime validators and adapters.""" + + def __init__(self, max_bytes: int = 0): + self.max_bytes = max(0, int(max_bytes or 0)) + self.used_bytes = 0 + + def charge(self, size: int) -> None: + if not self.max_bytes: + return + used = self.used_bytes + max(0, int(size)) + if used > self.max_bytes: + raise UpstreamResponseError(502, json.dumps({"error": { + "message": f"upstream response exceeds the {self.max_bytes}-byte collection budget", + "type": "upstream_error", "code": "response_too_large"}}).encode()) + self.used_bytes = used + + def charge_text(self, value: str) -> None: + self.charge(len(value.encode("utf-8"))) + + +def new_tool_state() -> dict: + """Return the state shared by the accumulator and protocol adapters.""" + return { + "id": None, + "name": None, + "arguments": "", + "identity_complete": False, + "identity_emitted": False, + "_argument_phase": False, + } + + +def _tool_identity_ready(state: dict, declared_names=None, *, terminal: bool = False) -> bool: + """Use the argument phase or terminal marker as the metadata boundary, not string shapes.""" + identity = (state.get("id"), state.get("name")) + if not all(isinstance(value, str) and value for value in identity): + return False + if not terminal and not state.get("_argument_phase"): + return False + names = {value for value in (declared_names or ()) if isinstance(value, str) and value} + return not names or identity[1] in names + + +def tool_identity_complete(state: dict, declared_names=None, *, terminal: bool = False) -> bool: + """Refresh and return whether a tool identity is safe to expose.""" + ready = _tool_identity_ready(state, declared_names, terminal=terminal) + state["identity_complete"] = ready + return ready + + +def seal_tool_identity(state: dict, declared_names=None) -> bool: + """Seal all currently retained identity fields at a terminal boundary.""" + state["_terminal"] = True + return tool_identity_complete(state, declared_names, terminal=True) + + +def merge_tool_call_delta(state: dict, tool: dict, *, declared_names=None, charge=None) -> dict: + """Append Chat metadata fragments without deduplication; keep emitted identities stable.""" + if not isinstance(state, dict): + raise ValueError("tool state") + if not isinstance(tool, dict): + raise ValueError("tool") + state.setdefault("id", None) + state.setdefault("name", None) + state.setdefault("arguments", "") + state.setdefault("identity_complete", False) + state.setdefault("identity_emitted", False) + state.setdefault("_argument_phase", False) + function = tool.get("function", {}) + if not isinstance(function, dict): + raise ValueError("function") + + def merge_field(key: str, piece) -> None: + if piece is None: + return + if not isinstance(piece, str): + raise ValueError(key) + if not piece: + return + current = state.get(key) or "" + if state.get("identity_emitted"): + if piece == current: + return + raise ValueError("conflicting tool identity") + # A repeated prefix can be a real delta ("tes" + "t"), not a retransmission. + if charge is not None: + charge(piece) + state[key] = current + piece + + merge_field("id", tool.get("id")) + merge_field("name", function.get("name")) + piece = function.get("arguments") or "" + if not isinstance(piece, str): + raise ValueError("arguments") + if piece: + if charge is not None: + charge(piece) + state["arguments"] = (state.get("arguments") or "") + piece + state["_argument_phase"] = True + tool_identity_complete(state, declared_names, terminal=bool(state.get("_terminal"))) + return state + + +def seal_tool_identities(states, declared_names=None) -> None: + """Seal every state in a tracker at the upstream terminal boundary.""" + for state in (states or {}).values(): + if isinstance(state, dict): + seal_tool_identity(state, declared_names) def parse_retry_after(value, *, now=None) -> int | None: @@ -57,9 +169,14 @@ def parse_retry_after(value, *, now=None) -> int | None: class ChatSSEAccumulator: """Collect Chat SSE and reject error events, empty output and incomplete streams.""" - def __init__(self, *, collect=True, max_collect_bytes: int = 0): + def __init__(self, *, collect=True, max_collect_bytes: int = 0, retain_tools=None, + budget: StreamOutputBudget | None = None, declared_names=None): self.collect = collect - self.max_collect_bytes = max(0, int(max_collect_bytes or 0)) + self.retain_tools = collect if retain_tools is None else bool(retain_tools) + self.budget = budget if budget is not None else StreamOutputBudget(max_collect_bytes) + self.max_collect_bytes = self.budget.max_bytes + self.declared_names = frozenset( + value for value in (declared_names or ()) if isinstance(value, str) and value) self.collected_bytes = 0 self.content = [] self.reasoning = [] @@ -71,12 +188,25 @@ def __init__(self, *, collect=True, max_collect_bytes: int = 0): def feed_line(self, line): line = line.strip() - if self.done or not line.startswith("data:"): + if not line or not line.startswith("data:"): return data = line[5:].strip() if data == "[DONE]": self.done = True return + if self.done: + # Preserve the historical tolerance for ignored malformed trailers, + # while rejecting a valid non-empty output frame after [DONE]. + try: + trailing = json.loads(data) + except ValueError: + return + if (isinstance(trailing, dict) and "choices" not in trailing + and trailing.get("error") is None + and not any(trailing.get(key) for key in ("content", "reasoning_content", "refusal", + "tool_calls", "function_call"))): + return + raise httpx.RemoteProtocolError("output after [DONE]") try: chunk = json.loads(data) except ValueError: @@ -119,7 +249,7 @@ def _consume_chunk(self, chunk): if choice.get("finish_reason") is not None and not isinstance(choice["finish_reason"], str): raise ValueError("finish_reason") self.saw_choice = True - self.finish_reason = choice.get("finish_reason") or self.finish_reason + finish_reason = choice.get("finish_reason") or None delta = choice.get("delta", {}) if not isinstance(delta, dict): raise ValueError("delta") @@ -130,21 +260,36 @@ def _consume_chunk(self, chunk): self.saw_output = True if "tool_calls" in delta and not isinstance(delta["tool_calls"], list): raise ValueError("tool_calls") + if self.finish_reason is not None: + has_output = (any(delta.get(key) for key in ("content", "reasoning_content", "refusal")) + or bool(delta.get("tool_calls")) or bool(delta.get("function_call"))) + if has_output: + raise ValueError("output after finish_reason") + if finish_reason is not None and finish_reason != self.finish_reason: + raise ValueError("changed finish_reason") + if finish_reason is not None: + self.finish_reason = finish_reason if self.collect: for key in ("content", "reasoning_content", "refusal"): if delta.get(key): - getattr(self, key if key != "reasoning_content" else "reasoning").append(delta[key]) self._charge(len(delta[key].encode("utf-8"))) + getattr(self, key if key != "reasoning_content" else "reasoning").append(delta[key]) for tool in delta.get("tool_calls") or []: if not isinstance(tool, dict): raise ValueError("tool") - index = tool.get("index", 0) - if isinstance(index, bool) or not isinstance(index, int) or index < 0: + idx = tool.get("index", 0) + if isinstance(idx, bool) or not isinstance(idx, int) or idx < 0: raise ValueError("tool index") + if (self.retain_tools and not self.collect + and tool.get("type") is not None and tool.get("type") != "function"): + raise ValueError("tool type") if tool.get("id") is not None and not isinstance(tool["id"], str): raise ValueError("tool id") - slot = self.tools.setdefault(index, {"id": None, "name": None, "arguments": ""}) - slot["id"] = tool.get("id") or slot["id"] + if idx not in self.tools: + if self.retain_tools and not self.collect and len(self.tools) >= MAX_TOOL_CALLS: + raise ValueError("too many tool calls") + self.tools[idx] = new_tool_state() + slot = self.tools[idx] function = tool.get("function", {}) if not isinstance(function, dict): raise ValueError("function") @@ -153,26 +298,43 @@ def _consume_chunk(self, chunk): raise ValueError(key) if tool.get("id") or function.get("name") or function.get("arguments"): self.saw_output = True - slot["name"] = function.get("name") or slot["name"] - if self.collect: + if self.retain_tools and not self.collect: + merge_tool_call_delta( + slot, tool, declared_names=self.declared_names, charge=self._charge_text) + else: + if tool.get("id"): + slot["id"] = tool["id"] + if function.get("name"): + slot["name"] = function["name"] piece = function.get("arguments") or "" - slot["arguments"] += piece - self._charge(len(piece.encode("utf-8"))) + if piece and self.collect: + self._charge(len(piece.encode("utf-8"))) + slot["arguments"] += piece + slot["identity_complete"] = tool_identity_complete( + slot, self.declared_names, terminal=bool(slot.get("_terminal"))) self.filter_detector.feed(delta, choice.get("finish_reason")) + if finish_reason is not None: + seal_tool_identities(self.tools, self.declared_names) + + def _charge_text(self, value: str) -> None: + self._charge(len(value.encode("utf-8"))) def _charge(self, size: int): - """Fail when collected bytes exceed the configured memory budget.""" - if not self.collect or not self.max_collect_bytes: + """Fail when retained output metadata exceeds the configured shared memory budget.""" + if not self.max_collect_bytes or (not self.collect and not self.retain_tools): return - self.collected_bytes += size - if self.collected_bytes > self.max_collect_bytes: - raise UpstreamResponseError(502, json.dumps({"error": { - "message": f"upstream response exceeds the {self.max_collect_bytes}-byte collection budget", - "type": "upstream_error", "code": "response_too_large"}}).encode()) + self.budget.charge(size) + self.collected_bytes = self.budget.used_bytes + + def validated_tool_calls(self): + return [{"id": value["id"], "type": "function", + "function": {"name": value["name"], "arguments": value["arguments"]}} + for _, value in sorted(self.tools.items())] def result(self): - if not self.saw_choice or not (self.done or self.finish_reason): + if not self.saw_choice or not (self.done or self.finish_reason is not None): raise httpx.RemoteProtocolError("Upstream SSE ended without a completion marker") + seal_tool_identities(self.tools, self.declared_names) if not self.saw_output: if self.finish_reason in ("content_filter", "content-filter", "refusal"): raw = {"error": {"type": "upstream_error", "code": self.finish_reason, @@ -181,9 +343,7 @@ def result(self): raw = {"error": {"type": "upstream_error", "code": "empty_response", "message": "Upstream SSE ended without output"}} raise UpstreamResponseError(502, json.dumps(raw).encode("utf-8")) - tools = [{"id": value["id"], "type": "function", - "function": {"name": value["name"], "arguments": value["arguments"]}} - for _, value in sorted(self.tools.items())] or None + tools = self.validated_tool_calls() or None return {"content": "".join(self.content), "reasoning_content": "".join(self.reasoning) or None, "refusal": "".join(self.refusal) or None, "tool_calls": tools, "finish_reason": self.finish_reason, diff --git a/converter.py b/converter.py index 647f684..a435c70 100644 --- a/converter.py +++ b/converter.py @@ -18,6 +18,7 @@ import uuid from collections import OrderedDict from contextlib import asynccontextmanager, nullcontext +from dataclasses import dataclass from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Optional @@ -54,13 +55,14 @@ def desensitize_body(body, roles=("system",), desensitize_harness_user=False, from app.model_blocks import ModelBlocks from app.usage_snapshots import UsageSnapshots from app.client_hangup import ClientHungUp, await_or_hangup -from app.observability import (AuditMiddleware, observe_recovery, observe_route, +from app.observability import (AuditMiddleware, observe_recovery, observe_route, observe_stream_mode, observe_usage, observe_attempt, observe_failure, observe_failure_seq) from app.credential_io import (CredentialFileError, read_import_file, atomic_write_credential, credential_file_lock) -from app.upstream_io import (ChatSSEAccumulator, UpstreamHTTPError, UpstreamResponseError, - open_backend_stream, parse_retry_after, read_bounded_error) +from app.upstream_io import (ChatSSEAccumulator, StreamOutputBudget, UpstreamHTTPError, + UpstreamResponseError, open_backend_stream, parse_retry_after, + read_bounded_error) from app.inference_resources import (AccountCapacity, InferenceResourcesMiddleware, inference_lifespan, request_resources, release_credential) from app.request_context import SessionIdentifierError, current_context @@ -93,6 +95,38 @@ def desensitize_body(body, roles=("system",), desensitize_harness_user=False, CBC_VERSION = CLI_VERSION USER_AGENT = CLI_USER_AGENT +_STREAM_MODES = ("compatible", "realtime") +_REQUEST_POLICY_KEY = object() # Process-local object key; never serializable or client injectable. + + +@dataclass(frozen=True) +class _StreamRequestPolicy: + mode: str + aggregate: bool + max_collect_bytes: int + + @property + def realtime(self): + return self.mode == "realtime" + + +def _snapshot_stream_policy(protocol: str, body: dict) -> _StreamRequestPolicy: + """Freeze hot streaming choices and memory policy for one request and all failovers.""" + mode = CONFIG.get("stream_mode", "compatible") + if mode not in _STREAM_MODES: + raise ValueError("invalid stream mode") + compatible_aggregate = protocol == "responses" or bool(body.get("tools")) + return _StreamRequestPolicy(mode, mode == "compatible" and compatible_aggregate, + max(0, int(CONFIG.get("max_collect_bytes", 0) or 0))) + + +def _body_with_stream_policy(body: dict, policy: _StreamRequestPolicy) -> dict: + """Attach a process-local snapshot without exposing an upstream payload override.""" + routed = dict(body) + routed[_REQUEST_POLICY_KEY] = policy + return routed + + # --------------------------------------------------------------------------- # Platform-specific credential directories # --------------------------------------------------------------------------- @@ -1783,7 +1817,7 @@ async def _protocol_http_exception(request: Request, exc: HTTPException): "max_collect_bytes": 8 * 1024 * 1024, "max_concurrent": 64, "upstream_keepalive": False, "max_inflight_per_account": 0, "request_context_mode": "legacy", - "model_capability_guard": True, + "stream_mode": "compatible", "model_capability_guard": True, "failover_max": 0, # Credential failovers allowed before the first response byte "retry_write_timeout": False, # Opt-in replay after incomplete writes "usage_daily": None, # Usage aggregated by date and model @@ -2761,6 +2795,9 @@ async def chat_completions(request: Request, client_wants_stream = _client_wants_stream(payload) body = {k: payload[k] for k in PASSTHROUGH_BODY_KEYS if k in payload} body = await run_in_threadpool(_prepare_chat_body, body, session_payload=payload) + stream_policy = (_snapshot_stream_policy("chat", body) if client_wants_stream else None) + if stream_policy is not None: + observe_stream_mode(stream_policy.mode) # Record request metadata. model_name = payload.get("model", "auto") @@ -2777,9 +2814,10 @@ async def chat_completions(request: Request, _log_json(f"[{rid}] REQUEST BODY (发往后端，预览)", body) t0 = time.time() - if client_wants_stream: + if stream_policy is not None: def attempt(routed, cred, headers, url): - return _stream_upstream(url, headers, routed, model_name, t0, rid, cred=cred) + return _stream_upstream(url, headers, _body_with_stream_policy(routed, stream_policy), + model_name, t0, rid, cred=cred) return _routed_stream(payload, prepared, model_name, rid, t0, attempt, body, cred, headers, url) @@ -2869,41 +2907,96 @@ async def _collect_stream(response: httpx.Response, *, accumulator=None) -> dict def _tool_choice_satisfied(tool_calls, body): + calls = tool_calls or [] choice = body.get("tool_choice") if choice == "none": - return not tool_calls - if choice != "required": - return True - names = {tool.get("function", {}).get("name") for tool in body.get("tools", []) - if isinstance(tool, dict) and isinstance(tool.get("function"), dict)} - return bool(tool_calls) and all(call.get("function", {}).get("name") in names for call in tool_calls) + return not calls + if choice == "required": + names = _declared_tool_names(body) + return bool(calls) and all(call.get("function", {}).get("name") in names for call in calls) + if isinstance(choice, dict): + name = (choice.get("function") or {}).get("name") if isinstance(choice.get("function"), dict) else None + return bool(calls) and all(call.get("function", {}).get("name") == name for call in calls) + return True -def _tool_calls_healthy(tool_calls, body: dict | None = None) -> bool: - """Validate tool names and require arguments to encode a JSON object.""" - if not tool_calls: - return True - names = {tool.get("function", {}).get("name") for tool in (body or {}).get("tools", []) - if isinstance(tool, dict) and isinstance(tool.get("function"), dict)} if body is not None else None - for tc in tool_calls: - if not isinstance(tc.get("id"), str) or not tc["id"].strip(): +def _declared_tool_names(body: dict) -> frozenset[str]: + """Extract declared function names for conservative realtime identity boundaries.""" + names = set() + for tool in body.get("tools") or []: + if not isinstance(tool, dict): + continue + function = tool.get("function") + if isinstance(function, dict) and isinstance(function.get("name"), str): + names.add(function["name"]) + elif isinstance(tool.get("name"), str): + names.add(tool["name"]) + return frozenset(name for name in names if name) + + +def _tool_calls_healthy(tool_calls, body: dict | None = None, *, require_declarations=False) -> bool: + """Validate unique calls, declared names and JSON-object arguments.""" + calls = tool_calls or [] + names = set(_declared_tool_names(body)) if body is not None else None + seen_ids = set() + for tc in calls: + if not isinstance(tc, dict): + return False + call_id = tc.get("id") + if (not isinstance(call_id, str) or not call_id.strip() + or (require_declarations and call_id in seen_ids) + or (require_declarations and tc.get("type") != "function")): return False + if not require_declarations: + seen_ids.add(call_id) + seen_ids.add(call_id) fn = tc.get("function") or {} + if not isinstance(fn, dict): + return False name = fn.get("name") or "" - if not name.strip() or not (fn.get("arguments") or "").strip(): + arguments = fn.get("arguments") or "" + if not isinstance(name, str) or not name.strip() or not isinstance(arguments, str) or not arguments.strip(): return False - # Valid JSON must still be an object; check names only when tools were declared. - if names and name not in names: + if names is not None and ((require_declarations or names) and name not in names): return False try: - arguments = json.loads(fn.get("arguments") or "") - except Exception: + decoded = json.loads(arguments) + except (TypeError, ValueError, UnicodeError, RecursionError): return False - if not isinstance(arguments, dict): + if not isinstance(decoded, dict): return False + if (require_declarations and body is not None + and body.get("parallel_tool_calls") is False and len(calls) > 1): + return False return True +def _validate_realtime_tools(tool_calls, body: dict, finish_reason=None, *, filtered=False): + """Validate terminal state without regenerating or replaying the request. + + Explicit refusal/filter and length results are incomplete upstream outcomes, + not malformed tool calls. Any tool bytes that are present still have to be + structurally healthy, but a required/named choice need not be satisfied when + the upstream legitimately stopped before producing a call. + """ + if not _tool_calls_healthy(tool_calls, body, require_declarations=True): + healthy = False + elif finish_reason is None: + # Realtime success terminals require an explicit upstream completion + # marker; [DONE] alone remains compatible only on the legacy path. + healthy = False + elif filtered or finish_reason in ("length", "content_filter", "content-filter", "refusal"): + healthy = True + else: + healthy = (_tool_choice_satisfied(tool_calls, body) + and (finish_reason == "tool_calls" if tool_calls else finish_reason != "tool_calls")) + if healthy: + return + raw = {"error": {"message": "Invalid upstream tool_calls", "type": "upstream_error", + "code": "invalid_tool_calls"}} + raise UpstreamResponseError(502, json.dumps(raw).encode("utf-8")) + + def _merge_chat_sse_text(text: str) -> dict: """Use the shared SSE accumulator for collected text responses.""" accumulator = ChatSSEAccumulator(max_collect_bytes=CONFIG.get("max_collect_bytes", 0)) @@ -3051,12 +3144,20 @@ def _hungup_response(rid, model_name, t0): return Response(status_code=204) -async def _fetch_checked_chat(url, headers, body, model_name, rid, cred=None, *, filter_retry=False): - """Collect and validate replies with bounded tool repair and one eligible filter fallback.""" +async def _fetch_checked_chat(url, headers, body, model_name, rid, cred=None, *, filter_retry=False, + max_collect_bytes=None): + """Collect and validate replies with bounded tool repair and one eligible filter fallback. + + ``max_collect_bytes`` is an optional frozen request-policy value. Streaming + callers pass it explicitly so hot configuration changes cannot enlarge a + collection or one of its repair attempts after the request has started. + """ tool_attempt = 0 filter_retried = False + collection_limit = (CONFIG.get("max_collect_bytes", 0) if max_collect_bytes is None + else max_collect_bytes) while True: - accumulator = ChatSSEAccumulator(max_collect_bytes=CONFIG.get("max_collect_bytes", 0)) + accumulator = ChatSSEAccumulator(max_collect_bytes=collection_limit) rejection = None async with _backend_stream(url, headers, body, rid=rid, model_name=model_name) as response: if response.status_code != 200: @@ -3113,37 +3214,62 @@ async def _fetch_checked_chat(url, headers, body, model_name, rid, cred=None, *, total_tokens=discarded.get("total_tokens")) _log(f"[{rid}] tool_calls 损坏，重试 {tool_attempt}/{budget} | {model_name}") -async def _chat_sse_lines(url, headers, body, model_name, t0, rid, cred=None, *, aggregate=False): +async def _chat_sse_lines(url, headers, body, model_name, t0, rid, cred=None, *, + policy=None, tracker=None, state=None): """Yield validated Chat SSE with bounded filter detection and no streaming filter retries.""" - if aggregate: - result = await _fetch_checked_chat(url, headers, body, model_name, rid, cred) + policy = policy or _snapshot_stream_policy("chat", body) + if policy.aggregate: + result = await _fetch_checked_chat( + url, headers, body, model_name, rid, cred, + max_collect_bytes=policy.max_collect_bytes) for line in _chat_result_to_sse_lines(_completion_to_merged(result)): yield line yield "" _log_finish(model_name, t0, result, rid) return - tracker = ChatSSEAccumulator(collect=False) + if tracker is None: + tracker = ChatSSEAccumulator( + collect=False, retain_tools=policy.realtime, + budget=StreamOutputBudget(policy.max_collect_bytes)) + if state is not None: + state["tracker"] = tracker + + def completed(): + merged = tracker.result() + if policy.realtime: + _validate_realtime_tools( + merged.get("tool_calls"), body, merged.get("finish_reason"), + filtered=tracker.filter_detector.detected) + if state is not None: + state["merged"] = merged + return merged + preview = bytearray() - budget = CONFIG["log_body_limit"] if CONFIG.get("log_path") else 0 - async with _backend_stream(url, headers, body, rid=rid, model_name=model_name) as response: - if response.status_code != 200: - _check_upstream_status(response.status_code, await read_bounded_error(response), cred, body.get("model"), - headers=response.headers) - else: - _note_cred_model_ok(cred, body.get("model")) - async for line in response.aiter_lines(): - tracker.feed_line(line) - if tracker.done or tracker.finish_reason: - tracker.result() # Validate completion before emitting a success marker. - remaining = budget - len(preview) - if remaining > 0: - preview.extend((line[:remaining] + "\n").encode("utf-8")[:remaining]) - yield line - if tracker.done: - yield "" - break - merged = tracker.result() - observe_usage(merged.get("usage") or {}) + # Realtime output is never copied into the retired raw-text preview log. + budget = 0 if policy.realtime else ( + CONFIG["log_body_limit"] if CONFIG.get("log_path") else 0) + try: + async with _backend_stream(url, headers, body, rid=rid, model_name=model_name) as response: + if response.status_code != 200: + _check_upstream_status(response.status_code, await read_bounded_error(response), cred, body.get("model"), + headers=response.headers) + else: + _note_cred_model_ok(cred, body.get("model")) + async for line in response.aiter_lines(): + tracker.feed_line(line) + if tracker.done or tracker.finish_reason: + completed() # Never expose a success marker before terminal validation. + remaining = budget - len(preview) + if remaining > 0: + preview.extend((line[:remaining] + "\n").encode("utf-8")[:remaining]) + yield line + if tracker.done: + yield "" + break + merged = completed() + finally: + if tracker.usage: + observe_usage(tracker.usage) if tracker.filter_detector.detected: _note_content_filter(rid, model_name, final=True) return @@ -3154,16 +3280,23 @@ async def _chat_sse_lines(url, headers, body, model_name, t0, rid, cred=None, *, async def _stream_upstream(url: str, headers: dict, body: dict, model_name: str = "?", t0: float = 0.0, rid: str = "", cred=None): + policy = body.pop(_REQUEST_POLICY_KEY, None) or _snapshot_stream_policy("chat", body) sent = False + upstream = _chat_sse_lines(url, headers, body, model_name, t0, rid, cred, policy=policy) try: - async for line in _chat_sse_lines(url, headers, body, model_name, t0, rid, cred, aggregate=bool(body.get("tools"))): - sent = True - yield (_public_sse_line(line, model_name) + "\n").encode("utf-8") - except (httpx.HTTPError, UpstreamResponseError) as error: - if not sent: - raise # Preserve the HTTP error while no response bytes have been sent. - status, raw = _upstream_failure(error, model_name, t0, rid) - yield _err_event(raw, status) + try: + async for line in upstream: + sent = True + yield (_public_sse_line(line, model_name) + "\n").encode("utf-8") + except (httpx.HTTPError, UpstreamResponseError) as error: + if not sent: + raise # Preserve the HTTP error while no response bytes have been sent. + status, raw = _upstream_failure(error, model_name, t0, rid) + yield _err_event(raw, status) + finally: + # The inner generator may be suspended at a yielded line when its + # consumer reports an adapter error or a downstream disconnect. + await _close_stream(upstream) @@ -3464,6 +3597,9 @@ async def create_response(request: Request, chat_body = await run_in_threadpool(_prepare_chat_body, chat_body) client_wants_stream = _client_wants_stream(payload) + stream_policy = (_snapshot_stream_policy("responses", chat_body) if client_wants_stream else None) + if stream_policy is not None: + observe_stream_mode(stream_policy.mode) model_name = payload.get("model", "auto") rid = _request_id() _log(f"[{rid}] ▶ RESPONSES {model_name} | stream={client_wants_stream} | input_items={len(payload.get('input', []))}") @@ -3484,9 +3620,10 @@ async def create_response(request: Request, _log_json(f"[{rid}] RESPONSES → CHAT BODY (预览)", chat_body) t0 = time.time() - if client_wants_stream: + if stream_policy is not None: def attempt(routed, cred, headers, url): - return _stream_responses(url, headers, routed, model_name, t0, rid, cred=cred) + return _stream_responses(url, headers, _body_with_stream_policy(routed, stream_policy), + model_name, t0, rid, cred=cred) return _routed_stream(payload, prepared, model_name, rid, t0, attempt, chat_body, cred, headers, url) @@ -3521,35 +3658,81 @@ async def fetch(routed, cred, headers, url): async def _stream_adapted(url, headers, body, model_name, t0, rid, cred=None, *, anthropic=False): """Map protocol events while sharing connection, aggregation and failure handling.""" - converter = (AnthropicStreamConverter(model=model_name) if anthropic else ResponsesStreamConverter(model=model_name, parallel_tool_calls=body.get("parallel_tool_calls", True))) + protocol = "messages" if anthropic else "responses" + policy = body.pop(_REQUEST_POLICY_KEY, None) or _snapshot_stream_policy(protocol, body) + state = {} + tracker = None + declared_names = _declared_tool_names(body) + if policy.realtime: + budget = StreamOutputBudget(policy.max_collect_bytes) + tracker = ChatSSEAccumulator(collect=False, retain_tools=True, budget=budget, + declared_names=declared_names) + converter = (AnthropicStreamConverter(model=model_name, realtime=True, budget=budget, + tool_states=tracker.tools, declared_names=declared_names) + if anthropic else + ResponsesStreamConverter(model=model_name, + parallel_tool_calls=body.get("parallel_tool_calls", True), + realtime=True, budget=budget, tool_states=tracker.tools, + declared_names=declared_names)) + else: + converter = (AnthropicStreamConverter(model=model_name) if anthropic else + ResponsesStreamConverter(model=model_name, + parallel_tool_calls=body.get("parallel_tool_calls", True))) sent = False + upstream = _chat_sse_lines( + url, headers, body, model_name, t0, rid, cred, + policy=policy, tracker=tracker, state=state) try: - async for line in _chat_sse_lines( - url, headers, body, model_name, t0, rid, cred, - aggregate=not anthropic or bool(body.get("tools"))): - events = converter.feed_line(_public_sse_line(line, model_name)) + try: + async for line in upstream: + events = converter.feed_line(_public_sse_line(line, model_name)) + if events: + sent = True + yield events.encode("utf-8") + if policy.realtime: + converter.set_validated_tools((state.get("merged") or {}).get("tool_calls")) + if tracker.filter_detector.detected: + converter.mark_content_filter() + events = converter.finish() if events: sent = True yield events.encode("utf-8") - events = converter.finish() - if events: - sent = True - yield events.encode("utf-8") - except (httpx.HTTPError, UpstreamResponseError) as error: - if not sent: - raise # Preserve the HTTP error before any response bytes are sent. - status, raw = _upstream_failure(error, model_name, t0, rid) - event = {"type": "error", "error": { - "message": sanitize_log_text(raw.decode("utf-8", "replace"), 512), - "type": "api_error" if anthropic else "upstream_error", "code": status}} - prefix = "event: error\n" if anthropic else "" - yield (prefix + f"data: {json.dumps(event, ensure_ascii=False)}\n\n").encode("utf-8") + except (httpx.HTTPError, UpstreamResponseError) as error: + if not sent: + raise # Preserve the HTTP error before any response bytes have been sent. + status, raw = _upstream_failure(error, model_name, t0, rid) + event = {"type": "error", "error": { + "message": sanitize_log_text(raw.decode("utf-8", "replace"), 512), + "type": "api_error" if anthropic else "upstream_error", "code": status}} + prefix = "event: error\n" if anthropic else "" + yield (prefix + f"data: {json.dumps(event, ensure_ascii=False)}\n\n").encode("utf-8") + except ValueError: + # Adapter metadata/append-only checks are protocol failures, not + # uncaught application errors after a response has opened. + failure = UpstreamResponseError(502, b'{"error":{"message":"Invalid upstream tool_calls",' + b'"type":"upstream_error","code":"invalid_tool_calls"}}') + if not sent: + raise failure from None + status, raw = _upstream_failure(failure, model_name, t0, rid) + event = {"type": "error", "error": { + "message": sanitize_log_text(raw.decode("utf-8", "replace"), 512), + "type": "api_error" if anthropic else "upstream_error", "code": status}} + prefix = "event: error\n" if anthropic else "" + yield (prefix + f"data: {json.dumps(event, ensure_ascii=False)}\n\n").encode("utf-8") + finally: + # Close the nested Chat generator explicitly; do not wait for GC after + # an adapter error, disconnect or cancellation. + await _close_stream(upstream) async def _stream_responses(url: str, headers: dict, body: dict, model_name: str = "?", t0: float = 0.0, rid: str = "", cred=None): - async for chunk in _stream_adapted(url, headers, body, model_name, t0, rid, cred): - yield chunk + stream = _stream_adapted(url, headers, body, model_name, t0, rid, cred) + try: + async for chunk in stream: + yield chunk + finally: + await _close_stream(stream) # --------------------------------------------------------------------------- @@ -3580,6 +3763,10 @@ async def create_message(request: Request, raise HTTPException(status_code=400, detail={"error": {"message": f"request conversion error: {e}", "type": "invalid_request_error"}}) chat_body = await run_in_threadpool(_prepare_chat_body, chat_body, session_payload=payload) + client_wants_stream = _client_wants_stream(payload) + stream_policy = (_snapshot_stream_policy("messages", chat_body) if client_wants_stream else None) + if stream_policy is not None: + observe_stream_mode(stream_policy.mode) model_name = payload.get("model", "auto") chat_messages = chat_body.get("messages", []) rid = _request_id() @@ -3590,21 +3777,26 @@ async def create_message(request: Request, _log_json(f"[{rid}] ANTHROPIC → CHAT BODY (预览)", chat_body) t0 = time.time() - if not _client_wants_stream(payload): + if not client_wants_stream: return await _nonstream_adapted(url, headers, chat_body, model_name, t0, rid, cred, anthropic=True, payload=payload, canonical=prepared, request=request) def attempt(routed, cred, headers, url): - return _stream_anthropic(url, headers, routed, model_name, t0, rid, cred=cred) + return _stream_anthropic(url, headers, _body_with_stream_policy(routed, stream_policy), + model_name, t0, rid, cred=cred) return _routed_stream(payload, prepared, model_name, rid, t0, attempt, chat_body, cred, headers, url) async def _stream_anthropic(url: str, headers: dict, body: dict, model_name: str = "?", t0: float = 0.0, rid: str = "", cred=None): - async for chunk in _stream_adapted(url, headers, body, model_name, t0, rid, cred, anthropic=True): - yield chunk + stream = _stream_adapted(url, headers, body, model_name, t0, rid, cred, anthropic=True) + try: + async for chunk in stream: + yield chunk + finally: + await _close_stream(stream) @app.post("/v1/messages/count_tokens") @@ -3822,6 +4014,9 @@ def main(): ap.add_argument("--request-context-mode", choices=("legacy", "scoped"), default=os.environ.get("CODEBUDDY2API_REQUEST_CONTEXT_MODE", "legacy"), help="请求上下文：legacy 保持旧会话头，scoped 启用显式会话与逐尝试追踪；默认 legacy") + ap.add_argument("--stream-mode", choices=_STREAM_MODES, + default=os.environ.get("CODEBUDDY2API_STREAM_MODE", "compatible"), + help="流式传输：compatible 保持兼容聚合策略，realtime 增量发送且不重生成工具参数；默认 compatible") ap.add_argument("--model-capability-guard", type=_boolean_arg, nargs="?", const=True, default=os.environ.get("CODEBUDDY2API_MODEL_CAPABILITY_GUARD", "true"), help="按账号模型声明预检图片、工具、思考和输出上限；false 仅关闭新增能力预检") @@ -3855,7 +4050,7 @@ def main(): for key in ("max_images", "image_policy", "max_request_bytes", "log_body_limit", "tool_call_max_retry", "max_inbound_bytes", "max_collect_bytes", "max_concurrent", "failover_max", "retry_write_timeout", "upstream_keepalive", "max_inflight_per_account", - "request_context_mode", "model_capability_guard", "admin_allowed_origins"): + "request_context_mode", "stream_mode", "model_capability_guard", "admin_allowed_origins"): CONFIG[key] = getattr(args, key) CONFIG["api_key"] = args.api_key CONFIG["desensitize"] = args.desensitize diff --git a/docker-compose.yml b/docker-compose.yml index 702c20d..4b89661 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,7 @@ services: CODEBUDDY2API_UPSTREAM_KEEPALIVE: CODEBUDDY2API_MAX_INFLIGHT_PER_ACCOUNT: CODEBUDDY2API_REQUEST_CONTEXT_MODE: + CODEBUDDY2API_STREAM_MODE: CODEBUDDY2API_MODEL_CAPABILITY_GUARD: CODEBUDDY2API_TOOL_CALL_MAX_RETRY: ${CODEBUDDY2API_TOOL_CALL_MAX_RETRY:-3} CODEBUDDY2API_FAILOVER_MAX: diff --git a/docs/advanced.md b/docs/advanced.md index 604ac66..53c9d41 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -32,7 +32,8 @@ Compose explicitly passes some environment variables and CLI flags, so deleting | `--image-policy` | `truncate` | Keep newest images; `error` rejects excess images with 413 | | `--tool-call-max-retry` | `3` | Extra generations after malformed tool calls (each consumes credits); `0` disables retries | | `--max-inbound-bytes` | `67108864` | Raw body limit for generation and token-count POSTs, before parsing (chunked included); other routes are not buffered; 413 beyond it | -| `--max-collect-bytes` | `8388608` | Total collection budget for aggregated output (content + reasoning + tool arguments); `response_too_large` beyond it; `0` disables | +| `--max-collect-bytes` | `8388608` | Total retained-output budget for aggregation and realtime validation (content + reasoning + tool arguments/metadata); `response_too_large` beyond it; `0` disables | +| `--stream-mode compatible\|realtime` | `compatible` | `compatible` preserves aggregate/replay behavior; `realtime` incrementally streams all three protocols and does not regenerate tool arguments | | `--max-concurrent` | `64` | Concurrency limit for the three generation endpoints only; excess requests get 503 with Retry-After; token counting is unaffected; `0` disables | | `--max-inflight-per-account` | `0` | Per-process, per-account in-flight client inference limit; `0` disables, full accounts return 503 | | `--upstream-keepalive [true/false]` | `false` | Bounded connection reuse isolated by official origin; requires restart | @@ -42,7 +43,7 @@ Compose explicitly passes some environment variables and CLI flags, so deleting | `--max-request-bytes` | `33554432` | Positive byte limit for the processed upstream JSON | | `--log-body-limit` | `65536` | Legacy text-preview option; text output is retired and SQLite diagnostics use their own budget | -Environment variables include `CODEBUDDY_AUTH_DIR`, `CODEBUDDY_IMPORT_DIR`, `CODEBUDDY2API_KEY`, `CODEBUDDY2API_ADMIN_CSRF`, `CODEBUDDY2API_ADMIN_ORIGINS`, `CODEBUDDY2API_KEEP_TOOL_METADATA`, `CODEBUDDY2API_LOG`, `CODEBUDDY2API_MAX_IMAGES`, `CODEBUDDY2API_IMAGE_POLICY`, `CODEBUDDY2API_MAX_REQUEST_BYTES`, `CODEBUDDY2API_LOG_BODY_LIMIT`, `CODEBUDDY2API_FAILOVER_MAX` and `CODEBUDDY2API_RETRY_WRITE_TIMEOUT`. See [deployment](deployment.md) for startup examples. +Environment variables include `CODEBUDDY_AUTH_DIR`, `CODEBUDDY_IMPORT_DIR`, `CODEBUDDY2API_KEY`, `CODEBUDDY2API_ADMIN_CSRF`, `CODEBUDDY2API_ADMIN_ORIGINS`, `CODEBUDDY2API_KEEP_TOOL_METADATA`, `CODEBUDDY2API_STREAM_MODE`, `CODEBUDDY2API_LOG`, `CODEBUDDY2API_MAX_IMAGES`, `CODEBUDDY2API_IMAGE_POLICY`, `CODEBUDDY2API_MAX_REQUEST_BYTES`, `CODEBUDDY2API_LOG_BODY_LIMIT`, `CODEBUDDY2API_FAILOVER_MAX` and `CODEBUDDY2API_RETRY_WRITE_TIMEOUT`. See [deployment](deployment.md) for startup examples. ### Tool metadata retention @@ -54,6 +55,46 @@ Off by default, preserving the existing policy: desensitization strips tool desc Use a source/image build and Compose configuration containing this feature; recreate containers after changing their environment. Retained descriptions may increase input tokens and content-filter rejections; compatibility across accounts/models is not guaranteed. Set `false` to restore the previous policy. This option does not restore other schema fields or deep nodes removed by existing Responses projection, nor relax the request-size budget. +### Streaming modes + +`stream_mode` defaults to `compatible`. Configure it as `--stream-mode compatible|realtime`, `CODEBUDDY2API_STREAM_MODE`, or the hot **Streaming mode / 实时模式** enum in the WebUI. Explicit CLI and environment sources lock the WebUI field. The selected value and `max_collect_bytes` are frozen when each request starts, so a hot change affects only later requests and never an in-flight failover path. There is no per-request override. + +- `compatible` preserves existing behavior: Responses streams aggregate first; Chat and Messages aggregate when tools are present and otherwise pass through upstream increments. Aggregated output is validated and replayed in fragments. Non-stream requests always use the validated aggregate path in either mode. +- `realtime` forwards reasoning, text, refusal and tool-argument increments for all three protocols. Responses assigns stable indexes when items start; Anthropic uses stable block indexes. Adapters buffer tools with missing identity until the argument phase or terminal marker, append metadata fragments without guessing from prefixes, and reject identity changes after an item starts. `max_collect_bytes` bounds retained UTF-8 output; `0` disables that limit. + +Realtime mode never regenerates malformed or incomplete tool arguments. Tool IDs, names, declared names, JSON-object arguments and `tool_choice` are checked at the terminal boundary before a success terminal is sent. In realtime, a tool-bearing completion must also carry the upstream `tool_calls` finish marker; a `stop` marker with tool calls is rejected, while compatible mode keeps its legacy acceptance behavior. Before any downstream byte, failures retain the upstream HTTP error and existing bounded pre-response failover rules. After any byte, malformed tools, disconnects, stream errors and budget overflow produce a protocol error terminal without credential replay or switching; valid `length`, refusal and content-filter results keep their native protocol distinctions (Responses reports truncation/filtering as `incomplete`, never `completed`) and are not regenerated. Clients must therefore accept partial output followed by an error rather than assuming every opened SSE stream completes successfully. Audit records retain only an allowlisted `stream_mode` marker plus available upstream usage. + +For runtime fallback, select `compatible` to restore aggregate streaming and tool-argument repair without changing saved state. Before running older source, remove the new CLI/environment option, stop the gateway and back up the **current** data directory, including its SQLite/WAL/SHM generation. Do not restore a stale pre-upgrade database: that could roll back newer claims, sessions, revocations and account state. The following offline procedure writes only the removal of `settings.stream_mode` and a revision increment, then checks integrity; all other settings and tables remain intact. Never run it against a live database or mix SQLite generations. + +```sh +python3 - /path/to/control.sqlite3 <<'PY' +import json, sqlite3, sys +con = sqlite3.connect(sys.argv[1]) +try: + con.execute("BEGIN IMMEDIATE") + row = con.execute("SELECT revision,payload FROM control WHERE id=1").fetchone() + if row is None: + raise SystemExit("missing control row") + revision, payload = row + data = json.loads(payload) + if set(data) != {"settings", "models", "credentials"} or not isinstance(data["settings"], dict): + raise SystemExit("unexpected control payload") + if "stream_mode" not in data["settings"]: + raise SystemExit("stream_mode is absent; no write needed") + del data["settings"]["stream_mode"] + con.execute("UPDATE control SET revision=?,payload=? WHERE id=1", + (revision + 1, json.dumps(data, ensure_ascii=False, allow_nan=False))) + con.commit() + if con.execute("PRAGMA integrity_check").fetchone()[0] != "ok": + raise SystemExit("integrity check failed") +finally: + con.close() +print("ok") +PY +``` + +Older strict setting validators reject the unknown saved key, so changing only its value does not make old source compatible. This procedure is intentionally offline and operator-scoped; do not perform it on production without a current backup and a stopped service. + ### Connection reuse and account capacity Both settings are available in the WebUI; their environment variables are `CODEBUDDY2API_UPSTREAM_KEEPALIVE` and `CODEBUDDY2API_MAX_INFLIGHT_PER_ACCOUNT`. Unset variables leave Compose settings unlocked. Connection reuse defaults to off; when enabled, each official origin permits 64 connections with 16 idle connections and a 30-second keepalive expiry. Authentication is request-scoped, upstream cookies are not stored, and shutdown closes the pools. Proxy environment, timeouts and replay rules are unchanged; disable and restart to restore fresh connections. @@ -186,7 +227,7 @@ Both international profiles merge image-bearing consecutive `user` runs only aft - Images count across all history and tool results, including duplicates, in message/content array order. The default keeps the newest 16, removing only excess images while retaining text and message structure; emptied image content receives a text placeholder. - `--image-policy error` returns local `413 / too_many_images`. JSON still over budget after processing returns `413 / request_too_large`, without further text truncation to fit the limit. - Image count does not guarantee acceptable individual image sizes or model vision support. URL/base64 images can be converted; Responses image `file_id` is unsupported. -- When `stream` is omitted all three endpoints follow the protocol default and return a complete JSON response; `stream` must be a boolean. Streaming Responses and Chat/Messages with tools aggregate and validate before emitting SSE; not every path forwards tokens in real time. +- With `stream_mode=compatible` (the default), streaming Responses and Chat/Messages with tools aggregate and validate before emitting SSE; Chat/Messages without tools pass through upstream increments. `realtime` streams all three incrementally, while omitted/non-stream requests remain complete validated JSON. - Inference errors follow the client protocol: OpenAI routes return a top-level `error` object and Messages returns `{"type": "error", ...}`. Status codes are retained; errors after streaming starts are reported through SSE without replay. - Valid upstream `Retry-After` values (0–86400 seconds or equivalent HTTP dates) are returned as seconds before streaming starts; 429 only cools the selected account/model. Invalid or expired values fall back to the body's reset time or 600 seconds. Pool-generated 429 responses include the remaining wait. - Chat and Responses preserve an explicit client `prompt_cache_key` without generating one; cache hits and savings depend on the upstream. @@ -221,7 +262,7 @@ Both international profiles merge image-bearing consecutive `user` runs only aft | Streaming request fails before the first byte | Reported with the real HTTP status, exactly like `stream=false`. A 200 carrying only an in-band `error` event is read by clients as an empty answer, so the session ends silently while the audit log records a success | | Credential failover (`--failover-max`) | Off by default. When enabled, a failure before any byte reached the client is retried on another credential up to N times and audited as `success` with a `failover_recovered` marker. Qualifying failures: upstream HTTP 401/403/429/502/503/504 rejections and bodies the upstream provably never received (`ConnectError`/`ConnectTimeout`). Content-filter rejections, 502s from an already-open stream, read timeouts and protocol errors are never replayed; without another credential the original status surfaces. Billing note: 401/403/429/503 and transport failures happen at admission and cannot be billed; a 502/504 may already have been billed upstream, but its result never reached the client, so refusing to replay recovers no credit — it only turns a paid-for attempt into a broken session. Such replays are tagged `上游可能已处理该请求` in the log for reconciliation | | Write-timeout replay (`--retry-write-timeout`) | Off by default. A write timeout proves the body was not fully sent, not that the upstream ignored the bytes it received, so it stays excluded from connect retry and failover until enabled. Long cross-border sessions fail here more often than in the handshake; enable only when the upstream is confirmed not to bill partial bodies. These replays carry the same `上游可能已处理该请求` log tag | -| Malformed tool calls | Aggregate validation permits up to `--tool-call-max-retry` (default 3) additional generations, each consuming credits and recorded with its usage in the attempt details; exhaustion returns an error | +| Malformed tool calls | Compatible aggregate validation permits up to `--tool-call-max-retry` (default 3) additional generations, each consuming credits and recorded with its usage in the attempt details; exhaustion returns an error. Realtime mode never regenerates: it reports a protocol error before a success terminal | | Empty or truncated upstream stream | No valid output, a missing end marker or an error is not reported as success | | Content-filter rejection | With desensitization and `--no-compact`, a complete non-streaming filter-only rejection may receive one shorter-template retry on the same account. No streaming filter retry, circuit opening or account rotation | | Slow responses | Inspect timing and failed attempts in the WebUI, then choose a faster model supported by the account | @@ -229,4 +270,4 @@ Both international profiles merge image-bearing consecutive `user` runs only aft ## Downgrades and rollback -Feature switches hold no hidden state: disabling a guard or mode stops it for new requests, and reverting source restores previous behavior. The exceptions are persisted settings and automation state: `control.sqlite3` stores WebUI settings, model rules and reward reservations, and older code rejects unknown fields. Before downgrading source, remove newly added startup options and restore a control-store backup from before the upgrade, including its WAL/SHM files without mixing. Rollback never undoes completed upstream check-ins, claims or travel dispatches. +Feature switches hold no hidden state: disabling a guard or mode stops it for new requests, and reverting source restores previous behavior. The exceptions are persisted settings and automation state: `control.sqlite3` stores WebUI settings, model rules and reward reservations, and older code rejects unknown fields. Before downgrading source, remove newly added startup options, stop the service, back up the **current** data directory, and use the narrowly scoped offline `settings.stream_mode` removal procedure above with a revision increment and integrity check. Do not restore a pre-upgrade database or mix WAL/SHM generations: doing so could roll back newer claims, sessions, revocations and account state. Rollback never undoes completed upstream check-ins, claims or travel dispatches. diff --git a/docs/advanced.zh-CN.md b/docs/advanced.zh-CN.md index 2a09e06..4b8893c 100644 --- a/docs/advanced.zh-CN.md +++ b/docs/advanced.zh-CN.md @@ -32,7 +32,8 @@ Compose 会显式传入部分环境变量及 CLI 参数，删除 `.env` 中的 | `--image-policy` | `truncate` | 保留最新图片；设为 `error` 时超限返回 413 | | `--tool-call-max-retry` | `3` | 工具参数损坏时的额外生成上限（每次都消耗额度）；`0` 不重试 | | `--max-inbound-bytes` | `67108864` | 生成及 token 估算 POST 的解析前原始字节上限（含 chunked），超限 413；其他路由不缓冲请求体 | -| `--max-collect-bytes` | `8388608` | 聚合路径输出收集总字节上限（正文+思考+工具参数），超限返回 `response_too_large`；`0` 不限制 | +| `--max-collect-bytes` | `8388608` | 聚合及实时校验所需保留输出的总字节上限（正文+思考+工具参数/元数据），超限返回 `response_too_large`；`0` 不限制 | +| `--stream-mode compatible\|realtime` | `compatible` | `compatible` 保持聚合/片段重放行为；`realtime` 让三个协议增量发送，且不重生成工具参数 | | `--max-concurrent` | `64` | 仅限制三个生成端点；占满立即 503（含 Retry-After），不限制 token 估算；`0` 不限制 | | `--max-inflight-per-account` | `0` | 每进程、每账号的客户端推理在途上限；`0` 不限制，满载立即 503 | | `--upstream-keepalive [true/false]` | `false` | 启用按官方入口隔离的有界连接复用；重启生效 | @@ -42,7 +43,7 @@ Compose 会显式传入部分环境变量及 CLI 参数，删除 `.env` 中的 | `--max-request-bytes` | `33554432` | 处理后的上游 JSON 字节上限，须为正整数 | | `--log-body-limit` | `65536` | 旧文本预览兼容项；文本输出已停用，SQLite 诊断使用独立预算 | -环境变量包括 `CODEBUDDY_AUTH_DIR`、`CODEBUDDY_IMPORT_DIR`、`CODEBUDDY2API_KEY`、`CODEBUDDY2API_ADMIN_CSRF`、`CODEBUDDY2API_ADMIN_ORIGINS`、`CODEBUDDY2API_KEEP_TOOL_METADATA`、`CODEBUDDY2API_LOG`，以及 `CODEBUDDY2API_MAX_IMAGES`、`CODEBUDDY2API_IMAGE_POLICY`、`CODEBUDDY2API_MAX_REQUEST_BYTES`、`CODEBUDDY2API_LOG_BODY_LIMIT`、`CODEBUDDY2API_FAILOVER_MAX`、`CODEBUDDY2API_RETRY_WRITE_TIMEOUT`。启动示例见[部署指南](deployment.zh-CN.md)。 +环境变量包括 `CODEBUDDY_AUTH_DIR`、`CODEBUDDY_IMPORT_DIR`、`CODEBUDDY2API_KEY`、`CODEBUDDY2API_ADMIN_CSRF`、`CODEBUDDY2API_ADMIN_ORIGINS`、`CODEBUDDY2API_KEEP_TOOL_METADATA`、`CODEBUDDY2API_STREAM_MODE`、`CODEBUDDY2API_LOG`，以及 `CODEBUDDY2API_MAX_IMAGES`、`CODEBUDDY2API_IMAGE_POLICY`、`CODEBUDDY2API_MAX_REQUEST_BYTES`、`CODEBUDDY2API_LOG_BODY_LIMIT`、`CODEBUDDY2API_FAILOVER_MAX`、`CODEBUDDY2API_RETRY_WRITE_TIMEOUT`。启动示例见[部署指南](deployment.zh-CN.md)。 ### 工具元数据保留 @@ -54,6 +55,46 @@ Compose 会显式传入部分环境变量及 CLI 参数，删除 `.env` 中的 需使用包含此功能的源码/镜像和 Compose 配置；修改容器环境后重新创建容器。保留描述可能增加输入 token 和审核拦截风险，不保证所有账号/模型都同样兼容；设为 `false` 可恢复旧策略。此开关不恢复 Responses 原有投影裁掉的其他 schema 字段或深层节点，也不放宽请求体预算。 +### 流式模式 + +`stream_mode` 默认 `compatible`，可通过 `--stream-mode compatible|realtime`、`CODEBUDDY2API_STREAM_MODE` 或 WebUI 热更新枚举「流式模式」配置。显式 CLI/环境来源会锁定 WebUI 项。每个请求入口都会冻结此选择及 `max_collect_bytes`，因此热更新只影响后续请求，不改变在途流或其换号路径；客户端不能按请求覆盖。 + +- `compatible` 保持现有行为：Responses 流式先聚合；Chat/Messages 带工具时先聚合，无工具时沿用上游增量。聚合结果先校验，再按片段重放。两种模式下，非流式请求始终走已校验的聚合路径。 +- `realtime` 让三个协议都增量发送思考、正文、拒绝及工具参数。Responses 在输出项开始时分配稳定索引，Anthropic 使用稳定 block index。适配器在参数阶段或结束标记处确认工具身份，缺失时暂缓工具输出；元数据分片按顺序追加，不按字符串前缀猜测，输出项开始后禁止更换身份。`max_collect_bytes` 约束所保留的 UTF-8 输出，`0` 不限制。 + +实时模式绝不重生成损坏或不完整的工具参数。发送成功终端前，会在终端边界校验工具 ID、名称、已声明名称、JSON object 参数及 `tool_choice`。实时模式下，存在工具调用时还必须带上游 `tool_calls` 结束标记；带工具却标为 `stop` 会拒绝，而 compatible 模式保留旧的兼容接受行为。下游尚未收到字节时，失败保留真实上游 HTTP 状态及既有、有界的响应前换号规则；已发送任何字节后，参数损坏、断连、流错误和预算超限均以协议错误终端结束，不重放或切换凭据。合法的 `length`、拒绝和审核结果保留各自协议区别（Responses 的截断/过滤为 `incomplete`，绝非 `completed`），且不会重生成。因此客户端必须接受「部分正文后跟错误」，不能假定已开流的 SSE 一定成功结束。审计只增加白名单 `stream_mode` 标记及上游实际提供的用量。 + +运行时选回 `compatible` 即可恢复聚合流式及工具参数重生成，保留当前保存状态。源码降级前，移除新增 CLI/环境选项，停止网关并备份**当前**数据目录及同一代 SQLite/WAL/SHM。不要恢复升级前旧库，否则可能回滚新的领取、会话、撤销和账号状态。下例会离线写入当前数据库：仅删除 `settings.stream_mode`、递增 revision 并检查完整性，其它设置和表保持不变；禁止对运行中的数据库执行或混用 SQLite 文件代数。 + +```sh +python3 - /path/to/control.sqlite3 <<'PY' +import json, sqlite3, sys +con = sqlite3.connect(sys.argv[1]) +try: + con.execute("BEGIN IMMEDIATE") + row = con.execute("SELECT revision,payload FROM control WHERE id=1").fetchone() + if row is None: + raise SystemExit("missing control row") + revision, payload = row + data = json.loads(payload) + if set(data) != {"settings", "models", "credentials"} or not isinstance(data["settings"], dict): + raise SystemExit("unexpected control payload") + if "stream_mode" not in data["settings"]: + raise SystemExit("stream_mode is absent; no write needed") + del data["settings"]["stream_mode"] + con.execute("UPDATE control SET revision=?,payload=? WHERE id=1", + (revision + 1, json.dumps(data, ensure_ascii=False, allow_nan=False))) + con.commit() + if con.execute("PRAGMA integrity_check").fetchone()[0] != "ok": + raise SystemExit("integrity check failed") +finally: + con.close() +print("ok") +PY +``` + +旧版严格设置校验器会拒绝这个未知键，仅把值改成 `compatible` 不能让旧源码兼容。该流程必须离线执行；没有当前备份和已停止服务时，不要对生产库执行。 + ### 连接复用与账号容量 WebUI 系统设置可配置这两项；环境变量为 `CODEBUDDY2API_UPSTREAM_KEEPALIVE` 和 `CODEBUDDY2API_MAX_INFLIGHT_PER_ACCOUNT`，未设置时 Compose 不锁定 WebUI。连接复用默认关闭，启用后每个官方入口最多 64 条连接、保留 16 条空闲连接，空闲复用期限 30 秒；认证头逐请求设置，不保存上游 Cookie，关闭服务时释放连接池。原有代理环境、超时和重放规则不变；关闭并重启恢复逐请求连接。 @@ -186,7 +227,7 @@ WebUI 可以直接上传文件；以下限制针对 `POST /admin/credentials` - 图片计入全部历史和工具结果，重复图片逐次计数，按消息与内容块数组顺序判断新旧。默认保留最新 16 张，只移除超额图片并保留文本和消息结构；图片清空的内容用文本占位。 - `--image-policy error` 在本地返回 `413 / too_many_images`。处理后仍超过字节上限则返回 `413 / request_too_large`，不为满足预算继续截断文本。 - 图片数量合规不保证单图大小或模型视觉能力满足上游要求。URL/base64 图片可转换，Responses 图片 `file_id` 不支持。 -- 省略 `stream` 时三个端点都按协议默认返回完整 JSON（非流式）；`stream` 必须是布尔值。Responses 流式以及带工具的 Chat / Messages 流式先聚合校验，再输出 SSE，并非所有路径都实时逐 token 转发。 +- 省略 `stream` 时三个端点都按协议默认返回完整 JSON（非流式）；`stream` 必须是布尔值。默认 `stream_mode=compatible` 时，Responses 流式以及带工具的 Chat / Messages 流式先聚合校验再输出 SSE；`realtime` 则让三个协议都增量发送，省略/非流式请求仍返回完整的已校验 JSON。 - 推理错误按客户端协议成形：OpenAI 路由为顶层 `error` 对象，Messages 路由为 `{"type": "error", ...}`；保留状态码，开流后的错误只用 SSE 报告，不重放。 - 上游有效 `Retry-After`（0–86400 秒或对应 HTTP 日期）规范化为秒并在开流前返回；429 仅冷却对应账号/模型。无效或过期值回落正文重置时间或默认 600 秒；本地全凭据冷却的 429 返回剩余等待秒数。 - Chat 与 Responses 保留客户端显式 `prompt_cache_key`，不自动生成；缓存命中和节费取决于上游。 @@ -221,7 +262,7 @@ WebUI 可以直接上传文件；以下限制针对 `POST /admin/credentials` | 流式在第一个字节之前失败 | 按真实状态码返回，与 `stream=false` 同口径。只带一个流内 `error` 事件的 200 会被客户端读成「模型答了个空」，会话静默结束，审计里还记成一次成功 | | 换凭证重放（`--failover-max`） | 默认关闭。开启后，失败发生在「一个字节都没发给下游」之前时换一个凭证重放，最多 N 次，审计记为 `success` 并留下 `failover_recovered` 尝试标记。可重放的失败：上游 HTTP 401/403/429/502/503/504 拒绝，与确定没开始收正文的传输失败（建连失败/超时）；内容审核拒绝、上游已回 200 后合成的 502、读超时与协议错误一律不重放；换不出其他凭证时如实回第一次的状态码。计费口径：401/403/429/503 与建连类失败发生在受理阶段，不会扣费；502/504 可能已被上游处理并计费，但结果到不了下游，不重放也退不回额度——只是把一次已付费请求变成断掉的会话。这类重放在日志里标注「上游可能已处理该请求」，便于对账 | | 写超时重放（`--retry-write-timeout`） | 默认关闭。写超时只能证明正文没发完，不能证明上游忽略了已收到的部分，因此默认既不参与连接重试也不参与换凭证重放；跨境长会话比握手更容易遇到写超时，确认上游不按半截正文计费后再开启。这类重放同样带「上游可能已处理该请求」日志标记 | -| 工具参数损坏 | 聚合校验失败按 `--tool-call-max-retry`（默认 3）额外生成，可能消耗更多额度；被丢弃的生成带用量记入尝试明细；耗尽后返回错误 | +| 工具参数损坏 | 兼容聚合校验按 `--tool-call-max-retry`（默认 3）额外生成，可能消耗更多额度；被丢弃的生成带用量记入尝试明细；耗尽后返回错误。实时模式绝不重生成，只在成功终端前报告协议错误 | | 上游空流或残流 | 没有有效输出、缺少结束标记或包含错误的流不伪装为成功 | | 内容审核拒绝 | 脱敏 + `--no-compact` 下，仅完整非流式纯拒绝且模板确实缩短时，最多同账号兜底一次；流式不做审核重试，也不因此熔断或切号 | | 响应慢 | 在 WebUI 查看耗时与失败尝试，再选择当前账号支持的更快模型 | @@ -229,4 +270,4 @@ WebUI 可以直接上传文件；以下限制针对 `POST /admin/credentials` ## 降级与回滚 -功能开关不藏隐状态：关闭守卫或模式即对新请求停止生效，回退源码即恢复旧行为。例外是持久化设置与自动化状态：`control.sqlite3` 保存 WebUI 设置、模型规则与奖励预留，旧代码会拒绝未知字段。源码降级前移除新增的启动参数，并恢复升级前的控制库备份（含 WAL/SHM 文件，不混用）。回滚无法撤销已完成的上游签到、领取或旅行派出。 +功能开关不藏隐状态：关闭守卫或模式即对新请求停止生效，回退源码即恢复旧行为。例外是持久化设置与自动化状态：`control.sqlite3` 保存 WebUI 设置、模型规则与奖励预留，旧代码会拒绝未知字段。源码降级前移除新增的启动参数，停止服务，备份**当前**数据目录，并使用上面的窄范围离线 `settings.stream_mode` 删除流程，递增 revision 并执行完整性检查。不要恢复升级前数据库，也不要混用 WAL/SHM 代数，否则可能回滚较新的领取、会话、撤销和账号状态。回滚无法撤销已完成的上游签到、领取或旅行派出。 diff --git a/docs/clients.md b/docs/clients.md index effd1a5..33f0ef5 100644 --- a/docs/clients.md +++ b/docs/clients.md @@ -64,13 +64,15 @@ Use the common Base URL, API key and model ID with Cherry Studio, ZCode, LobeCha The generation endpoints are `POST /v1/chat/completions`, `POST /v1/responses` and `POST /v1/messages`. Set `stream: false` explicitly for JSON responses or `stream: true` for SSE. +Streaming policy is a server setting, not a client request field. The default `compatible` mode aggregates Responses and tool-bearing Chat/Messages as before; `realtime` sends all three protocols incrementally. Non-streaming requests remain validated JSON in either mode. See [Streaming modes](advanced.md#streaming-modes) for configuration, partial-output errors and the loss of automatic tool-argument regeneration. + ## Protocol behavior worth knowing - `developer` messages become `system`; the first system message is placed first before matching tool results, without mutating the original payload. - Chat accepts mixed Anthropic `tool_use` / `tool_result` history, preserving call IDs, arguments, result images and error markers; ordinary `thinking` becomes `reasoning_content`, not visible text. Native Chat fields stay unchanged. - Conflicting fields, unmatched tool results, unsupported mixed blocks and `redacted_thinking` return HTTP 400 before routing. Split user messages accept only `role` and `content`, with all `tool_result` blocks before ordinary text/images; Anthropic thinking signatures are not forwarded. - Named function choices are sent upstream as `required` with only that function available; invalid names are rejected locally. -- Errors follow the client protocol's own shape (OpenAI `error` object vs Anthropic `{"type":"error"}`), and status codes are preserved. +- Errors follow the client protocol's own shape (OpenAI `error` object vs Anthropic `{"type":"error"}`), and status codes are preserved. Realtime mode can deliver useful deltas before a later invalid terminal, disconnect or size error; valid truncation/filter distinctions remain native. Clients must not treat an opened SSE connection as proof of successful completion. - `POST /v1/messages/count_tokens` returns a character-based heuristic estimate for budgeting, not an exact count. See the [advanced reference](advanced.md#request-boundaries) for the full request-processing rules. diff --git a/docs/clients.zh-CN.md b/docs/clients.zh-CN.md index 94db46d..90b5559 100644 --- a/docs/clients.zh-CN.md +++ b/docs/clients.zh-CN.md @@ -64,13 +64,15 @@ Cherry Studio、ZCode、LobeChat、NextChat、Open WebUI 或自研 SDK 客户端 生成端点为 `POST /v1/chat/completions`、`POST /v1/responses` 与 `POST /v1/messages`。需要完整 JSON 时显式设 `stream: false`，需要 SSE 时设 `stream: true`。 +流式策略是服务端设置，不是客户端请求字段。默认 `compatible` 保持 Responses 及带工具 Chat/Messages 的聚合行为；`realtime` 让三个协议都增量发送。两种模式的非流式请求均为经过校验的 JSON。配置、部分输出错误及不再自动重生成工具参数等限制见[流式模式](advanced.zh-CN.md#流式模式)。 + ## 值得了解的协议行为 - 先将 `developer` 转为 `system` 并置顶首条系统消息，再关联工具结果；不改动调用方原始载荷。 - Chat 兼容混入的 Anthropic `tool_use` / `tool_result` 历史，保留调用 ID、参数、结果图片与错误标记；普通 `thinking` 转为 `reasoning_content`，不混入正文，原生 Chat 字段保持不变。 - 字段冲突、工具结果无法关联、不支持的混合内容块及 `redacted_thinking` 在选路前返回 HTTP 400。需拆分的用户消息只能包含 `role`、`content`，且 `tool_result` 必须在普通文本/图片之前；Anthropic 思考签名不转发。 - 指定名称的函数选择会以 `required` 且仅含该函数的形式发往上游；无效名称在本地拒绝。 -- 错误按客户端协议各自的形态返回（OpenAI 的 `error` 对象与 Anthropic 的 `{"type":"error"}`），状态码保留。 +- 错误按客户端协议各自的形态返回（OpenAI 的 `error` 对象与 Anthropic 的 `{"type":"error"}`），状态码保留。实时模式可能先送出有效增量，随后才遇到非法终端状态、断连或大小错误；合法截断/过滤仍保留协议原生区别。客户端不能仅凭 SSE 已开启就认定最终成功。 - `POST /v1/messages/count_tokens` 返回按字符估算的启发式结果，用于预算参考，不是精确计数。 完整的请求处理规则见[进阶参考](advanced.zh-CN.md#请求边界)。 diff --git a/tests/test_environment_config.py b/tests/test_environment_config.py index 469e906..3260932 100644 --- a/tests/test_environment_config.py +++ b/tests/test_environment_config.py @@ -136,6 +136,23 @@ def test_capability_guard_defaults_precedence_and_validation(self): with self.assertRaises(SystemExit): self.start({'CODEBUDDY2API_MODEL_CAPABILITY_GUARD': 'invalid'}) + def test_stream_mode_precedence_validation_and_hot_schema(self): + _, items, config = self.start(saved={'stream_mode': 'realtime'}) + self.assertEqual(config['stream_mode'], 'realtime') + self.assertEqual(items['stream_mode']['source'], 'management') + self.assertEqual(items['stream_mode']['choices'], ['compatible', 'realtime']) + self.assertFalse(items['stream_mode']['locked']) + _, items, config = self.start({'CODEBUDDY2API_STREAM_MODE': 'compatible'}) + self.assertEqual(config['stream_mode'], 'compatible') + self.assertEqual(items['stream_mode']['source'], 'environment') + self.assertTrue(items['stream_mode']['locked']) + _, items, config = self.start({'CODEBUDDY2API_STREAM_MODE': 'invalid'}, + cli=('--stream-mode=realtime',), saved={'stream_mode': 'compatible'}) + self.assertEqual(config['stream_mode'], 'realtime') + self.assertEqual(items['stream_mode']['source'], 'cli') + with self.assertRaises(ValueError): + self.start({'CODEBUDDY2API_STREAM_MODE': 'invalid'}) + def test_admin_allowed_origins_precedence_normalization_and_locking(self): key = 'CODEBUDDY2API_ADMIN_ORIGINS' self.assertEqual(self.start()[2]['admin_allowed_origins'], '') @@ -204,7 +221,7 @@ def test_compose_forwards_dotenv_limits_and_retries_without_changing_internal_bi 'CODEBUDDY2API_MAX_CONCURRENT': '2', 'CODEBUDDY2API_TOOL_CALL_MAX_RETRY': '1', 'CODEBUDDY2API_FAILOVER_MAX': '1', 'CODEBUDDY2API_RETRY_WRITE_TIMEOUT': 'true', 'CODEBUDDY2API_UPSTREAM_KEEPALIVE': 'true', 'CODEBUDDY2API_MAX_INFLIGHT_PER_ACCOUNT': '2', - 'CODEBUDDY2API_REQUEST_CONTEXT_MODE': 'scoped', + 'CODEBUDDY2API_REQUEST_CONTEXT_MODE': 'scoped', 'CODEBUDDY2API_STREAM_MODE': 'realtime', 'CODEBUDDY2API_MODEL_CAPABILITY_GUARD': 'false', 'CODEBUDDY2API_ADMIN_ORIGINS': 'https://chat.example.com', 'CODEBUDDY2API_KEEP_TOOL_METADATA': 'false', 'CODEBUDDY_IMPORT_DIR': '/data/auth/incoming'} @@ -221,7 +238,8 @@ def test_compose_unset_optional_settings_do_not_override_webui(self): service = self.compose({}) for name in ('CODEBUDDY2API_KEEP_TOOL_METADATA', 'CODEBUDDY2API_FAILOVER_MAX', 'CODEBUDDY2API_RETRY_WRITE_TIMEOUT', 'CODEBUDDY2API_UPSTREAM_KEEPALIVE', 'CODEBUDDY2API_MAX_INFLIGHT_PER_ACCOUNT', - 'CODEBUDDY2API_REQUEST_CONTEXT_MODE', 'CODEBUDDY2API_MODEL_CAPABILITY_GUARD', + 'CODEBUDDY2API_REQUEST_CONTEXT_MODE', 'CODEBUDDY2API_STREAM_MODE', + 'CODEBUDDY2API_MODEL_CAPABILITY_GUARD', 'CODEBUDDY2API_ADMIN_ORIGINS'): self.assertIsNone(service['environment'].get(name)) self.assertEqual(service['ports'][0]['host_ip'], '127.0.0.1') diff --git a/tests/test_realtime_streaming.py b/tests/test_realtime_streaming.py new file mode 100644 index 0000000..7311fa4 --- /dev/null +++ b/tests/test_realtime_streaming.py @@ -0,0 +1,1134 @@ +#!/usr/bin/env python3 +"""Deterministic realtime streaming and terminal-validation tests.""" +import sys +import tempfile +from contextlib import asynccontextmanager +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import asyncio +import json +import os +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import httpx +from fastapi import FastAPI +from fastapi.testclient import TestClient + +import converter +from app.audit_store import AuditStore +from app.control_store import ControlStore +from app.observability import AuditMiddleware +from app.settings import apply_persisted_settings, resolve_settings +from app.startup import load_startup_env, resolve_startup_key +from app.adapters.anthropic_adapter import AnthropicStreamConverter +from app.adapters.responses_adapter import ResponsesStreamConverter +from app.upstream_io import ChatSSEAccumulator, StreamOutputBudget, UpstreamResponseError +from app.inference_resources import AccountCapacity, request_resources + + +def _line(delta, finish=None, usage=None): + chunk = {"id": "synthetic", "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}]} + if usage is not None: + chunk["usage"] = usage + return "data: " + json.dumps(chunk, ensure_ascii=False) + "\n\n" + + +def _events(raw): + result = [] + for block in raw.strip().split("\n\n"): + if not block: + continue + data = next((line[6:] for line in block.splitlines() if line.startswith("data: ")), None) + if data is not None: + result.append(json.loads(data)) + return result + + +class _PausedLines: + def __init__(self, first, tail, at_boundary): + self.first = first + self.tail = tail + self.at_boundary = at_boundary + self.release = asyncio.Event() + + async def __aiter__(self): + yield self.first + # The next __anext__ means the gateway has consumed the first line. In realtime + # mode any corresponding client event has already passed through ASGI send. + self.at_boundary.set() + await self.release.wait() + for line in self.tail: + yield line + + +class _PausedResponse: + def __init__(self, lines): + self.status_code = 200 + self.headers = {"Content-Type": "text/event-stream"} + self.lines = lines + + async def aiter_lines(self): + async for line in self.lines: + yield line + + +class _FixedResponse: + def __init__(self, lines=(), *, status=200, headers=None, error=None, body=b""): + self.status_code = status + self.headers = headers or {"Content-Type": "text/event-stream"} + self.lines = lines + self.error = error + self.body = body + + async def aiter_lines(self): + for line in self.lines: + yield line + if self.error is not None: + raise self.error + + async def aiter_bytes(self): + yield self.body + + +class RealtimeTransportTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.config = { + "api_key": "", "cred": None, "cred_pool": None, "model_guard": False, + "max_images": 16, "image_policy": "truncate", "max_request_bytes": 32 * 1024 * 1024, + "log_body_limit": 0, "log_path": None, "desensitize": False, "no_compact": False, + "max_collect_bytes": 0, "max_concurrent": 0, "stream_mode": "compatible", + "request_context_mode": "legacy", "model_capability_guard": False, + "upstream_keepalive": False, "retry_write_timeout": False, "failover_max": 0, + } + self.enterContext(patch.dict(converter.CONFIG, self.config, clear=False)) + self.enterContext(patch.object(converter, "_route_chat", + side_effect=lambda payload, body, rid: (body, None, {}, "https://synthetic.invalid"))) + self.enterContext(patch.object(converter, "_log")) + self.enterContext(patch.object(converter, "_note_cred_model_ok")) + + async def asgi_post(self, path, payload, backend, *, route=None, config=None): + """Drive the real ASGI app while retaining each send message for assertions.""" + sent = [] + request = asyncio.Queue() + await request.put({"type": "http.request", "body": json.dumps(payload).encode(), + "more_body": False}) + + async def receive(): + return await request.get() + + async def send(message): + sent.append(message) + + if route is None: + def route(payload_, body, rid): + return body, None, {}, "https://synthetic.invalid" + scope = {"type": "http", "method": "POST", "path": path, + "raw_path": path.encode(), "query_string": b"", "headers": [], + "scheme": "http", "http_version": "1.1", "server": ("test", 80), + "client": ("test", 1), "asgi": {"version": "3.0", "spec_version": "2.3"}} + with patch.dict(converter.CONFIG, config or {}, clear=False), \ + patch.object(converter, "_route_chat", side_effect=route), \ + patch.object(converter, "_backend_stream", backend), \ + patch.object(converter, "_log"), patch.object(converter, "_note_cred_model_ok"): + await asyncio.wait_for(converter.app(scope, receive, send), 2) + return sent + + def payload(self, protocol, tools): + if protocol == "chat": + body = {"model": "auto", "stream": True, + "messages": [{"role": "user", "content": "hi"}]} + if tools: + body["tools"] = [{"type": "function", "function": { + "name": "synthetic_tool", "parameters": {"type": "object"}}}] + elif protocol == "responses": + body = {"model": "auto", "stream": True, "input": "hi"} + if tools: + body["tools"] = [{"type": "function", "name": "synthetic_tool", + "parameters": {"type": "object"}}] + else: + body = {"model": "auto", "stream": True, "max_tokens": 32, + "messages": [{"role": "user", "content": "hi"}]} + if tools: + body["tools"] = [{"name": "synthetic_tool", "input_schema": {"type": "object"}}] + return body + + async def drive(self, protocol, mode, tools, *, change_to=None, first_delta=None): + if first_delta is None: + first_delta = ({"tool_calls": [{"index": 0, "id": "call_1", "type": "function", + "function": {"name": "synthetic_tool", "arguments": '{"x":'}}]} + if tools else {"content": "early"}) + finish = "tool_calls" if tools else "stop" + tail = [_line({}, finish, {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}), + "data: [DONE]\n\n"] + if tools: + tail.insert(0, _line({"tool_calls": [{"index": 0, "function": {"arguments": "1}"}}]})) + boundary = asyncio.Event() + response = _PausedResponse(_PausedLines(_line(first_delta), tail, boundary)) + sent = [] + request = asyncio.Queue() + await request.put({"type": "http.request", "body": json.dumps(self.payload(protocol, tools)).encode(), + "more_body": False}) + + async def receive(): + return await request.get() + + async def send(message): + sent.append(message) + + @asynccontextmanager + async def backend(*args, **kwargs): + yield response + + converter.CONFIG["stream_mode"] = mode + path = {"chat": "/v1/chat/completions", "responses": "/v1/responses", + "messages": "/v1/messages"}[protocol] + scope = {"type": "http", "method": "POST", "path": path, + "raw_path": path.encode(), "query_string": b"", "headers": [], + "scheme": "http", "http_version": "1.1", "server": ("test", 80), + "client": ("test", 1), "asgi": {"version": "3.0", "spec_version": "2.3"}} + with patch.object(converter, "_backend_stream", backend): + task = asyncio.create_task(converter.app(scope, receive, send)) + try: + await asyncio.wait_for(boundary.wait(), 2) + before_release = b"".join(message.get("body", b"") for message in sent + if message["type"] == "http.response.body") + if change_to is not None: + converter.CONFIG["stream_mode"] = change_to + response.lines.release.set() + await asyncio.wait_for(task, 2) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + wire = b"".join(message.get("body", b"") for message in sent + if message["type"] == "http.response.body") + return before_release, wire + + @staticmethod + def meaningful(protocol, wire): + if protocol == "chat": + return b"early" in wire or b'\\"x\\":' in wire + if protocol == "responses": + return any(marker in wire for marker in ( + b"response.output_text.delta", b"response.reasoning_summary_text.delta", + b"response.function_call_arguments.delta")) + return b"content_block_delta" in wire and (b"early" in wire or b"partial_json" in wire) + + async def collect(self, protocol, response, *, max_collect_bytes=0, tool_choice="required"): + body = {"model": "auto", "stream": True, "parallel_tool_calls": True, + "messages": [{"role": "assistant", "content": ""}, {"role": "user", "content": "hi"}], + "tools": [{"type": "function", "function": { + "name": "synthetic_tool", "parameters": {"type": "object"}}}], + "tool_choice": tool_choice} + attempts = 0 + + @asynccontextmanager + async def backend(*args, **kwargs): + nonlocal attempts + attempts += 1 + yield response + + converter.CONFIG.update(stream_mode="realtime", max_collect_bytes=max_collect_bytes) + factory = {"chat": lambda: converter._stream_upstream("https://synthetic.invalid", {}, body, "auto"), + "responses": lambda: converter._stream_responses("https://synthetic.invalid", {}, body, "auto"), + "messages": lambda: converter._stream_anthropic("https://synthetic.invalid", {}, body, "auto")}[protocol] + chunks, failure = [], None + with patch.object(converter, "_backend_stream", backend): + try: + async for chunk in factory(): + chunks.append(chunk) + except (httpx.HTTPError, UpstreamResponseError) as error: + failure = error + return b"".join(chunks), attempts, failure + + async def test_invalid_terminal_state_never_succeeds_or_replays(self): + cases = { + "malformed arguments": [ + _line({"tool_calls": [{"index": 0, "id": "call", "type": "function", + "function": {"name": "synthetic_tool", "arguments": "{"}}]}, + usage={"total_tokens": 7}), + _line({"tool_calls": [{"index": 0, "function": {"arguments": "invalid"}}]}), + _line({}, "tool_calls")], + "missing completion": [_line({"content": "partial"})], + "partial disconnect": [_line({"content": "partial"}),], + } + for protocol in ("chat", "responses", "messages"): + for name, lines in cases.items(): + with self.subTest(protocol=protocol, case=name): + error = httpx.ReadError("synthetic reset") if name == "partial disconnect" else None + wire, attempts, failure = await self.collect( + protocol, _FixedResponse(lines, error=error)) + self.assertEqual(attempts, 1) + self.assertIsNone(failure) + self.assertTrue(wire) + self.assertNotIn(b"data: [DONE]", wire) + self.assertNotIn(b"response.completed", wire) + self.assertNotIn(b"message_stop", wire) + self.assertIn(b"error", wire) + + async def test_compatible_budget_snapshot_applies_to_repair_attempts(self): + policy = converter._StreamRequestPolicy("compatible", True, 3) + first = [ + _line({"tool_calls": [{"index": 0, "id": "call_bad", "type": "function", + "function": {"name": "wrong", "arguments": "{}"}}]}, + "tool_calls"), + "data: [DONE]\n\n", + ] + second = [_line({"content": "long output"}), _line({}, "stop"), "data: [DONE]\n\n"] + responses = iter((_FixedResponse(first), _FixedResponse(second))) + state = {"attempts": 0, "closed": 0} + + @asynccontextmanager + async def backend(*args, **kwargs): + state["attempts"] += 1 + try: + yield next(responses) + finally: + state["closed"] += 1 + + body = {"model": "auto", "messages": [{"role": "user", "content": "hi"}], + "tools": [{"type": "function", "function": {"name": "declared"}}], + "tool_choice": "required"} + with patch.dict(converter.CONFIG, {"max_collect_bytes": 999, "tool_call_max_retry": 1}), \ + patch.object(converter, "_backend_stream", backend), \ + patch.object(converter, "_log"), patch.object(converter, "_note_cred_model_ok"): + with self.assertRaises(UpstreamResponseError): + _ = [chunk async for chunk in converter._chat_sse_lines( + "https://synthetic.invalid", {}, body, "auto", 0, "diag", policy=policy)] + self.assertEqual(state, {"attempts": 2, "closed": 2}) + + async def test_realtime_native_http_error_keeps_real_preflight_status_at_asgi(self): + @asynccontextmanager + async def backend(*args, **kwargs): + yield _FixedResponse(status=429, headers={"Retry-After": "7"}, + body=b'{"error":{"message":"limited"}}') + + sent = await self.asgi_post( + "/v1/responses", {"model": "auto", "stream": True, "input": "hi"}, backend, + config={"stream_mode": "realtime", "max_collect_bytes": 0}) + start = next(message for message in sent if message["type"] == "http.response.start") + body = b"".join(message.get("body", b"") for message in sent + if message["type"] == "http.response.body") + self.assertEqual(start["status"], 429) + self.assertIn(b"limited", body) + self.assertNotIn(b"text/event-stream", b"".join( + value for key, value in start.get("headers", []) if key.lower() == b"content-type")) + + async def test_realtime_malformed_terminal_and_utf8_budget_have_no_success(self): + @asynccontextmanager + async def malformed(*args, **kwargs): + yield _FixedResponse([ + _line({"content": "first"}), + _line({}, "stop"), + _line({"content": "late"}), + "data: [DONE]\n\n", + ]) + + sent = await self.asgi_post( + "/v1/responses", {"model": "auto", "stream": True, "input": "hi"}, malformed, + config={"stream_mode": "realtime", "max_collect_bytes": 0}) + wire = b"".join(message.get("body", b"") for message in sent + if message["type"] == "http.response.body") + self.assertIn(b"error", wire) + self.assertNotIn(b"response.completed", wire) + + @asynccontextmanager + async def oversized(*args, **kwargs): + yield _FixedResponse([ + _line({"content": "你"}, usage={"total_tokens": 3}), + _line({}, "stop"), + "data: [DONE]\n\n", + ]) + + usage = [] + with patch.object(converter, "observe_usage", side_effect=usage.append): + sent = await self.asgi_post( + "/v1/responses", {"model": "auto", "stream": True, "input": "hi"}, oversized, + config={"stream_mode": "realtime", "max_collect_bytes": 1}) + wire = b"".join(message.get("body", b"") for message in sent + if message["type"] == "http.response.body") + self.assertIn(b"response_too_large", wire) + self.assertNotIn(b"response.completed", wire) + self.assertEqual(usage, [{"total_tokens": 3}]) + + async def test_realtime_budget_error_closes_upstream_and_releases_capacity(self): + capacity = AccountCapacity() + state = {"closed": False} + usage = [] + response = _FixedResponse([ + _line({"content": "a"}, usage={"total_tokens": 2}), + _line({"content": "b"}), + _line({}, "stop"), + "data: [DONE]\n\n", + ]) + + @asynccontextmanager + async def backend(*args, **kwargs): + try: + yield response + finally: + state["closed"] = True + + def route(payload, body, rid): + resources = request_resources.get() + lease = capacity.acquire("synthetic-account", 1, object(), 0) + resources.add(lease) + return body, lease, {}, "https://synthetic.invalid" + + sent = [] + request = asyncio.Queue() + await request.put({"type": "http.request", "body": json.dumps({ + "model": "auto", "stream": True, "input": "hi"}).encode(), "more_body": False}) + + async def receive(): + return await request.get() + + async def send(message): + sent.append(message) + + scope = {"type": "http", "method": "POST", "path": "/v1/responses", + "raw_path": b"/v1/responses", "query_string": b"", "headers": [], + "scheme": "http", "http_version": "1.1", "server": ("test", 80), + "client": ("test", 1), "asgi": {"version": "3.0", "spec_version": "2.3"}} + with patch.dict(converter.CONFIG, {"stream_mode": "realtime", "max_collect_bytes": 1}), \ + patch.object(converter, "_route_chat", side_effect=route), \ + patch.object(converter, "_backend_stream", backend), \ + patch.object(converter, "observe_usage", side_effect=usage.append), \ + patch.object(converter, "_log"), patch.object(converter, "_note_cred_model_ok"): + await asyncio.wait_for(converter.app(scope, receive, send), 2) + wire = b"".join(message.get("body", b"") for message in sent + if message["type"] == "http.response.body") + self.assertIn(b"response_too_large", wire) + self.assertTrue(state["closed"]) + self.assertEqual(usage, [{"total_tokens": 2}]) + self.assertEqual(capacity._counts, {}) + + async def test_realtime_cancellation_releases_capacity_and_closes_upstream(self): + capacity = AccountCapacity() + state = {"closed": False} + paused = _PausedResponse(_PausedLines( + _line({"content": "first"}, usage={"total_tokens": 2}), + [_line({}, "stop", {"total_tokens": 2}), "data: [DONE]\n\n"], + asyncio.Event())) + + @asynccontextmanager + async def backend(*args, **kwargs): + try: + yield paused + finally: + state["closed"] = True + + def route(payload, body, rid): + resources = request_resources.get() + lease = capacity.acquire("synthetic-account", 1, object(), 0) + resources.add(lease) + return body, lease, {}, "https://synthetic.invalid" + + sent = [] + request = asyncio.Queue() + await request.put({"type": "http.request", "body": json.dumps({ + "model": "auto", "stream": True, "input": "hi"}).encode(), "more_body": False}) + + async def receive(): + return await request.get() + + async def send(message): + sent.append(message) + + scope = {"type": "http", "method": "POST", "path": "/v1/responses", + "raw_path": b"/v1/responses", "query_string": b"", "headers": [], + "scheme": "http", "http_version": "1.1", "server": ("test", 80), + "client": ("test", 1), "asgi": {"version": "3.0", "spec_version": "2.3"}} + with patch.dict(converter.CONFIG, {"stream_mode": "realtime", "max_collect_bytes": 0}), \ + patch.object(converter, "_route_chat", side_effect=route), \ + patch.object(converter, "_backend_stream", backend), \ + patch.object(converter, "observe_usage"), patch.object(converter, "_log"), \ + patch.object(converter, "_note_cred_model_ok"): + task = asyncio.create_task(converter.app(scope, receive, send)) + await asyncio.wait_for(paused.lines.at_boundary.wait(), 2) + self.assertEqual(sum(capacity._counts.values()), 1) + task.cancel() + result = await asyncio.gather(task, return_exceptions=True) + self.assertIsInstance(result[0], asyncio.CancelledError) + self.assertTrue(state["closed"]) + self.assertEqual(capacity._counts, {}) + + async def test_realtime_prefirst_byte_failover_is_bounded(self): + capacity = AccountCapacity() + first_lease = capacity.acquire("first-account", 1, object(), 0) + second_lease = capacity.acquire("second-account", 1, object(), 0) + state = {"attempts": 0, "closed": 0} + responses = iter((_FixedResponse(status=429, headers={"Retry-After": "3"}, + body=b'{"error":{"message":"limited"}}'), + _FixedResponse([_line({"content": "ok"}), _line({}, "stop"), + "data: [DONE]\n\n"]))) + + @asynccontextmanager + async def backend(*args, **kwargs): + state["attempts"] += 1 + try: + yield next(responses) + finally: + state["closed"] += 1 + + body = {"model": "auto", "messages": [{"role": "user", "content": "hi"}]} + policy = converter._StreamRequestPolicy("realtime", False, 0) + + def make(routed, cred, headers, url): + return converter._stream_responses( + url, headers, converter._body_with_stream_policy(routed, policy), "auto", cred=cred) + + def route(payload, canonical, rid, *, tried=()): + return canonical, second_lease, {}, "https://synthetic.invalid" + + with patch.dict(converter.CONFIG, {"stream_mode": "realtime", "failover_max": 1, + "max_collect_bytes": 0}), \ + patch.object(converter, "_backend_stream", backend), \ + patch.object(converter, "_route_chat", side_effect=route), \ + patch.object(converter, "_log"), patch.object(converter, "_note_cred_model_ok"): + stream, first = await converter._stream_plan( + {"model": "auto"}, body, "auto", "diag", 0, make, + body, first_lease, {}, "https://synthetic.invalid") + self.assertTrue(first) + await converter._close_stream(stream) + converter.release_credential(second_lease) + self.assertEqual(state["attempts"], 2) + self.assertEqual(state["closed"], 2) + self.assertEqual(capacity._counts, {}) + + async def test_realtime_native_length_and_filter_distinctions_do_not_replay(self): + for protocol in ("chat", "responses", "messages"): + for finish_reason in ("length", "content_filter"): + with self.subTest(protocol=protocol, finish_reason=finish_reason): + lines = [_line({"content": "partial"}), _line({}, finish_reason), + "data: [DONE]\n\n"] + wire, attempts, failure = await self.collect( + protocol, _FixedResponse(lines), tool_choice="auto") + self.assertEqual(attempts, 1) + self.assertIsNone(failure) + self.assertNotIn(b"response.completed", wire) + if protocol == "responses": + self.assertIn(b"response.incomplete", wire) + elif protocol == "messages": + self.assertIn(b"message_stop", wire) + self.assertIn(b"max_tokens" if finish_reason == "length" else b"end_turn", wire) + else: + self.assertIn(b"data: [DONE]", wire) + + async def test_realtime_budget_error_after_output_has_no_success_terminal(self): + for protocol in ("chat", "responses", "messages"): + with self.subTest(protocol=protocol): + lines = [_line({"content": "ok"}), + _line({"tool_calls": [{"index": 0, "id": "c", "type": "function", + "function": {"name": "t", "arguments": "{}"}}]})] + wire, attempts, failure = await self.collect( + protocol, _FixedResponse(lines), max_collect_bytes=2) + self.assertEqual(attempts, 1) + self.assertIsNone(failure) + self.assertIn(b"ok", wire) + self.assertIn(b"response_too_large", wire) + self.assertNotIn(b"data: [DONE]", wire) + self.assertNotIn(b"response.completed", wire) + self.assertNotIn(b"message_stop", wire) + + async def test_realtime_http_failure_preserves_status_and_retry_after(self): + for protocol in ("chat", "responses", "messages"): + with self.subTest(protocol=protocol): + response = _FixedResponse(status=429, headers={"Retry-After": "7"}, + body=b'{"error":{"message":"limited"}}') + wire, attempts, failure = await self.collect(protocol, response) + self.assertEqual(wire, b"") + self.assertEqual(attempts, 1) + self.assertIsInstance(failure, UpstreamResponseError) + self.assertEqual((failure.status, failure.headers.get("Retry-After")), (429, "7")) + + async def test_reasoning_delta_reaches_asgi_client_before_release(self): + before, wire = await self.drive( + "responses", "realtime", False, + first_delta={"reasoning_content": "meaningful reasoning"}) + self.assertIn(b"response.reasoning_summary_text.delta", before) + self.assertIn(b"meaningful reasoning", before) + reasoning_deltas = [event["delta"] for event in _events(wire.decode()) + if event["type"] == "response.reasoning_summary_text.delta"] + self.assertEqual("".join(reasoning_deltas), "meaningful reasoning") + self.assertIn(b"response.completed", wire) + + async def test_hot_mode_change_affects_only_the_next_request(self): + before, _ = await self.drive("responses", "realtime", True, change_to="compatible") + self.assertTrue(self.meaningful("responses", before), before) + before, _ = await self.drive("responses", "compatible", True) + self.assertFalse(self.meaningful("responses", before), before) + + async def test_upstream_empty_finish_reason_is_not_a_terminal_marker(self): + for protocol in ("chat", "responses", "messages"): + for mode in ("compatible", "realtime"): + for tools in (False, True): + with self.subTest(protocol=protocol, mode=mode, tools=tools): + delta = ({"tool_calls": [{"index": 0, "id": "call_1", "type": "function", + "function": {"name": "synthetic_tool", "arguments": "{}"}}]} + if tools else {"content": "early"}) + rows = [_line({"role": "assistant", "content": "", "reasoning_content": "", + "tool_calls": []}, ""), + _line(delta, ""), _line({}, "tool_calls" if tools else "stop"), + _line({}, "", {"total_tokens": 2}), "data: [DONE]\n\n"] + @asynccontextmanager + async def backend(*args, **kwargs): + yield _FixedResponse(rows) + path = {"chat": "/v1/chat/completions", "responses": "/v1/responses", + "messages": "/v1/messages"}[protocol] + sent = await self.asgi_post(path, self.payload(protocol, tools), backend, + config={"stream_mode": mode}) + self.assertEqual(sent[0]["status"], 200) + wire = b"".join(m.get("body", b"") for m in sent) + self.assertNotIn(b'"error"', wire) + terminal = {"chat": b"data: [DONE]", "responses": b"response.completed", + "messages": b"message_stop"}[protocol] + self.assertIn(terminal, wire) + for protocol in ("chat", "responses", "messages"): + with self.subTest(protocol=protocol, no_terminal=True): + wire, attempts, failure = await self.collect(protocol, _FixedResponse([ + _line({"content": "early"}, ""), "data: [DONE]\n\n"]), tool_choice="none") + self.assertEqual(attempts, 1) + self.assertNotIn(b"response.completed", wire) + self.assertNotIn(b"message_stop", wire) + self.assertNotIn(b"data: [DONE]", wire) + self.assertTrue(failure or b'"error"' in wire) + + + async def test_protocol_mode_and_tool_matrix_has_no_aggregate_in_realtime(self): + for protocol in ("chat", "responses", "messages"): + for tools in (False, True): + for mode in ("compatible", "realtime"): + with self.subTest(protocol=protocol, tools=tools, mode=mode): + before, wire = await self.drive(protocol, mode, tools) + # Responses and tool-bearing Chat/Messages are compatible-mode aggregates. + aggregated = mode == "compatible" and (protocol == "responses" or tools) + self.assertNotEqual(self.meaningful(protocol, before), aggregated, before) + terminal = {b"data: [DONE]", b"response.completed", b"message_stop"} + self.assertTrue(any(marker in wire for marker in terminal), wire) + + +class RealtimeAdapterTests(unittest.TestCase): + def realtime_converters(self): + budget = StreamOutputBudget(0) + tracker = ChatSSEAccumulator(collect=False, retain_tools=True, budget=budget) + responses = ResponsesStreamConverter(model="m", realtime=True, budget=budget, + tool_states=tracker.tools) + anthropic = AnthropicStreamConverter(model="m", realtime=True, budget=budget, + tool_states=tracker.tools) + return budget, tracker, responses, anthropic + + @staticmethod + def feed_pair(tracker, converter, delta, finish=None, usage=None): + raw = _line(delta, finish, usage) + tracker.feed_line(raw) + return converter.feed_line(raw) + + def test_responses_uses_first_seen_order_and_never_reuses_an_index(self): + _, tracker, converter, _ = self.realtime_converters() + raw = "" + raw += self.feed_pair(tracker, converter, {"content": "text-first"}) + raw += self.feed_pair(tracker, converter, {"reasoning_content": "think"}) + raw += self.feed_pair(tracker, converter, {"tool_calls": [{ + "index": 0, "id": "call_1", "function": {"name": "synthetic_tool", "arguments": "{}"}}]}) + raw += self.feed_pair(tracker, converter, {"reasoning_content": "-again"}) + raw += self.feed_pair(tracker, converter, {"content": "-again"}) + raw += self.feed_pair(tracker, converter, {}, "tool_calls") + raw += self.feed_pair(tracker, converter, {}, None, + {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}) + converter.set_validated_tools(tracker.result()["tool_calls"]) + raw += converter.finish() + events = _events(raw) + added = [event for event in events if event["type"] == "response.output_item.added"] + indices = [event["output_index"] for event in added] + self.assertEqual(indices, [0, 1, 2]) + self.assertEqual(len(set(indices)), 3) + completed = next(event for event in events if event["type"] == "response.completed") + self.assertEqual([item["type"] for item in completed["response"]["output"]], + ["message", "reasoning", "function_call"]) + sequence = [event["sequence_number"] for event in events] + self.assertEqual(sequence, list(range(1, len(sequence) + 1))) + final_indices = {item["id"]: index for index, item in enumerate(completed["response"]["output"])} + for event in added: + self.assertEqual(final_indices[event["item"]["id"]], event["output_index"]) + + def test_responses_tool_then_text_keeps_tool_index_zero(self): + _, tracker, converter, _ = self.realtime_converters() + raw = self.feed_pair(tracker, converter, {"tool_calls": [{ + "index": 0, "id": "call", "function": {"name": "synthetic_tool", "arguments": "{}"}}]}) + raw += self.feed_pair(tracker, converter, {"content": "after"}) + raw += self.feed_pair(tracker, converter, {}, "tool_calls") + converter.set_validated_tools(tracker.result()["tool_calls"]) + raw += converter.finish() + events = _events(raw) + added = [event for event in events if event["type"] == "response.output_item.added"] + self.assertEqual([(event["output_index"], event["item"]["type"]) for event in added], + [(0, "function_call"), (1, "message")]) + completed = next(event for event in events if event["type"] == "response.completed") + self.assertEqual([item["type"] for item in completed["response"]["output"]], + ["function_call", "message"]) + + def test_responses_delays_tool_start_and_flushes_fragmented_identity_and_arguments_in_order(self): + _, tracker, converter, _ = self.realtime_converters() + raw = self.feed_pair(tracker, converter, {"tool_calls": [{ + "index": 0, "function": {"name": "synthetic_", "arguments": '{"x":'}}]}) + self.assertNotIn("response.output_item.added", raw) + raw += self.feed_pair(tracker, converter, {"tool_calls": [{ + "index": 0, "id": "call_", "function": {"name": "tool", "arguments": "1}"}}]}) + raw += self.feed_pair(tracker, converter, {}, "tool_calls") + converter.set_validated_tools(tracker.result()["tool_calls"]) + raw += converter.finish() + events = _events(raw) + start = next(event for event in events if event["type"] == "response.output_item.added") + self.assertEqual(start["item"]["call_id"], "call_") + self.assertEqual(start["item"]["name"], "synthetic_tool") + deltas = [event["delta"] for event in events + if event["type"] == "response.function_call_arguments.delta"] + self.assertEqual("".join(deltas), '{"x":1}') + + def test_fragmented_id_and_conflicting_emitted_metadata(self): + _, tracker, converter, _ = self.realtime_converters() + self.feed_pair(tracker, converter, {"tool_calls": [{ + "index": 0, "id": "call_", "function": {"arguments": "{"}}]}) + raw = self.feed_pair(tracker, converter, {"tool_calls": [{ + "index": 0, "id": "1", "function": {"name": "synthetic_tool", "arguments": "}"}}]}) + start = next(event for event in _events(raw) if event["type"] == "response.output_item.added") + self.assertEqual(start["item"]["call_id"], "call_1") + with self.assertRaises(httpx.RemoteProtocolError): + tracker.feed_line(_line({"tool_calls": [{"index": 0, "id": "other", + "function": {"name": "synthetic_tool"}}]})) + + def test_anthropic_interleaved_parallel_tools_keep_stable_block_identities(self): + _, tracker, _, converter = self.realtime_converters() + raw = "" + raw += self.feed_pair(tracker, converter, {"reasoning_content": "plan"}) + raw += self.feed_pair(tracker, converter, {"tool_calls": [ + {"index": 0, "id": "a", "function": {"name": "one", "arguments": '{"x":'}}, + {"index": 1, "id": "b", "function": {"name": "two", "arguments": '{"y":'}}]}) + raw += self.feed_pair(tracker, converter, {"tool_calls": [ + {"index": 1, "function": {"arguments": "2}"}}, + {"index": 0, "function": {"arguments": "1}"}}]}) + raw += self.feed_pair(tracker, converter, {"content": "done"}) + raw += self.feed_pair(tracker, converter, {"reasoning_content": "checked"}) + raw += self.feed_pair(tracker, converter, {}, "tool_calls") + converter.set_validated_tools(tracker.result()["tool_calls"]) + raw += converter.finish() + events = _events(raw) + starts = [event for event in events if event["type"] == "content_block_start"] + tools = {event["index"]: event["content_block"] for event in starts + if event["content_block"]["type"] == "tool_use"} + self.assertEqual({key: value["id"] for key, value in tools.items()}, {1: "a", 2: "b"}) + arguments = {index: [] for index in tools} + for event in events: + if event["type"] == "content_block_delta" and event["delta"]["type"] == "input_json_delta": + arguments[event["index"]].append(event["delta"]["partial_json"]) + self.assertEqual(arguments[1], ['{"x":', "1}"]) + self.assertEqual(arguments[2], ['{"y":', "2}"]) + stops = [event["index"] for event in events if event["type"] == "content_block_stop"] + self.assertEqual(stops, [0, 3, 4, 1, 2]) + + def test_responses_tool_start_and_done_reconstruct_arguments_once(self): + complete = [ + _line({"tool_calls": [{"index": 0, "id": "call_a", "type": "function", + "function": {"name": "synthetic_tool", "arguments": '{"x":1}'}}]}), + _line({}, "tool_calls", {"total_tokens": 2}), + "data: [DONE]\n\n", + ] + delayed = [ + _line({"tool_calls": [{"index": 0, "id": "call_a", "type": "function", + "function": {"name": "synthetic_", "arguments": ""}}]}), + _line({"tool_calls": [{"index": 0, "function": {"name": "tool", + "arguments": '{"x":'}}]}), + _line({"tool_calls": [{"index": 0, "function": {"arguments": "1}"}}]}, + "tool_calls", {"total_tokens": 2}), + "data: [DONE]\n\n", + ] + for rows in (complete, delayed): + with self.subTest(delayed=rows is delayed): + budget = StreamOutputBudget(0) + tracker = ChatSSEAccumulator(collect=False, retain_tools=True, budget=budget, + declared_names={"synthetic_tool"}) + converter = ResponsesStreamConverter( + model="m", realtime=True, budget=budget, tool_states=tracker.tools, + declared_names={"synthetic_tool"}) + raw = "" + for row in rows: + tracker.feed_line(row) + raw += converter.feed_line(row) + merged = tracker.result() + converter.set_validated_tools(merged["tool_calls"]) + raw += converter.finish() + events = _events(raw) + added = [event for event in events + if event["type"] == "response.output_item.added" + and event["item"]["type"] == "function_call"] + self.assertEqual(len(added), 1) + deltas = [event["delta"] for event in events + if event["type"] == "response.function_call_arguments.delta"] + done = next(event for event in events + if event["type"] == "response.function_call_arguments.done") + self.assertEqual(added[0]["item"]["arguments"], "") + self.assertEqual(json.loads(added[0]["item"]["arguments"] + "".join(deltas)), + {"x": 1}) + self.assertEqual(done["arguments"], added[0]["item"]["arguments"] + "".join(deltas)) + final = next(event for event in events if event["type"] == "response.completed") + final_call = next(item for item in final["response"]["output"] + if item["type"] == "function_call") + self.assertEqual(final_call["arguments"], done["arguments"]) + + def test_fragmented_metadata_waits_for_a_safe_boundary(self): + cases = { + "split-name": [ + {"id": "call_a", "name": "synthetic_", "arguments": ""}, + {"name": "tool", "arguments": "{}"}, + ], + "split-id": [ + {"id": "call_", "name": "synthetic_tool", "arguments": ""}, + {"id": "abc", "arguments": "{}"}, + ], + "both-fragmented": [ + {"id": "call_", "name": "synthetic_", "arguments": ""}, + {"id": "abc", "name": "tool", "arguments": "{}"}, + ], + "arguments-before-metadata": [ + {"arguments": '{"x":'}, + {"id": "call_a", "name": "synthetic_tool"}, + ], + "empty-fragment": [ + {"id": "", "name": "", "arguments": ""}, + {"id": "call_a", "name": "synthetic_tool", "arguments": "{}"}, + ], + } + for name, metadata in cases.items(): + with self.subTest(case=name): + budget = StreamOutputBudget(0) + tracker = ChatSSEAccumulator(collect=False, retain_tools=True, budget=budget, + declared_names={"synthetic_tool"}) + converter = ResponsesStreamConverter( + model="m", realtime=True, budget=budget, tool_states=tracker.tools, + declared_names={"synthetic_tool"}) + raw = "" + starts = [] + for index, item in enumerate(metadata): + tool = {"index": 0, "type": "function", + "function": {"name": item.get("name", ""), + "arguments": item.get("arguments", "")}} + if item.get("id") is not None: + tool["id"] = item["id"] + row = _line({"tool_calls": [tool]}) + tracker.feed_line(row) + raw += converter.feed_line(row) + starts.extend(event for event in _events(raw) + if event["type"] == "response.output_item.added") + if index == 0 and item.get("id") == "": + self.assertFalse(starts) + terminal = _line({}, "tool_calls", {"total_tokens": 1}) + tracker.feed_line(terminal) + raw += converter.feed_line(terminal) + merged = tracker.result() + converter.set_validated_tools(merged["tool_calls"]) + raw += converter.finish() + events = _events(raw) + added = [event for event in events + if event["type"] == "response.output_item.added" + and event["item"]["type"] == "function_call"] + self.assertEqual(len(added), 1) + self.assertEqual(added[0]["item"]["call_id"], "call_a" if name != "split-id" + and name != "both-fragmented" else "call_abc") + self.assertEqual(added[0]["item"]["name"], "synthetic_tool") + arguments = "".join(event["delta"] for event in events + if event["type"] == "response.function_call_arguments.delta") + self.assertEqual(arguments, '{"x":' if name == "arguments-before-metadata" else "{}") + self.assertEqual(merged["tool_calls"][0]["id"], added[0]["item"]["call_id"]) + + if name != "arguments-before-metadata": + with self.assertRaises(httpx.RemoteProtocolError): + tracker.feed_line(_line({"tool_calls": [{ + "index": 0, "id": "other", "function": {"name": "synthetic_tool"}}]})) + + def test_tool_identity_fragments_preserve_repeated_prefixes(self): + for cls in (ResponsesStreamConverter, AnthropicStreamConverter): + for field, values in (("name", ("test", "aaaa", "tool_tool", "lookup")), + ("id", ("call_abc", "call_call", "aaaa"))): + for value in values: + for split in range(1, len(value)): + with self.subTest(adapter=cls.__name__, field=field, value=value, split=split): + name = value if field == "name" else "lookup" + identifier = value if field == "id" else "call_x" + tracker = ChatSSEAccumulator(collect=False, retain_tools=True, + declared_names={name}) + adapter = cls(realtime=True, tool_states=tracker.tools, declared_names={name}) + first = {"index": 0, "id": identifier, "function": {"name": name}} + last = {"index": 0, "function": {"arguments": "{}"}} + if field == "name": + first["function"][field] = value[:split] + last["function"][field] = value[split:] + else: + first[field], last[field] = value[:split], value[split:] + raw = "" + for tool in (first, last): + row = _line({"tool_calls": [tool]}) + tracker.feed_line(row) + raw += adapter.feed_line(row) + row = _line({}, "tool_calls") + tracker.feed_line(row) + raw += adapter.feed_line(row) + calls = tracker.result()["tool_calls"] + self.assertEqual(calls[0]["id"], identifier) + self.assertEqual(calls[0]["function"]["name"], name) + converter._validate_realtime_tools(calls, {"tools": [{ + "type": "function", "function": {"name": name}}]}, "tool_calls") + adapter.set_validated_tools(calls) + raw += adapter.finish() + if cls is ResponsesStreamConverter: + start = next(event["item"] for event in _events(raw) + if event["type"] == "response.output_item.added") + self.assertEqual((start["call_id"], start["name"]), (identifier, name)) + else: + start = next(event["content_block"] for event in _events(raw) + if event["type"] == "content_block_start") + self.assertEqual((start["id"], start["name"]), (identifier, name)) + + + def test_complete_metadata_streams_without_guessing_id_or_name_prefixes(self): + cases = [("read", "call_ok", {"read", "read_file"}), + ("lookup", "call_", {"lookup"}), ("tool_", "call_ok", {"tool_"})] + for cls in (ResponsesStreamConverter, AnthropicStreamConverter): + for name, identifier, declared in cases: + with self.subTest(adapter=cls.__name__, name=name, identifier=identifier): + tracker = ChatSSEAccumulator(collect=False, retain_tools=True, + declared_names=declared) + adapter = cls(realtime=True, tool_states=tracker.tools, declared_names=declared) + row = _line({"tool_calls": [{"index": 0, "id": identifier, + "function": {"name": name, "arguments": '{"x":'}}]}) + tracker.feed_line(row) + events = _events(adapter.feed_line(row)) + kind = ("response.function_call_arguments.delta" if cls is ResponsesStreamConverter + else "content_block_delta") + delta = next((event for event in events if event["type"] == kind), None) + self.assertIsNotNone(delta, "Complete metadata must not defer arguments until EOF") + self.assertEqual(delta["delta"] if cls is ResponsesStreamConverter + else delta["delta"]["partial_json"], '{"x":') + + + def test_standalone_realtime_adapters_merge_normal_metadata(self): + for cls in (ResponsesStreamConverter, AnthropicStreamConverter): + with self.subTest(adapter=cls.__name__): + converter = cls(model="m", realtime=True) + raw = converter.feed_line(_line({"tool_calls": [{ + "index": 0, "id": "call_a", "type": "function", + "function": {"name": "lookup", "arguments": ""}}]})) + self.assertNotIn("content_block_start" if cls is AnthropicStreamConverter + else "response.output_item.added", raw) + raw += converter.feed_line(_line({"tool_calls": [{ + "index": 0, "function": {"arguments": "{}"}}]})) + self.assertIn("content_block_start" if cls is AnthropicStreamConverter + else "response.output_item.added", raw) + + def test_realtime_terminal_marker_and_choice_boundaries(self): + body = { + "tools": [{"type": "function", "function": {"name": "declared"}}], + "tool_choice": "required", + "parallel_tool_calls": True, + } + valid = {"id": "call_a", "type": "function", + "function": {"name": "declared", "arguments": "{}"}} + with self.assertRaises(UpstreamResponseError): + converter._validate_realtime_tools([valid], body, "stop") + converter._validate_realtime_tools([valid], body, "tool_calls") + for finish in ("length", "content_filter", "refusal"): + converter._validate_realtime_tools([], body, finish) + named = body | {"tool_choice": {"type": "function", "function": {"name": "declared"}}} + converter._validate_realtime_tools([], named, "content_filter") + with self.assertRaises(UpstreamResponseError): + converter._validate_realtime_tools([], named, "stop") + malformed_filtered = {**valid, "function": {"name": "declared", "arguments": "{"}} + with self.assertRaises(UpstreamResponseError): + converter._validate_realtime_tools([malformed_filtered], body, "content_filter") + + tracker = ChatSSEAccumulator(collect=False, retain_tools=True) + tracker.feed_line(_line({"content": "first"})) + tracker.feed_line(_line({}, "stop")) + with self.assertRaises(httpx.RemoteProtocolError): + tracker.feed_line(_line({"content": "late"})) + with self.assertRaises(httpx.RemoteProtocolError): + tracker.feed_line(_line({}, "length")) + for cls in (ResponsesStreamConverter, AnthropicStreamConverter): + with self.subTest(adapter=cls.__name__): + adapter = cls(model="m", realtime=True) + adapter.feed_line(_line({"content": "first"})) + adapter.feed_line(_line({}, "stop")) + with self.assertRaises(ValueError): + adapter.feed_line(_line({"content": "late"})) + marker_adapter = cls(model="m", realtime=True) + marker_adapter.feed_line(_line({"tool_calls": [{ + "index": 0, "id": "call_a", "type": "function", + "function": {"name": "lookup", "arguments": "{}"}}]})) + marker_adapter.feed_line(_line({}, "stop")) + with self.assertRaises(ValueError): + marker_adapter.finish() + + def test_delayed_tool_then_text_uses_emission_order_and_stable_indices(self): + budget = StreamOutputBudget(0) + tracker = ChatSSEAccumulator(collect=False, retain_tools=True, budget=budget, + declared_names={"synthetic_tool"}) + converter = ResponsesStreamConverter(model="m", realtime=True, budget=budget, + tool_states=tracker.tools, + declared_names={"synthetic_tool"}) + rows = [ + _line({"tool_calls": [{"index": 0, "id": "call_a", "function": { + "name": "synthetic_", "arguments": ""}}]}), + _line({"content": "available text"}), + _line({"tool_calls": [{"index": 0, "function": { + "name": "tool", "arguments": "{}"}}]}, "tool_calls"), + "data: [DONE]\n\n", + ] + raw = "" + after_text = "" + for row in rows: + tracker.feed_line(row) + raw += converter.feed_line(row) + if row is rows[1]: + after_text = raw + before_text_events = _events(after_text) + self.assertTrue(any(event["type"] == "response.output_text.delta" + for event in before_text_events)) + self.assertFalse(any(event["type"] == "response.output_item.added" + and event["item"]["type"] == "function_call" + for event in before_text_events)) + merged = tracker.result() + converter.set_validated_tools(merged["tool_calls"]) + raw += converter.finish() + events = _events(raw) + added = [event for event in events if event["type"] == "response.output_item.added"] + self.assertEqual([(event["output_index"], event["item"]["type"]) for event in added], + [(0, "message"), (1, "function_call")]) + final = next(event for event in events if event["type"] == "response.completed") + self.assertEqual([item["type"] for item in final["response"]["output"]], + ["message", "function_call"]) + self.assertEqual(final["response"]["output"][1]["arguments"], "{}") + + def test_utf8_budget_counts_each_logical_fragment_once(self): + budget = StreamOutputBudget(3) + budget.charge_text("你") + with self.assertRaises(UpstreamResponseError): + budget.charge_text("!") + + def test_terminal_tool_validation_rejects_bad_identity_json_names_and_choices(self): + valid = {"id": "call", "type": "function", + "function": {"name": "declared", "arguments": "{}"}} + body = {"tools": [{"type": "function", "function": {"name": "declared"}}], + "tool_choice": "required", "parallel_tool_calls": True} + self.assertTrue(converter._tool_calls_healthy([valid], body)) + self.assertTrue(converter._tool_choice_satisfied([valid], body)) + malformed = [ + None, {}, {**valid, "id": ""}, {**valid, "function": {**valid["function"], "name": "other"}}, + {**valid, "function": {**valid["function"], "arguments": "[]"}}, + {**valid, "function": {**valid["function"], "arguments": "{"}}, + valid, valid, + ] + for calls in malformed: + with self.subTest(calls=calls), self.assertRaises(UpstreamResponseError): + converter._validate_realtime_tools(calls, body) + for choice in ("none", "auto", "required"): + selected = body | {"tool_choice": choice} + expected = choice != "none" + self.assertEqual(converter._tool_choice_satisfied([valid] if expected else [], selected), True) + with self.assertRaises(UpstreamResponseError): + converter._validate_realtime_tools([], {"tools": [], "tool_choice": "required"}) + with self.assertRaises(UpstreamResponseError): + converter._validate_realtime_tools([valid], body | {"tool_choice": "none"}) + second = {**valid, "id": "call_2"} + with self.assertRaises(UpstreamResponseError): + converter._validate_realtime_tools([valid, second], body | {"parallel_tool_calls": False}) + + +class RealtimeAuditTests(unittest.TestCase): + def test_failed_realtime_terminal_is_error_with_mode_and_actual_usage(self): + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + store = AuditStore(root / "audit.sqlite3") + self.addCleanup(store.close) + application = FastAPI() + application.router.routes = list(converter.app.router.routes) + application.add_middleware(AuditMiddleware, store) + response_lines = [ + _line({"tool_calls": [{"index": 0, "id": "call", "type": "function", + "function": {"name": "synthetic_tool", "arguments": "{"}}]}, + usage={"prompt_tokens": 2, "completion_tokens": 3, "total_tokens": 5}), + _line({"tool_calls": [{"index": 0, "function": {"arguments": "invalid"}}]}), + _line({}, "tool_calls")] + + @asynccontextmanager + async def backend(*args, **kwargs): + yield _FixedResponse(response_lines) + + config = {"api_key": "", "model_guard": False, "max_images": 16, + "image_policy": "truncate", "max_request_bytes": 32 * 1024 * 1024, + "max_collect_bytes": 0, "log_path": None, "stream_mode": "realtime"} + with patch.dict(converter.CONFIG, config, clear=False), \ + patch.object(converter, "_route_chat", side_effect=lambda payload, body, rid: (body, None, {}, "https://synthetic.invalid")), \ + patch.object(converter, "_backend_stream", backend), patch.object(converter, "_log"), \ + patch.object(converter, "_note_cred_model_ok"): + client = self.enterContext(TestClient(application)) + response = client.post("/v1/responses", json={ + "model": "auto", "stream": True, "input": "hi", + "tools": [{"type": "function", "name": "synthetic_tool", + "parameters": {"type": "object"}}]}) + self.assertEqual(response.status_code, 200, response.text) + self.assertNotIn("response.completed", response.text) + record = store.list_records()["items"][0] + self.assertEqual((record["outcome"], record["stream_mode"], record["total_tokens"]), + ("error", "realtime", 5)) + self.assertTrue(record["error_code"]) + + +class StreamPolicyTests(unittest.TestCase): + def test_dotenv_overrides_sqlite_and_process_environment_overrides_dotenv(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path = root / ".env" + path.write_text("CODEBUDDY2API_STREAM_MODE=realtime\n", encoding="utf-8") + store = ControlStore(root / "control.sqlite3") + self.addCleanup(store.close) + store.update_settings({"stream_mode": "compatible"}, store.snapshot()["revision"]) + with patch.dict(os.environ, {"CODEBUDDY2API_KEY": "synthetic-test-key"}, clear=True): + dotenv_keys = load_startup_env(path) + config = {"control_store": store} + apply_persisted_settings(config, environ=os.environ) + config["state_store"] = store.state + resolve_startup_key(config, SimpleNamespace(api_key="synthetic-test-key"), dotenv_keys) + item = next(value for value in resolve_settings(config) if value["key"] == "stream_mode") + self.assertEqual((config["stream_mode"], item["source"], item["locked"]), + ("realtime", "dotenv", True)) + + with patch.dict(os.environ, { + "CODEBUDDY2API_KEY": "synthetic-test-key", + "CODEBUDDY2API_STREAM_MODE": "compatible"}, clear=True): + dotenv_keys = load_startup_env(path) + config = {"control_store": store} + apply_persisted_settings(config, environ=os.environ) + config["state_store"] = store.state + resolve_startup_key(config, SimpleNamespace(api_key="synthetic-test-key"), dotenv_keys) + item = next(value for value in resolve_settings(config) if value["key"] == "stream_mode") + self.assertEqual((config["stream_mode"], item["source"]), ("compatible", "environment")) + + def test_snapshot_freezes_mode_budget_and_aggregate_selection(self): + with patch.dict(converter.CONFIG, {"stream_mode": "compatible", "max_collect_bytes": 17}): + chat = converter._snapshot_stream_policy("chat", {"tools": [{}]}) + responses = converter._snapshot_stream_policy("responses", {}) + messages = converter._snapshot_stream_policy("messages", {}) + self.assertTrue(chat.aggregate and responses.aggregate) + self.assertFalse(messages.aggregate) + self.assertEqual(chat.max_collect_bytes, 17) + converter.CONFIG.update(stream_mode="realtime", max_collect_bytes=99) + self.assertEqual(chat.mode, "compatible") + self.assertEqual(chat.max_collect_bytes, 17) + self.assertFalse(converter._snapshot_stream_policy("responses", {}).aggregate) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_webui_integration.py b/tests/test_webui_integration.py index dd9da9d..c088843 100644 --- a/tests/test_webui_integration.py +++ b/tests/test_webui_integration.py @@ -273,6 +273,20 @@ def test_managed_export_excludes_databases_and_tokens_never_reach_audit(self): self.assertNotIn(b"synthetic-access", path.read_bytes()) self.assertNotIn(b"synthetic-management-key", path.read_bytes()) + def test_managed_stream_mode_is_an_unlocked_hot_enum(self): + revision = self.control.snapshot()['revision'] + response = self.client.patch('/admin/settings', json={ + 'revision': revision, 'values': {'stream_mode': 'realtime'}}) + self.assertEqual(response.status_code, 200, response.text) + item = next(value for value in response.json()['items'] if value['key'] == 'stream_mode') + self.assertEqual((item['value'], item['source'], item['mode'], item['locked']), ( + 'realtime', 'management', 'hot', False)) + self.assertEqual(item['choices'], ['compatible', 'realtime']) + self.assertIn('不重生成工具参数', item['label']) + invalid = self.client.patch('/admin/settings', json={ + 'revision': self.control.snapshot()['revision'], 'values': {'stream_mode': 'invalid'}}) + self.assertEqual(invalid.status_code, 400, invalid.text) + def test_managed_zero_price_setting_is_not_replaced_by_default(self): response = self.client.patch("/admin/settings", json={"revision": self.control.snapshot()["revision"], "values": {"credit_price_cny": 0, "credit_price_usd": 0}}) From 3968facb184ee841514c9ffcc61f6cb07ec821d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A2=A8=E8=8F=8A?= <277378677+maiphucgiang@users.noreply.github.com> Date: Thu, 24 Sep 2026 06:14:54 +0800 Subject: [PATCH 2/3] Preserve request policies and filter-only streaming outcomes --- app/upstream_io.py | 7 +- converter.py | 39 ++++----- docs/advanced.md | 4 +- docs/advanced.zh-CN.md | 4 +- tests/test_realtime_streaming.py | 133 +++++++++++++++++++++++++++++++ 5 files changed, 159 insertions(+), 28 deletions(-) diff --git a/app/upstream_io.py b/app/upstream_io.py index 4d7e7d0..b6a82d9 100644 --- a/app/upstream_io.py +++ b/app/upstream_io.py @@ -331,12 +331,13 @@ def validated_tool_calls(self): "function": {"name": value["name"], "arguments": value["arguments"]}} for _, value in sorted(self.tools.items())] - def result(self): + def result(self, *, allow_empty_filter=False): if not self.saw_choice or not (self.done or self.finish_reason is not None): raise httpx.RemoteProtocolError("Upstream SSE ended without a completion marker") seal_tool_identities(self.tools, self.declared_names) - if not self.saw_output: - if self.finish_reason in ("content_filter", "content-filter", "refusal"): + filter_terminal = self.finish_reason in ("content_filter", "content-filter", "refusal") + if not self.saw_output and not (allow_empty_filter and filter_terminal): + if filter_terminal: raw = {"error": {"type": "upstream_error", "code": self.finish_reason, "message": "Upstream rejected the response without output"}} raise UpstreamResponseError(502, json.dumps(raw).encode("utf-8")) diff --git a/converter.py b/converter.py index a435c70..01f6454 100644 --- a/converter.py +++ b/converter.py @@ -2795,9 +2795,8 @@ async def chat_completions(request: Request, client_wants_stream = _client_wants_stream(payload) body = {k: payload[k] for k in PASSTHROUGH_BODY_KEYS if k in payload} body = await run_in_threadpool(_prepare_chat_body, body, session_payload=payload) - stream_policy = (_snapshot_stream_policy("chat", body) if client_wants_stream else None) - if stream_policy is not None: - observe_stream_mode(stream_policy.mode) + stream_policy = _snapshot_stream_policy("chat", body) + observe_stream_mode(stream_policy.mode) # Record request metadata. model_name = payload.get("model", "auto") @@ -2814,7 +2813,7 @@ async def chat_completions(request: Request, _log_json(f"[{rid}] REQUEST BODY (发往后端，预览)", body) t0 = time.time() - if stream_policy is not None: + if client_wants_stream: def attempt(routed, cred, headers, url): return _stream_upstream(url, headers, _body_with_stream_policy(routed, stream_policy), model_name, t0, rid, cred=cred) @@ -2824,7 +2823,7 @@ def attempt(routed, cred, headers, url): # Aggregate upstream SSE for non-streaming clients. async def fetch(routed, cred, headers, url): return await _fetch_checked_chat(url, headers, routed, model_name, rid, cred, - filter_retry=True) + filter_retry=True, max_collect_bytes=stream_policy.max_collect_bytes) # Watch for disconnects across the entire failover sequence. try: collected = await await_or_hangup( @@ -3146,12 +3145,7 @@ def _hungup_response(rid, model_name, t0): async def _fetch_checked_chat(url, headers, body, model_name, rid, cred=None, *, filter_retry=False, max_collect_bytes=None): - """Collect and validate replies with bounded tool repair and one eligible filter fallback. - - ``max_collect_bytes`` is an optional frozen request-policy value. Streaming - callers pass it explicitly so hot configuration changes cannot enlarge a - collection or one of its repair attempts after the request has started. - """ + """Collect with bounded repair and one eligible filter retry using a frozen request budget.""" tool_attempt = 0 filter_retried = False collection_limit = (CONFIG.get("max_collect_bytes", 0) if max_collect_bytes is None @@ -3235,7 +3229,7 @@ async def _chat_sse_lines(url, headers, body, model_name, t0, rid, cred=None, *, state["tracker"] = tracker def completed(): - merged = tracker.result() + merged = tracker.result(allow_empty_filter=policy.realtime) if policy.realtime: _validate_realtime_tools( merged.get("tool_calls"), body, merged.get("finish_reason"), @@ -3597,9 +3591,8 @@ async def create_response(request: Request, chat_body = await run_in_threadpool(_prepare_chat_body, chat_body) client_wants_stream = _client_wants_stream(payload) - stream_policy = (_snapshot_stream_policy("responses", chat_body) if client_wants_stream else None) - if stream_policy is not None: - observe_stream_mode(stream_policy.mode) + stream_policy = _snapshot_stream_policy("responses", chat_body) + observe_stream_mode(stream_policy.mode) model_name = payload.get("model", "auto") rid = _request_id() _log(f"[{rid}] ▶ RESPONSES {model_name} | stream={client_wants_stream} | input_items={len(payload.get('input', []))}") @@ -3620,7 +3613,7 @@ async def create_response(request: Request, _log_json(f"[{rid}] RESPONSES → CHAT BODY (预览)", chat_body) t0 = time.time() - if stream_policy is not None: + if client_wants_stream: def attempt(routed, cred, headers, url): return _stream_responses(url, headers, _body_with_stream_policy(routed, stream_policy), model_name, t0, rid, cred=cred) @@ -3628,16 +3621,17 @@ def attempt(routed, cred, headers, url): chat_body, cred, headers, url) return await _nonstream_adapted(url, headers, chat_body, model_name, t0, rid, cred, - payload=payload, canonical=prepared, request=request) + payload=payload, canonical=prepared, request=request, policy=stream_policy) async def _nonstream_adapted(url, headers, body, model_name, t0, rid, cred, *, anthropic=False, - payload=None, canonical=None, request=None): + payload=None, canonical=None, request=None, policy=None): + policy = policy or _snapshot_stream_policy("messages" if anthropic else "responses", body) converter = (AnthropicStreamConverter(model=model_name) if anthropic else ResponsesStreamConverter(model=model_name, parallel_tool_calls=body.get("parallel_tool_calls", True))) async def fetch(routed, cred, headers, url): return await _fetch_checked_chat(url, headers, routed, model_name, rid, cred, - filter_retry=True) + filter_retry=True, max_collect_bytes=policy.max_collect_bytes) try: collected = await await_or_hangup( _routed_fetch(payload, body if canonical is None else canonical, @@ -3764,9 +3758,8 @@ async def create_message(request: Request, chat_body = await run_in_threadpool(_prepare_chat_body, chat_body, session_payload=payload) client_wants_stream = _client_wants_stream(payload) - stream_policy = (_snapshot_stream_policy("messages", chat_body) if client_wants_stream else None) - if stream_policy is not None: - observe_stream_mode(stream_policy.mode) + stream_policy = _snapshot_stream_policy("messages", chat_body) + observe_stream_mode(stream_policy.mode) model_name = payload.get("model", "auto") chat_messages = chat_body.get("messages", []) rid = _request_id() @@ -3780,7 +3773,7 @@ async def create_message(request: Request, if not client_wants_stream: return await _nonstream_adapted(url, headers, chat_body, model_name, t0, rid, cred, anthropic=True, payload=payload, canonical=prepared, - request=request) + request=request, policy=stream_policy) def attempt(routed, cred, headers, url): return _stream_anthropic(url, headers, _body_with_stream_policy(routed, stream_policy), diff --git a/docs/advanced.md b/docs/advanced.md index 53c9d41..cea89f5 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -57,13 +57,15 @@ Use a source/image build and Compose configuration containing this feature; recr ### Streaming modes -`stream_mode` defaults to `compatible`. Configure it as `--stream-mode compatible|realtime`, `CODEBUDDY2API_STREAM_MODE`, or the hot **Streaming mode / 实时模式** enum in the WebUI. Explicit CLI and environment sources lock the WebUI field. The selected value and `max_collect_bytes` are frozen when each request starts, so a hot change affects only later requests and never an in-flight failover path. There is no per-request override. +`stream_mode` defaults to `compatible`. Configure it through `--stream-mode compatible|realtime`, `CODEBUDDY2API_STREAM_MODE`, or the WebUI enum; explicit CLI/environment sources lock that field. Every generation request, including non-streaming, records the selected mode and freezes it with `max_collect_bytes` before routing. Hot changes affect only later requests, not in-flight failovers. Non-streaming responses remain aggregated JSON; there is no client mode override. - `compatible` preserves existing behavior: Responses streams aggregate first; Chat and Messages aggregate when tools are present and otherwise pass through upstream increments. Aggregated output is validated and replayed in fragments. Non-stream requests always use the validated aggregate path in either mode. - `realtime` forwards reasoning, text, refusal and tool-argument increments for all three protocols. Responses assigns stable indexes when items start; Anthropic uses stable block indexes. Adapters buffer tools with missing identity until the argument phase or terminal marker, append metadata fragments without guessing from prefixes, and reject identity changes after an item starts. `max_collect_bytes` bounds retained UTF-8 output; `0` disables that limit. Realtime mode never regenerates malformed or incomplete tool arguments. Tool IDs, names, declared names, JSON-object arguments and `tool_choice` are checked at the terminal boundary before a success terminal is sent. In realtime, a tool-bearing completion must also carry the upstream `tool_calls` finish marker; a `stop` marker with tool calls is rejected, while compatible mode keeps its legacy acceptance behavior. Before any downstream byte, failures retain the upstream HTTP error and existing bounded pre-response failover rules. After any byte, malformed tools, disconnects, stream errors and budget overflow produce a protocol error terminal without credential replay or switching; valid `length`, refusal and content-filter results keep their native protocol distinctions (Responses reports truncation/filtering as `incomplete`, never `completed`) and are not regenerated. Clients must therefore accept partial output followed by an error rather than assuming every opened SSE stream completes successfully. Audit records retain only an allowlisted `stream_mode` marker plus available upstream usage. +Explicit filter-only terminals are valid even without output: realtime Responses emits `response.incomplete` with `content_filter`, retaining available usage. This does not permit an ordinary empty response, missing terminal or error frame to succeed. + For runtime fallback, select `compatible` to restore aggregate streaming and tool-argument repair without changing saved state. Before running older source, remove the new CLI/environment option, stop the gateway and back up the **current** data directory, including its SQLite/WAL/SHM generation. Do not restore a stale pre-upgrade database: that could roll back newer claims, sessions, revocations and account state. The following offline procedure writes only the removal of `settings.stream_mode` and a revision increment, then checks integrity; all other settings and tables remain intact. Never run it against a live database or mix SQLite generations. ```sh diff --git a/docs/advanced.zh-CN.md b/docs/advanced.zh-CN.md index 4b8893c..27a3a20 100644 --- a/docs/advanced.zh-CN.md +++ b/docs/advanced.zh-CN.md @@ -57,13 +57,15 @@ Compose 会显式传入部分环境变量及 CLI 参数，删除 `.env` 中的 ### 流式模式 -`stream_mode` 默认 `compatible`，可通过 `--stream-mode compatible|realtime`、`CODEBUDDY2API_STREAM_MODE` 或 WebUI 热更新枚举「流式模式」配置。显式 CLI/环境来源会锁定 WebUI 项。每个请求入口都会冻结此选择及 `max_collect_bytes`，因此热更新只影响后续请求，不改变在途流或其换号路径；客户端不能按请求覆盖。 +`stream_mode` 默认 `compatible`，可通过 `--stream-mode compatible|realtime`、`CODEBUDDY2API_STREAM_MODE` 或 WebUI 枚举配置；显式 CLI/环境来源会锁定该项。所有生成请求（含非流式）均在选路前冻结模式及 `max_collect_bytes`，并记录所选模式。热更新只影响后续请求，不改变在途请求及换号；非流式仍返回聚合 JSON，客户端不能按请求覆盖模式。 - `compatible` 保持现有行为：Responses 流式先聚合；Chat/Messages 带工具时先聚合，无工具时沿用上游增量。聚合结果先校验，再按片段重放。两种模式下，非流式请求始终走已校验的聚合路径。 - `realtime` 让三个协议都增量发送思考、正文、拒绝及工具参数。Responses 在输出项开始时分配稳定索引，Anthropic 使用稳定 block index。适配器在参数阶段或结束标记处确认工具身份，缺失时暂缓工具输出；元数据分片按顺序追加，不按字符串前缀猜测，输出项开始后禁止更换身份。`max_collect_bytes` 约束所保留的 UTF-8 输出，`0` 不限制。 实时模式绝不重生成损坏或不完整的工具参数。发送成功终端前，会在终端边界校验工具 ID、名称、已声明名称、JSON object 参数及 `tool_choice`。实时模式下，存在工具调用时还必须带上游 `tool_calls` 结束标记；带工具却标为 `stop` 会拒绝，而 compatible 模式保留旧的兼容接受行为。下游尚未收到字节时，失败保留真实上游 HTTP 状态及既有、有界的响应前换号规则；已发送任何字节后，参数损坏、断连、流错误和预算超限均以协议错误终端结束，不重放或切换凭据。合法的 `length`、拒绝和审核结果保留各自协议区别（Responses 的截断/过滤为 `incomplete`，绝非 `completed`），且不会重生成。因此客户端必须接受「部分正文后跟错误」，不能假定已开流的 SSE 一定成功结束。审计只增加白名单 `stream_mode` 标记及上游实际提供的用量。 +明确的纯审核拒绝即使没有正文也保留原生终态：实时 Responses 返回 `response.incomplete`，原因是 `content_filter`，并保留已有用量。普通空响应、缺少终止标记及错误帧仍不得伪装成功。 + 运行时选回 `compatible` 即可恢复聚合流式及工具参数重生成，保留当前保存状态。源码降级前，移除新增 CLI/环境选项，停止网关并备份**当前**数据目录及同一代 SQLite/WAL/SHM。不要恢复升级前旧库，否则可能回滚新的领取、会话、撤销和账号状态。下例会离线写入当前数据库：仅删除 `settings.stream_mode`、递增 revision 并检查完整性，其它设置和表保持不变；禁止对运行中的数据库执行或混用 SQLite 文件代数。 ```sh diff --git a/tests/test_realtime_streaming.py b/tests/test_realtime_streaming.py index 7311fa4..55f1db7 100644 --- a/tests/test_realtime_streaming.py +++ b/tests/test_realtime_streaming.py @@ -519,6 +519,32 @@ async def test_realtime_native_length_and_filter_distinctions_do_not_replay(self else: self.assertIn(b"data: [DONE]", wire) + async def test_realtime_filter_only_terminals_keep_protocol_status_and_usage(self): + for protocol in ("chat", "responses", "messages"): + for reason in ("content_filter", "content-filter", "refusal"): + for start_frame in (False, True): + with self.subTest(protocol=protocol, reason=reason, start_frame=start_frame): + rows = ([_line({"role": "assistant"}, "")] if start_frame else []) + rows += [_line({}, reason, {"total_tokens": 2}), "data: [DONE]\n\n"] + wire, attempts, failure = await self.collect(protocol, _FixedResponse(rows)) + self.assertEqual(attempts, 1) + self.assertIsNone(failure) + self.assertNotIn(b'"error"', wire) + self.assertNotIn(b"response.completed", wire) + if protocol == "responses": + response = next(event["response"] for event in _events(wire.decode()) + if event["type"] == "response.incomplete") + self.assertEqual(response["output"], []) + self.assertEqual(response["incomplete_details"]["reason"], "content_filter") + self.assertEqual(response["usage"]["total_tokens"], 2) + elif protocol == "chat": + self.assertIn(b"data: [DONE]", wire) + self.assertIn(reason.encode(), wire) + else: + self.assertIn(b"message_stop", wire) + self.assertIn(b"end_turn", wire) + + async def test_realtime_budget_error_after_output_has_no_success_terminal(self): for protocol in ("chat", "responses", "messages"): with self.subTest(protocol=protocol): @@ -599,6 +625,44 @@ async def backend(*args, **kwargs): self.assertTrue(failure or b'"error"' in wire) + async def test_nonstream_failover_retains_entry_budget_across_hot_changes(self): + for protocol in ("chat", "responses", "messages"): + for mode in ("compatible", "realtime"): + for explicit in (False, True): + with self.subTest(protocol=protocol, mode=mode, explicit_stream_false=explicit): + attempts = [] + credentials = [object(), object()] + def route(payload, body, rid, *, tried=()): + # Routing runs after entry policy capture, before the first collection. + converter.CONFIG["max_collect_bytes"] = 64 + return body, credentials[len(tried)], {}, "https://synthetic.invalid" + @asynccontextmanager + async def backend(url, headers, body, **kwargs): + json.dumps(body) # Policy snapshots must not leak onto the wire. + attempts.append(body) + if len(attempts) == 1: + converter.CONFIG["max_collect_bytes"] = 128 + yield _FixedResponse(status=503, body=b'{"error":{"message":"busy"}}') + else: + yield _FixedResponse([_line({"content": "oversize"}), _line({}, "stop")]) + body = self.payload(protocol, tools=False) + if explicit: + body["stream"] = False + else: + body.pop("stream") + path = {"chat": "/v1/chat/completions", "responses": "/v1/responses", + "messages": "/v1/messages"}[protocol] + sent = await self.asgi_post(path, body, backend, route=route, config={ + "stream_mode": mode, "max_collect_bytes": 4, "failover_max": 1}) + self.assertEqual(len(attempts), 2) + self.assertEqual(sent[0]["status"], 502) + self.assertIn(b"response_too_large", b"".join(m.get("body", b"") for m in sent)) + # A later request sees the new budget, while the prior request did not. + sent = await self.asgi_post(path, body, backend, route=route, config={ + "stream_mode": mode, "max_collect_bytes": 128, "failover_max": 1}) + self.assertEqual(sent[0]["status"], 200) + + async def test_protocol_mode_and_tool_matrix_has_no_aggregate_in_realtime(self): for protocol in ("chat", "responses", "messages"): for tools in (False, True): @@ -1048,6 +1112,75 @@ def test_terminal_tool_validation_rejects_bad_identity_json_names_and_choices(se class RealtimeAuditTests(unittest.TestCase): + def test_all_generation_audits_record_the_entry_mode(self): + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + store = AuditStore(root / "audit.sqlite3") + self.addCleanup(store.close) + application = FastAPI() + application.router.routes = list(converter.app.router.routes) + application.add_middleware(AuditMiddleware, store) + @asynccontextmanager + async def backend(*args, **kwargs): + yield _FixedResponse([_line({"content": "ok"}), _line({}, "stop", {"total_tokens": 2})]) + config = {"api_key": "", "model_guard": False, "max_images": 16, "image_policy": "truncate", + "max_request_bytes": 32 * 1024 * 1024, "max_collect_bytes": 128, "log_path": None} + with patch.dict(converter.CONFIG, config), patch.object(converter, "_backend_stream", backend), \ + patch.object(converter, "_log"), patch.object(converter, "_note_cred_model_ok"): + client = self.enterContext(TestClient(application)) + for protocol in ("chat", "responses", "messages"): + for mode in ("compatible", "realtime"): + for stream in (None, False, True): + with self.subTest(protocol=protocol, mode=mode, stream=stream): + converter.CONFIG["stream_mode"] = mode + def route(payload, body, rid): + converter.CONFIG["stream_mode"] = "realtime" if mode == "compatible" else "compatible" + return body, None, {}, "https://synthetic.invalid" + body = {"model": "auto", "max_tokens": 32} + body.update({"input": "hi"} if protocol == "responses" else { + "messages": [{"role": "user", "content": "hi"}]}) + if stream is not None: + body["stream"] = stream + path = {"chat": "/v1/chat/completions", "responses": "/v1/responses", + "messages": "/v1/messages"}[protocol] + with patch.object(converter, "_route_chat", side_effect=route): + response = client.post(path, json=body) + self.assertEqual(response.status_code, 200, response.text) + record = store.list_records()["items"][0] + self.assertEqual(record["stream_mode"], mode) + self.assertEqual(record["outcome"], "success") + + + def test_empty_realtime_filter_audits_as_rejected_without_replay(self): + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + store = AuditStore(root / "audit.sqlite3") + self.addCleanup(store.close) + application = FastAPI() + application.router.routes = list(converter.app.router.routes) + application.add_middleware(AuditMiddleware, store) + attempts = [] + @asynccontextmanager + async def backend(*args, **kwargs): + attempts.append(1) + yield _FixedResponse([_line({}, "content_filter", {"total_tokens": 2})]) + config = {"api_key": "", "model_guard": False, "max_images": 16, "image_policy": "truncate", + "max_request_bytes": 32 * 1024 * 1024, "max_collect_bytes": 128, + "log_path": None, "stream_mode": "realtime", "failover_max": 2} + with patch.dict(converter.CONFIG, config), patch.object(converter, "_backend_stream", backend), \ + patch.object(converter, "_route_chat", side_effect=lambda payload, body, rid: (body, None, {}, "https://synthetic.invalid")), \ + patch.object(converter, "_log"), patch.object(converter, "_note_cred_model_ok"): + client = self.enterContext(TestClient(application)) + response = client.post("/v1/responses", json={"model": "auto", "input": "hi", "stream": True}) + self.assertEqual(response.status_code, 200) + self.assertIn("response.incomplete", response.text) + self.assertNotIn("response.completed", response.text) + self.assertEqual(len(attempts), 1) + record = store.list_records()["items"][0] + self.assertEqual((record["outcome"], record["stream_mode"], record["total_tokens"]), + ("error", "realtime", 2)) + self.assertEqual(record["error_code"], "response_incomplete") + self.assertTrue(any(attempt["stage"] == "content_filter" for attempt in record["attempts"])) + + def test_failed_realtime_terminal_is_error_with_mode_and_actual_usage(self): root = Path(self.enterContext(tempfile.TemporaryDirectory())) store = AuditStore(root / "audit.sqlite3") From 4d474148a0b0b07ded90678565725fffd253be65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A2=A8=E8=8F=8A?= <277378677+maiphucgiang@users.noreply.github.com> Date: Thu, 24 Sep 2026 06:35:09 +0800 Subject: [PATCH 3/3] Serialize realtime Messages content block lifecycles --- app/adapters/anthropic_adapter.py | 46 +++++++++- app/upstream_io.py | 2 +- docs/advanced.md | 2 + docs/advanced.zh-CN.md | 2 + tests/test_realtime_streaming.py | 140 +++++++++++++++++++++++++++++- 5 files changed, 188 insertions(+), 4 deletions(-) diff --git a/app/adapters/anthropic_adapter.py b/app/adapters/anthropic_adapter.py index 1be6885..d33d65a 100644 --- a/app/adapters/anthropic_adapter.py +++ b/app/adapters/anthropic_adapter.py @@ -268,6 +268,8 @@ def __init__(self, model: str = "unknown", *, realtime: bool = False, # Tool blocks indexed by upstream call position. self._tool_uses: dict[int, dict] = {} self._next_block_idx = 0 + self._active_wire_block: int | None = None + self._deferred_blocks: dict[int, dict] = {} # Completion metadata self._finish_reason: str | None = None @@ -628,7 +630,49 @@ def _new_tool_arguments(self, slot: dict) -> str: def _evt(self, event_type: str, data: dict) -> str: """Format an Anthropic SSE event with its event name.""" payload = {"type": event_type, **data} - return f"event: {event_type}\ndata: {json.dumps(payload, ensure_ascii=False)}\n\n" + wire = f"event: {event_type}\ndata: {json.dumps(payload, ensure_ascii=False)}\n\n" + if self._realtime: + if event_type.startswith("content_block_"): + return self._serialize_block_event(event_type, data["index"], wire) + if event_type in ("message_delta", "message_stop") and ( + self._active_wire_block is not None or self._deferred_blocks): + raise ValueError("message ended before content blocks closed") + return wire + + def _serialize_block_event(self, kind: str, index: int, wire: str) -> str: + """Keep one downstream block open; later blocks wait within the shared budget.""" + if kind == "content_block_start": + if index == self._active_wire_block or index in self._deferred_blocks: + raise ValueError("duplicate content block start") + if self._active_wire_block is None: + self._active_wire_block = index + return wire + self._budget.charge_text(wire) + self._deferred_blocks[index] = {"events": [wire], "closed": False} + return "" + + if index == self._active_wire_block: + if kind != "content_block_stop": + return wire + self._active_wire_block = None + ready = [wire] + while self._deferred_blocks: + next_index = next(iter(self._deferred_blocks)) + block = self._deferred_blocks.pop(next_index) + ready.extend(block["events"]) + if not block["closed"]: + self._active_wire_block = next_index + break + return "".join(ready) + + block = self._deferred_blocks.get(index) + if block is None or block["closed"]: + raise ValueError("content event outside its block lifecycle") + self._budget.charge_text(wire) + block["events"].append(wire) + if kind == "content_block_stop": + block["closed"] = True + return "" def _build_content_blocks(self) -> list[dict]: """Build content blocks for a non-streaming response.""" diff --git a/app/upstream_io.py b/app/upstream_io.py index b6a82d9..6a63ac6 100644 --- a/app/upstream_io.py +++ b/app/upstream_io.py @@ -36,7 +36,7 @@ def __init__(self, status, raw, *, retry_after=None): class StreamOutputBudget: - """Account for each logical output fragment once across realtime validators and adapters.""" + """Bound retained output and deferred protocol events across validators and adapters.""" def __init__(self, max_bytes: int = 0): self.max_bytes = max(0, int(max_bytes or 0)) diff --git a/docs/advanced.md b/docs/advanced.md index cea89f5..5fc4fee 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -62,6 +62,8 @@ Use a source/image build and Compose configuration containing this feature; recr - `compatible` preserves existing behavior: Responses streams aggregate first; Chat and Messages aggregate when tools are present and otherwise pass through upstream increments. Aggregated output is validated and replayed in fragments. Non-stream requests always use the validated aggregate path in either mode. - `realtime` forwards reasoning, text, refusal and tool-argument increments for all three protocols. Responses assigns stable indexes when items start; Anthropic uses stable block indexes. Adapters buffer tools with missing identity until the argument phase or terminal marker, append metadata fragments without guessing from prefixes, and reject identity changes after an item starts. `max_collect_bytes` bounds retained UTF-8 output; `0` disables that limit. +Realtime Messages keeps one content block open at a time. The active tool remains incremental; later tool, text or thinking blocks may wait until upstream completion. Deferred event bytes share `max_collect_bytes`. A tool still awaiting its identity does not block unrelated text before its block starts. + Realtime mode never regenerates malformed or incomplete tool arguments. Tool IDs, names, declared names, JSON-object arguments and `tool_choice` are checked at the terminal boundary before a success terminal is sent. In realtime, a tool-bearing completion must also carry the upstream `tool_calls` finish marker; a `stop` marker with tool calls is rejected, while compatible mode keeps its legacy acceptance behavior. Before any downstream byte, failures retain the upstream HTTP error and existing bounded pre-response failover rules. After any byte, malformed tools, disconnects, stream errors and budget overflow produce a protocol error terminal without credential replay or switching; valid `length`, refusal and content-filter results keep their native protocol distinctions (Responses reports truncation/filtering as `incomplete`, never `completed`) and are not regenerated. Clients must therefore accept partial output followed by an error rather than assuming every opened SSE stream completes successfully. Audit records retain only an allowlisted `stream_mode` marker plus available upstream usage. Explicit filter-only terminals are valid even without output: realtime Responses emits `response.incomplete` with `content_filter`, retaining available usage. This does not permit an ordinary empty response, missing terminal or error frame to succeed. diff --git a/docs/advanced.zh-CN.md b/docs/advanced.zh-CN.md index 27a3a20..3a7bf73 100644 --- a/docs/advanced.zh-CN.md +++ b/docs/advanced.zh-CN.md @@ -62,6 +62,8 @@ Compose 会显式传入部分环境变量及 CLI 参数，删除 `.env` 中的 - `compatible` 保持现有行为：Responses 流式先聚合；Chat/Messages 带工具时先聚合，无工具时沿用上游增量。聚合结果先校验，再按片段重放。两种模式下，非流式请求始终走已校验的聚合路径。 - `realtime` 让三个协议都增量发送思考、正文、拒绝及工具参数。Responses 在输出项开始时分配稳定索引，Anthropic 使用稳定 block index。适配器在参数阶段或结束标记处确认工具身份，缺失时暂缓工具输出；元数据分片按顺序追加，不按字符串前缀猜测，输出项开始后禁止更换身份。`max_collect_bytes` 约束所保留的 UTF-8 输出，`0` 不限制。 +实时 Messages 同时只打开一个内容块：当前工具仍增量输出，后续工具、正文或思考块可能等待上游结束，暂存事件字节计入 `max_collect_bytes`。尚未确认身份、未开始内容块的工具不会阻塞其它正文。 + 实时模式绝不重生成损坏或不完整的工具参数。发送成功终端前，会在终端边界校验工具 ID、名称、已声明名称、JSON object 参数及 `tool_choice`。实时模式下，存在工具调用时还必须带上游 `tool_calls` 结束标记；带工具却标为 `stop` 会拒绝，而 compatible 模式保留旧的兼容接受行为。下游尚未收到字节时，失败保留真实上游 HTTP 状态及既有、有界的响应前换号规则；已发送任何字节后，参数损坏、断连、流错误和预算超限均以协议错误终端结束，不重放或切换凭据。合法的 `length`、拒绝和审核结果保留各自协议区别（Responses 的截断/过滤为 `incomplete`，绝非 `completed`），且不会重生成。因此客户端必须接受「部分正文后跟错误」，不能假定已开流的 SSE 一定成功结束。审计只增加白名单 `stream_mode` 标记及上游实际提供的用量。 明确的纯审核拒绝即使没有正文也保留原生终态：实时 Responses 返回 `response.incomplete`，原因是 `content_filter`，并保留已有用量。普通空响应、缺少终止标记及错误帧仍不得伪装成功。 diff --git a/tests/test_realtime_streaming.py b/tests/test_realtime_streaming.py index 55f1db7..ad8d4d5 100644 --- a/tests/test_realtime_streaming.py +++ b/tests/test_realtime_streaming.py @@ -48,6 +48,38 @@ def _events(raw): return result +def _serial_blocks(test, raw, *, complete=True): + active = None + blocks = [] + for event in _events(raw): + kind = event["type"] + if kind == "content_block_start": + test.assertIsNone(active, event) + active = event["index"] + test.assertEqual(active, len(blocks)) + blocks.append({**event["content_block"], "_pieces": [], "_closed": False}) + elif kind == "content_block_delta": + test.assertEqual(event["index"], active) + delta = event["delta"] + field = {"text_delta": "text", "thinking_delta": "thinking", + "input_json_delta": "partial_json"}[delta["type"]] + blocks[active]["_pieces"].append(delta[field]) + elif kind == "content_block_stop": + test.assertEqual(event["index"], active) + block = blocks[active] + value = "".join(block["_pieces"]) + block["input" if block["type"] == "tool_use" else block["type"]] = ( + json.loads(value) if block["type"] == "tool_use" else value) + block["_closed"] = True + active = None + elif kind in ("message_delta", "message_stop"): + test.assertIsNone(active) + if complete: + test.assertIsNone(active) + test.assertTrue(all(block["_closed"] for block in blocks)) + return blocks + + class _PausedLines: def __init__(self, first, tail, at_boundary): self.first = first @@ -545,6 +577,46 @@ async def test_realtime_filter_only_terminals_keep_protocol_status_and_usage(sel self.assertIn(b"end_turn", wire) + async def test_messages_parallel_block_queue_keeps_first_tool_live_before_eof(self): + before, wire = await self.drive("messages", "realtime", True, first_delta={"tool_calls": [ + {"index": 0, "id": "first", "function": {"name": "synthetic_tool", "arguments": '{"x":'}}, + {"index": 1, "id": "second", "function": {"name": "synthetic_tool", "arguments": '{}'}}]}) + blocks = _serial_blocks(self, before.decode(), complete=False) + self.assertEqual([block["id"] for block in blocks], ["first"]) + self.assertEqual(blocks[0]["_pieces"], ['{"x":']) + self.assertNotIn(b"message_stop", before) + blocks = _serial_blocks(self, wire.decode()) + self.assertEqual([block["input"] for block in blocks], [{"x": 1}, {}]) + self.assertIn(b"message_stop", wire) + + async def test_messages_deferred_budget_failure_closes_upstream_without_replay(self): + closed = False + attempts = [] + @asynccontextmanager + async def backend(*args, **kwargs): + nonlocal closed + attempts.append(1) + try: + yield _FixedResponse([ + _line({"tool_calls": [{"index": 0, "id": "first", "function": { + "name": "synthetic_tool", "arguments": "{}"}}]}, usage={"total_tokens": 4}), + _line({"tool_calls": [{"index": 1, "id": "second", "function": { + "name": "synthetic_tool", "arguments": "{}"}}]}), + _line({}, "tool_calls")]) + finally: + closed = True + with patch.object(converter, "observe_usage") as usage: + sent = await self.asgi_post("/v1/messages", self.payload("messages", True), backend, + config={"stream_mode": "realtime", "max_collect_bytes": 120}) + self.assertTrue(closed) + self.assertEqual(len(attempts), 1) + usage.assert_any_call({"total_tokens": 4}) + wire = b"".join(event.get("body", b"") for event in sent) + self.assertIn(b"response_too_large", wire) + self.assertNotIn(b"message_stop", wire) + self.assertEqual(len(_serial_blocks(self, wire.decode(), complete=False)), 1) + + async def test_realtime_budget_error_after_output_has_no_success_terminal(self): for protocol in ("chat", "responses", "messages"): with self.subTest(protocol=protocol): @@ -692,6 +764,65 @@ def feed_pair(tracker, converter, delta, finish=None, usage=None): tracker.feed_line(raw) return converter.feed_line(raw) + def test_messages_serializes_parallel_tools_and_following_text_thinking(self): + for shared in (False, True): + with self.subTest(shared_tool_state=shared): + _, tracker, _, adapter = self.realtime_converters() + if not shared: + adapter = AnthropicStreamConverter(realtime=True) + def feed(delta, finish=None): + return self.feed_pair(tracker, adapter, delta, finish) + raw = feed({"tool_calls": [{"index": 0, "id": "first", + "function": {"name": "synthetic_tool", "arguments": '{"a":'}}]}) + blocked = feed({"tool_calls": [{"index": 1, "id": "second", + "function": {"name": "synthetic_tool", "arguments": '{"b":'}}]}) + blocked += feed({"reasoning_content": "later-thought"}) + blocked += feed({"content": "later-text"}) + self.assertEqual(blocked, "") + active_delta = feed({"tool_calls": [{"index": 0, "function": {"arguments": '1}'}}]}) + self.assertIn('1}', active_delta) + raw += active_delta + self.assertEqual(len(_serial_blocks(self, raw, complete=False)), 1) + self.assertEqual(feed({"tool_calls": [{"index": 1, "function": {"arguments": '2}'}}]}), "") + raw += feed({}, "tool_calls") + adapter.set_validated_tools(tracker.result()["tool_calls"]) + raw += adapter.finish() + blocks = _serial_blocks(self, raw) + self.assertEqual([b["type"] for b in blocks], ["tool_use", "tool_use", "thinking", "text"]) + self.assertEqual([b["input"] for b in blocks[:2]], [{"a": 1}, {"b": 2}]) + self.assertEqual(blocks[2]["thinking"], "later-thought") + self.assertEqual(blocks[3]["text"], "later-text") + + def test_messages_delayed_identity_does_not_hold_unrelated_text(self): + _, tracker, _, adapter = self.realtime_converters() + raw = self.feed_pair(tracker, adapter, {"tool_calls": [ + {"index": 0, "function": {"arguments": '{"v":1}'}}]}) + text = self.feed_pair(tracker, adapter, {"content": "visible-now"}) + self.assertIn("visible-now", text) + raw += text + raw += self.feed_pair(tracker, adapter, {"tool_calls": [ + {"index": 0, "id": "late", "function": {"name": "synthetic_tool"}}]}) + raw += self.feed_pair(tracker, adapter, {}, "tool_calls") + adapter.set_validated_tools(tracker.result()["tool_calls"]) + raw += adapter.finish() + blocks = _serial_blocks(self, raw) + self.assertEqual([b["type"] for b in blocks], ["text", "tool_use"]) + self.assertEqual(blocks[1]["input"], {"v": 1}) + + def test_messages_deferred_event_bytes_share_the_request_budget(self): + budget = StreamOutputBudget(120) + tracker = ChatSSEAccumulator(collect=False, retain_tools=True, budget=budget) + adapter = AnthropicStreamConverter(realtime=True, budget=budget, tool_states=tracker.tools) + self.feed_pair(tracker, adapter, {"tool_calls": [ + {"index": 0, "id": "a", "function": {"name": "t", "arguments": '{}'}}]}) + next_tool = _line({"tool_calls": [ + {"index": 1, "id": "b", "function": {"name": "t", "arguments": '{}'}}]}) + tracker.feed_line(next_tool) + with self.assertRaises(UpstreamResponseError) as raised: + adapter.feed_line(next_tool) + self.assertIn(b"response_too_large", raised.exception.raw) + + def test_responses_uses_first_seen_order_and_never_reuses_an_index(self): _, tracker, converter, _ = self.realtime_converters() raw = "" @@ -793,7 +924,8 @@ def test_anthropic_interleaved_parallel_tools_keep_stable_block_identities(self) self.assertEqual(arguments[1], ['{"x":', "1}"]) self.assertEqual(arguments[2], ['{"y":', "2}"]) stops = [event["index"] for event in events if event["type"] == "content_block_stop"] - self.assertEqual(stops, [0, 3, 4, 1, 2]) + self.assertEqual(stops, [0, 1, 2, 3, 4]) + _serial_blocks(self, raw) def test_responses_tool_start_and_done_reconstruct_arguments_once(self): complete = [ @@ -1127,6 +1259,7 @@ async def backend(*args, **kwargs): with patch.dict(converter.CONFIG, config), patch.object(converter, "_backend_stream", backend), \ patch.object(converter, "_log"), patch.object(converter, "_note_cred_model_ok"): client = self.enterContext(TestClient(application)) + seen = set() for protocol in ("chat", "responses", "messages"): for mode in ("compatible", "realtime"): for stream in (None, False, True): @@ -1145,7 +1278,10 @@ def route(payload, body, rid): with patch.object(converter, "_route_chat", side_effect=route): response = client.post(path, json=body) self.assertEqual(response.status_code, 200, response.text) - record = store.list_records()["items"][0] + records = [item for item in store.list_records()["items"] if item["id"] not in seen] + self.assertEqual(len(records), 1) + record = records[0] + seen.add(record["id"]) self.assertEqual(record["stream_mode"], mode) self.assertEqual(record["outcome"], "success")