From 525a2574bae95090a29bb0392e8d675fb5b554c2 Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 14:19:24 -0500 Subject: [PATCH 01/37] fix(llm): bind replayed thinking blocks with drop_block Claude and Bedrock requests that replay thinking blocks under adaptive or enabled thinking set thinking.block_binding.prefix_mismatch_behavior to drop_block and send the thinking-binding-controls-2026-08-01 beta, so the API drops blocks bound to a different conversation instead of rejecting the request. Responses report input_transformations in metadata. The Claude stream client keeps redacted_thinking blocks and returns thinking blocks and tool calls in response order. --- src/llm/src/README.md | 14 + src/llm/src/bedrock/client.lua | 13 +- src/llm/src/bedrock/generate.lua | 4 +- src/llm/src/bedrock/mapper.lua | 29 ++ src/llm/src/bedrock/structured_output.lua | 2 + src/llm/src/claude/client.lua | 108 +++--- src/llm/src/claude/generate.lua | 5 +- src/llm/src/claude/mapper.lua | 40 ++- src/llm/src/claude/structured_output.lua | 9 +- src/llm/src/util/thinking.lua | 18 + src/llm/test/_index.yaml | 2 + src/llm/test/bedrock/client_test.lua | 46 +++ src/llm/test/bedrock/generate_test.lua | 151 ++++++++ .../test/bedrock/structured_output_test.lua | 48 +++ src/llm/test/claude/client_test.lua | 185 ++++++++++ src/llm/test/claude/generate_test.lua | 340 ++++++++++++++++++ .../test/claude/structured_output_test.lua | 62 ++++ 17 files changed, 1015 insertions(+), 61 deletions(-) diff --git a/src/llm/src/README.md b/src/llm/src/README.md index c05dc889..ff7c13de 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -99,6 +99,20 @@ Legacy forms remain supported through the normalizer: Provider open context composition is unchanged: provider entry `driver.options`, then route `context`, then route `options`. Request defaults come from route `options`, with caller options on top. Timeout and retry retain their existing transport handling. Embed and evaluate paths do not normalize route facts. +### Thinking block binding + +The Claude driver stores each response's `thinking` and `redacted_thinking` blocks in `metadata.thinking_blocks`, in response order and in the shape the API returned them, and replays them verbatim on later turns; Bedrock does the same for signed `reasoningContent` text. A Claude thinking block is bound to the conversation that produced it: the top-level system prompt, the tools, and every earlier message. After an agent handoff, a prompt or tool change, or a history edit, models that enforce the check reject a request that replays such a block with a 400. + +When a request replays at least one thinking block and sends a thinking object of type `adaptive` or `enabled` (a positive thinking_effort), the driver sets `thinking.block_binding.prefix_mismatch_behavior = "drop_block"` and adds the `thinking-binding-controls-2026-08-01` beta. The API then drops the first mismatched block and every later thinking block instead of rejecting the request. The Claude driver appends the beta to the `anthropic-beta` header after the configured `beta_features`, each named once. Bedrock sends it in the `anthropic_beta` field of `additionalModelRequestFields` and requests `/input_transformations` through `additionalModelResponseFieldPaths`. Plain generation and structured output behave the same. + +A request without a thinking object (zero or absent effort, or `thinking = "none"`) carries no binding: `block_binding` exists only inside an adaptive or enabled thinking object, and the driver never turns thinking on to carry it, since on several models an omitted thinking object means no thinking. Such a request replays its blocks unbound, so keep that conversation append-only. Claude Sonnet 5.5 `thinking.type = "between_tools"` rejects `block_binding` the same way. + +Responses to a bound request report each dropped block in `metadata.input_transformations`, for example `{ type = "thinking_dropped", path = "messages.1.content.0", reason = "prefix_binding_mismatch" }`. `prefix_binding_mismatch` means the conversation before the block changed; `model_binding_mismatch` means the conversation moved to a model that cannot read the block. An empty array means nothing was dropped. Streaming Claude responses report the array from `message_start`; Bedrock reports it from `messageStop`. Ignore entry types and reasons you do not recognize. + +Model switches and fallbacks need no client-side handling: the API drops blocks the target model cannot read, so the drivers pass every block back unchanged. + +The binding controls are a beta on the Claude API, Claude Platform on AWS, Amazon Bedrock and Google Cloud Vertex AI; Microsoft Foundry is unconfirmed. An endpoint that rejects the beta fails the request with its own error. + ### Retry Drivers retry transient failures (connection errors, 408, 409, 425, 429, 5xx) with exponential backoff before any response body is read, so a streamed response is never replayed. Health probes (`status`) always send a single request. diff --git a/src/llm/src/bedrock/client.lua b/src/llm/src/bedrock/client.lua index 1f4c0539..05d42499 100644 --- a/src/llm/src/bedrock/client.lua +++ b/src/llm/src/bedrock/client.lua @@ -206,7 +206,11 @@ end -- Converse: POST /model/{modelId}/converse function bedrock_client.converse(model_id, payload, options) local path = "/model/" .. model_id .. "/converse" - return signed_request(path, payload, options) + local response, err = signed_request(path, payload, options) + if response and response.additionalModelResponseFields then + response.metadata.input_transformations = response.additionalModelResponseFields.input_transformations + end + return response, err end -- ConverseStream: POST /model/{modelId}/converse-stream (returns eventstream) @@ -410,6 +414,7 @@ function bedrock_client.process_converse_stream(stream_response, callbacks) local usage = {} local content_blocks = {} local error_reported = false + local metadata = stream_response.metadata or {} local function handle_event(event): string? -- Converse stream events use top-level keys instead of a "type" field @@ -489,6 +494,10 @@ function bedrock_client.process_converse_stream(stream_response, callbacks) end elseif event.messageStop then finish_reason = event.messageStop.stopReason + local response_fields = event.messageStop.additionalModelResponseFields + if response_fields then + metadata.input_transformations = response_fields.input_transformations + end elseif event.metadata then if event.metadata.usage then usage = event.metadata.usage @@ -525,7 +534,7 @@ function bedrock_client.process_converse_stream(stream_response, callbacks) thinking = final_thinking_blocks, finish_reason = finish_reason, usage = usage, - metadata = stream_response.metadata or {} + metadata = metadata } on_done(result) diff --git a/src/llm/src/bedrock/generate.lua b/src/llm/src/bedrock/generate.lua index 26de6a57..832623f8 100644 --- a/src/llm/src/bedrock/generate.lua +++ b/src/llm/src/bedrock/generate.lua @@ -76,7 +76,7 @@ local function handle_streaming(stream_response, context, stream_config, err) context.name_to_id_map, final_usage, finish_reason, - stream_response.metadata + stream_result.metadata ) end @@ -127,6 +127,8 @@ function generate_handler.handler(contract_args) converse_payload.additionalModelRequestFields = additional_fields end + generate_handler._mapper.bind_thinking(converse_payload) + if contract_args.tools and #contract_args.tools > 0 then local converse_tools, name_to_id_map = generate_handler._mapper.map_tools(contract_args.tools) diff --git a/src/llm/src/bedrock/mapper.lua b/src/llm/src/bedrock/mapper.lua index 2628c56c..163a6a99 100644 --- a/src/llm/src/bedrock/mapper.lua +++ b/src/llm/src/bedrock/mapper.lua @@ -530,6 +530,35 @@ function mapper.map_options(contract_options, accepts): (table, table, table, st return inference_config, additional_fields, adjusted end +local function replays_reasoning(messages): boolean + for _, message in ipairs(messages or {}) do + if message.role == "assistant" and type(message.content) == "table" then + for _, block in ipairs(message.content) do + if block.reasoningContent then + return true + end + end + end + end + return false +end + +-- Binds the reasoning a Converse payload replays to drop_block. Bedrock takes the beta in +-- the anthropic_beta model field and returns input_transformations only when the payload +-- names it in additionalModelResponseFieldPaths. +function mapper.bind_thinking(payload) + if not replays_reasoning(payload.messages) then + return + end + local fields = payload.additionalModelRequestFields + local beta = thinking.bind_replayed_blocks(fields and fields.thinking) + if not beta then + return + end + fields.anthropic_beta = { beta } + payload.additionalModelResponseFieldPaths = { "/input_transformations" } +end + -- Try to parse a text block as an embedded tool call. -- Some models (Llama 3.3 on Bedrock, etc.) return tool calls as JSON inside a -- text content block instead of using native Converse toolUse blocks. diff --git a/src/llm/src/bedrock/structured_output.lua b/src/llm/src/bedrock/structured_output.lua index aece11e3..203b1717 100644 --- a/src/llm/src/bedrock/structured_output.lua +++ b/src/llm/src/bedrock/structured_output.lua @@ -137,6 +137,8 @@ function structured_output_handler.handler(contract_args) converse_payload.additionalModelRequestFields = additional_fields end + structured_output_handler._mapper.bind_thinking(converse_payload) + local response, request_err = structured_output_handler._client.converse( contract_args.model, converse_payload, diff --git a/src/llm/src/claude/client.lua b/src/llm/src/claude/client.lua index 4e3f4ce8..56857026 100644 --- a/src/llm/src/claude/client.lua +++ b/src/llm/src/claude/client.lua @@ -110,6 +110,21 @@ local function parse_error_response(http_response: transport.HttpResponse): tran return error_info end +-- Configured betas first, then request betas, each named once. +local function merge_betas(configured: {string}?, requested: {string}?): {string} + local merged: {string} = {} + local seen: {[string]: boolean} = {} + for _, list in ipairs({ configured or {}, requested or {} }) do + for _, beta in ipairs(list) do + if not seen[beta] then + seen[beta] = true + table.insert(merged, beta) + end + end + end + return merged +end + local function prepare_headers(api_key, api_version, beta_features, method, additional_headers): {[string]: string} local headers: {[string]: string} = {} headers["x-api-key"] = tostring(api_key) @@ -147,7 +162,8 @@ function claude_client.request(endpoint_path, payload, options): (any, transport end local full_url = config.base_url .. endpoint_path - local headers: {[string]: string} = prepare_headers(config.api_key, config.api_version, config.beta_features, method, config.headers) + local beta_features = merge_betas(config.beta_features, options.beta_features) + local headers: {[string]: string} = prepare_headers(config.api_key, config.api_version, beta_features, method, config.headers) local http_options: {[string]: any} = { headers = headers, @@ -197,9 +213,42 @@ function claude_client.request(endpoint_path, payload, options): (any, transport end parsed.metadata = extract_response_metadata(response) + parsed.metadata.input_transformations = parsed.input_transformations return parsed end +-- Stream indices start at 0 and leave gaps for other block types; blocks keep response order. +local function in_index_order(blocks: {[number]: any}): {any} + local indices: {number} = {} + for index in pairs(blocks) do + table.insert(indices, index) + end + table.sort(indices) + local ordered: {any} = {} + for _, index in ipairs(indices) do + table.insert(ordered, blocks[index]) + end + return ordered +end + +local function stream_result(full_content, tool_calls, thinking_blocks, finish_reason, usage, metadata) + local final_tool_calls = {} + for _, tool_call in ipairs(in_index_order(tool_calls)) do + if type(tool_call) == "table" and tool_call.id then + table.insert(final_tool_calls, tool_call) + end + end + + return { + content = full_content, + tool_calls = final_tool_calls, + thinking = in_index_order(thinking_blocks), + finish_reason = finish_reason, + usage = usage, + metadata = metadata + } +end + function claude_client.process_stream(stream_response, callbacks) if not stream_response or not stream_response.stream then return nil, "Invalid stream response" @@ -219,6 +268,7 @@ function claude_client.process_stream(stream_response, callbacks) local usage = {} local content_blocks = {} local leftover = "" + local metadata = stream_response.metadata or {} while true do local chunk, err = stream_response.stream:read() @@ -270,6 +320,9 @@ function claude_client.process_stream(stream_response, callbacks) if data.message and data.message.usage then usage = data.message.usage end + if data.message then + metadata.input_transformations = data.message.input_transformations + end elseif event_type == "content_block_start" then if data.index ~= nil and data.content_block then content_blocks[data.index] = data.content_block @@ -280,6 +333,11 @@ function claude_client.process_stream(stream_response, callbacks) thinking = data.content_block.thinking or "", signature = data.content_block.signature or "" } + elseif data.content_block.type == "redacted_thinking" then + thinking_blocks[data.index] = { + type = "redacted_thinking", + data = data.content_block.data + } end end elseif event_type == "content_block_delta" then @@ -345,29 +403,7 @@ function claude_client.process_stream(stream_response, callbacks) end end elseif event_type == "message_stop" then - local final_tool_calls = {} - for _, tool_call in pairs(tool_calls) do - if type(tool_call) == "table" and tool_call.id then - table.insert(final_tool_calls, tool_call) - end - end - - local final_thinking_blocks = {} - for _, thinking_block in pairs(thinking_blocks) do - if type(thinking_block) == "table" and thinking_block.type == "thinking" then - table.insert(final_thinking_blocks, thinking_block) - end - end - - local result = { - content = full_content, - tool_calls = final_tool_calls, - thinking = final_thinking_blocks, - finish_reason = finish_reason, - usage = usage, - metadata = stream_response.metadata or {} - } - + local result = stream_result(full_content, tool_calls, thinking_blocks, finish_reason, usage, metadata) on_done(result) return full_content, nil, result elseif event_type == "error" then @@ -383,29 +419,7 @@ function claude_client.process_stream(stream_response, callbacks) ::continue:: end - local final_tool_calls = {} - for _, tool_call in pairs(tool_calls) do - if type(tool_call) == "table" and tool_call.id then - table.insert(final_tool_calls, tool_call) - end - end - - local final_thinking_blocks = {} - for _, thinking_block in pairs(thinking_blocks) do - if type(thinking_block) == "table" and thinking_block.type == "thinking" then - table.insert(final_thinking_blocks, thinking_block) - end - end - - local result = { - content = full_content, - tool_calls = final_tool_calls, - thinking = final_thinking_blocks, - finish_reason = finish_reason, - usage = usage, - metadata = stream_response.metadata or {} - } - + local result = stream_result(full_content, tool_calls, thinking_blocks, finish_reason, usage, metadata) on_done(result) return full_content, nil, result end diff --git a/src/llm/src/claude/generate.lua b/src/llm/src/claude/generate.lua index 2a327060..9efcf9af 100644 --- a/src/llm/src/claude/generate.lua +++ b/src/llm/src/claude/generate.lua @@ -77,7 +77,7 @@ local function handle_streaming(stream_response, context, stream_config, err) context.name_to_id_map, final_usage, finish_reason, - stream_response.metadata + stream_result.metadata ) end @@ -149,7 +149,8 @@ function generate_handler.handler(contract_args) local request_options = { timeout = contract_args.timeout or 600, - retry = contract_args.retry + retry = contract_args.retry, + beta_features = generate_handler._mapper.bind_thinking(claude_payload) } local stream_config = nil diff --git a/src/llm/src/claude/mapper.lua b/src/llm/src/claude/mapper.lua index d7b4aac3..e827e404 100644 --- a/src/llm/src/claude/mapper.lua +++ b/src/llm/src/claude/mapper.lua @@ -680,6 +680,32 @@ function mapper.map_options(contract_options, accepts): (table, table, string?) return claude_options, adjusted end +local function replays_thinking(messages): boolean + for _, message in ipairs(messages or {}) do + if message.role == "assistant" and type(message.content) == "table" then + for _, block in ipairs(message.content) do + if block.type == "thinking" or block.type == "redacted_thinking" then + return true + end + end + end + end + return false +end + +-- Binds the thinking blocks a Messages payload replays to drop_block and returns the +-- request betas the binding requires, or nil when the payload carries no binding. +function mapper.bind_thinking(payload): {string}? + if not replays_thinking(payload.messages) then + return nil + end + local beta = thinking.bind_replayed_blocks(payload.thinking) + if not beta then + return nil + end + return { beta } +end + function mapper.extract_response_content(claude_response) local content_text = "" local tool_calls = {} @@ -702,14 +728,14 @@ function mapper.extract_response_content(claude_response) name = block.name or "", arguments = block.input or {} }) - elseif block.type == "thinking" or block.type == "redacted_thinking" then - local thinking_block = { - type = block.type, - thinking = block.type == "thinking" and (block.thinking or "") or "", - data = block.type == "redacted_thinking" and block.data or nil, + elseif block.type == "thinking" then + table.insert(thinking_blocks, { + type = "thinking", + thinking = block.thinking or "", signature = block.signature or "" - } - table.insert(thinking_blocks, thinking_block) + }) + elseif block.type == "redacted_thinking" then + table.insert(thinking_blocks, { type = "redacted_thinking", data = block.data }) end end diff --git a/src/llm/src/claude/structured_output.lua b/src/llm/src/claude/structured_output.lua index 2a7200fd..427196c4 100644 --- a/src/llm/src/claude/structured_output.lua +++ b/src/llm/src/claude/structured_output.lua @@ -204,7 +204,11 @@ function structured_output_handler.handler(contract_args) local response, request_err = structured_output_handler._client.request( structured_output_handler._client.ENDPOINTS.MESSAGES, native_payload, - { timeout = contract_args.timeout, retry = contract_args.retry } + { + timeout = contract_args.timeout, + retry = contract_args.retry, + beta_features = structured_output_handler._mapper.bind_thinking(native_payload) + } ) if request_err then return nil, err:from(request_err):build() @@ -266,7 +270,8 @@ function structured_output_handler.handler(contract_args) local request_options = { timeout = contract_args.timeout, - retry = contract_args.retry + retry = contract_args.retry, + beta_features = structured_output_handler._mapper.bind_thinking(claude_payload) } local response, request_err = structured_output_handler._client.request( diff --git a/src/llm/src/util/thinking.lua b/src/llm/src/util/thinking.lua index 02cd02e9..0c9ab009 100644 --- a/src/llm/src/util/thinking.lua +++ b/src/llm/src/util/thinking.lua @@ -1,5 +1,23 @@ local thinking = {} +-- Beta that admits thinking.block_binding and adds input_transformations to responses. +thinking.BINDING_BETA = "thinking-binding-controls-2026-08-01" + +-- block_binding rides only on a thinking object of type adaptive or enabled; the API +-- rejects it alongside disabled or between_tools thinking and without a thinking object. +local BINDABLE_TYPES: {[string]: boolean} = { adaptive = true, enabled = true } + +-- Sets drop_block on a thinking object that replays earlier thinking blocks, so the API +-- drops blocks bound to a different conversation instead of rejecting the request. +-- Returns the beta the request then requires, or nil when the object takes no binding. +function thinking.bind_replayed_blocks(config: {[string]: any}?): string? + if type(config) ~= "table" or not BINDABLE_TYPES[config.type] then + return nil + end + config.block_binding = { prefix_mismatch_behavior = "drop_block" } + return thinking.BINDING_BETA +end + function thinking.effort_level(effort: number): string if effort >= 100 then return "max" end if effort >= 80 then return "xhigh" end diff --git a/src/llm/test/_index.yaml b/src/llm/test/_index.yaml index 53f86bed..5eb8105b 100644 --- a/src/llm/test/_index.yaml +++ b/src/llm/test/_index.yaml @@ -525,8 +525,10 @@ entries: - json - env - ctx + - http_client imports: generate_handler: wippy.llm.claude:generate + claude_client: wippy.llm.claude:client test: wippy.test:test method: run_tests diff --git a/src/llm/test/bedrock/client_test.lua b/src/llm/test/bedrock/client_test.lua index 8607df93..f3331b3d 100644 --- a/src/llm/test/bedrock/client_test.lua +++ b/src/llm/test/bedrock/client_test.lua @@ -126,6 +126,34 @@ local function define_tests() end) end) + describe("Thinking block binding", function() + it("reports input_transformations returned in additionalModelResponseFields", function() + use_context({}) + bedrock_client._http_client = { + post = function() + return { + status_code = 200, + headers = {}, + body = json.encode({ + stopReason = "end_turn", + additionalModelResponseFields = { + input_transformations = { + { type = "thinking_dropped", path = "messages.1.content.0", reason = "prefix_binding_mismatch" } + } + } + }) + } + end + } + + local response, err = bedrock_client.converse("test-model", { messages = {} }) + + test.is_nil(err) + test.eq(#response.metadata.input_transformations, 1) + test.eq(response.metadata.input_transformations[1].path, "messages.1.content.0") + end) + end) + describe("ConverseStream", function() -- Eventstream message with zeroed CRCs; the client does not verify them. local function header(name: string, type_id: number, value_bytes: string): string @@ -202,6 +230,24 @@ local function define_tests() return content, err, result, seen, source end + it("reports input_transformations from messageStop in stream metadata", function() + local events = conversation() + events[6] = event("messageStop", { + stopReason = "end_turn", + additionalModelResponseFields = { + input_transformations = { + { type = "thinking_dropped", path = "messages.1.content.0", reason = "model_binding_mismatch" } + } + } + }) + + local _, err, result = run(events) + + test.is_nil(err) + test.eq(result.finish_reason, "end_turn") + test.eq(result.metadata.input_transformations[1].reason, "model_binding_mismatch") + end) + it("should dispatch each delta as soon as its message arrives", function() local content, err, result, seen, source = run(conversation()) diff --git a/src/llm/test/bedrock/generate_test.lua b/src/llm/test/bedrock/generate_test.lua index a6857e97..3baa5a37 100644 --- a/src/llm/test/bedrock/generate_test.lua +++ b/src/llm/test/bedrock/generate_test.lua @@ -1,6 +1,34 @@ local generate_handler = require("generate_handler") local json = require("json") +local BINDING_BETA = "thinking-binding-controls-2026-08-01" +local CLAUDE_MODEL = "us.anthropic.claude-opus-5-5-v1:0" + +local function history_with_thinking(blocks) + return { + { role = "user", content = { { type = "text", text = "Plan the trip" } } }, + { + role = "assistant", + content = { { type = "text", text = "Here is the plan." } }, + metadata = { thinking_blocks = blocks } + }, + { role = "user", content = { { type = "text", text = "Now book it" } } } + } +end + +local SIGNED_THINKING = { type = "thinking", thinking = "Weighing the routes.", signature = "sig-original-1" } + +local function converse_response(extra) + local response = { + output = { message = { role = "assistant", content = { { text = "Booked." } } } }, + stopReason = "end_turn", + usage = { inputTokens = 30, outputTokens = 3 }, + metadata = {} + } + for k, v in pairs(extra or {}) do response[k] = v end + return response +end + local function define_tests() describe("Bedrock Generate Handler (Converse API)", function() @@ -275,6 +303,129 @@ local function define_tests() end) end) + describe("Thinking block binding", function() + local function capture() + local captured = {} + generate_handler._client = { + converse = function(model_id, payload, options) + captured.payload = payload + return converse_response() + end + } + return captured + end + + it("binds replayed reasoning with drop_block under adaptive thinking", function() + local captured = capture() + + generate_handler.handler({ + model = CLAUDE_MODEL, + messages = history_with_thinking({ SIGNED_THINKING }), + accepts = { thinking = "adaptive" }, + options = { thinking_effort = 50 } + }) + + local fields = captured.payload.additionalModelRequestFields + test.eq(fields.thinking.type, "adaptive") + test.eq(fields.thinking.block_binding.prefix_mismatch_behavior, "drop_block") + test.eq(#fields.anthropic_beta, 1) + test.eq(fields.anthropic_beta[1], BINDING_BETA) + test.eq(#captured.payload.additionalModelResponseFieldPaths, 1) + test.eq(captured.payload.additionalModelResponseFieldPaths[1], "/input_transformations") + local replayed = captured.payload.messages[2].content[1].reasoningContent.reasoningText + test.eq(replayed.text, SIGNED_THINKING.thinking) + test.eq(replayed.signature, SIGNED_THINKING.signature) + end) + + it("binds replayed reasoning under budget thinking", function() + local captured = capture() + + generate_handler.handler({ + model = CLAUDE_MODEL, + messages = history_with_thinking({ SIGNED_THINKING }), + options = { thinking_effort = 50 } + }) + + local fields = captured.payload.additionalModelRequestFields + test.eq(fields.thinking.type, "enabled") + test.eq(fields.thinking.block_binding.prefix_mismatch_behavior, "drop_block") + test.eq(fields.anthropic_beta[1], BINDING_BETA) + end) + + it("sends no binding when the history replays no reasoning", function() + local captured = capture() + + generate_handler.handler({ + model = CLAUDE_MODEL, + messages = history_with_thinking({}), + accepts = { thinking = "adaptive" }, + options = { thinking_effort = 50 } + }) + + local fields = captured.payload.additionalModelRequestFields + test.is_nil(fields.thinking.block_binding) + test.is_nil(fields.anthropic_beta) + test.is_nil(captured.payload.additionalModelResponseFieldPaths) + end) + + it("never adds a thinking object to carry the binding when the request sends none", function() + local captured = capture() + + generate_handler.handler({ + model = CLAUDE_MODEL, + messages = history_with_thinking({ SIGNED_THINKING }), + accepts = { thinking = "adaptive" }, + options = { thinking_effort = 0 } + }) + + test.is_nil(captured.payload.additionalModelRequestFields) + test.is_nil(captured.payload.additionalModelResponseFieldPaths) + local replayed = captured.payload.messages[2].content[1].reasoningContent.reasoningText + test.eq(replayed.signature, SIGNED_THINKING.signature) + end) + + it("reports input_transformations from a streaming response in metadata", function() + generate_handler._output = { + streamer = function() + return { buffer_content = function() end, send_thinking = function() end, flush = function() end } + end + } + generate_handler._client = { + converse_stream = function() + return { stream = {}, metadata = { request_id = "req_stream" } }, nil + end, + process_converse_stream = function(_, callbacks) + callbacks.on_content("Booked.") + local result = { + content = "Booked.", + finish_reason = "end_turn", + usage = { inputTokens = 30, outputTokens = 3 }, + metadata = { + request_id = "req_stream", + input_transformations = { + { type = "thinking_dropped", path = "messages.1.content.0", reason = "prefix_binding_mismatch" } + } + } + } + callbacks.on_done(result) + return "Booked.", nil, result + end + } + + local response = generate_handler.handler({ + model = CLAUDE_MODEL, + messages = history_with_thinking({ SIGNED_THINKING }), + accepts = { thinking = "adaptive" }, + options = { thinking_effort = 50 }, + stream = { reply_to = "test-process", topic = "binding_stream" } + }) + + test.is_true(response.success) + test.eq(response.metadata.request_id, "req_stream") + test.eq(response.metadata.input_transformations[1].reason, "prefix_binding_mismatch") + end) + end) + describe("Stream fallback signals", function() local real_output local streamed_args = { diff --git a/src/llm/test/bedrock/structured_output_test.lua b/src/llm/test/bedrock/structured_output_test.lua index d3d68675..7b40e4e2 100644 --- a/src/llm/test/bedrock/structured_output_test.lua +++ b/src/llm/test/bedrock/structured_output_test.lua @@ -149,6 +149,54 @@ local function define_tests() test.eq(options.retry.backoff_ms, 0) end) + it("binds replayed reasoning with drop_block", function() + local captured_payload: any = nil + structured_output_handler._client = { + converse = function(model_id, payload, options) + captured_payload = payload + return { + output = { + message = { + role = "assistant", + content = { + { toolUse = { toolUseId = "c1", name = "structured_output", input = { x = 1 } } } + } + } + }, + stopReason = "tool_use", + usage = { inputTokens = 10, outputTokens = 5 }, + metadata = {} + } + end + } + + structured_output_handler.handler({ + model = "us.anthropic.claude-opus-5-5-v1:0", + messages = { + { role = "user", content = { { type = "text", text = "Plan" } } }, + { + role = "assistant", + content = { { type = "text", text = "Planned." } }, + metadata = { thinking_blocks = { { type = "thinking", thinking = "t", signature = "sig" } } } + }, + { role = "user", content = { { type = "text", text = "Answer" } } } + }, + schema = { + type = "object", + properties = { x = { type = "number" } }, + required = { "x" }, + additionalProperties = false + }, + options = { thinking_effort = 50 }, + accepts = { thinking = "adaptive" } + }) + + local fields = captured_payload.additionalModelRequestFields + test.eq(fields.thinking.block_binding.prefix_mismatch_behavior, "drop_block") + test.eq(fields.anthropic_beta[1], "thinking-binding-controls-2026-08-01") + test.eq(captured_payload.additionalModelResponseFieldPaths[1], "/input_transformations") + end) + it("should send forced tool_choice in payload", function() local captured_payload = nil structured_output_handler._client = { diff --git a/src/llm/test/claude/client_test.lua b/src/llm/test/claude/client_test.lua index 91673494..e5f95ec0 100644 --- a/src/llm/test/claude/client_test.lua +++ b/src/llm/test/claude/client_test.lua @@ -1171,6 +1171,191 @@ local function define_tests() end) end) + describe("Thinking block binding", function() + local BINDING_BETA = "thinking-binding-controls-2026-08-01" + + local function use(context) + claude_client._ctx = { all = function() return context end } + claude_client._env = { get = function() return nil end } + end + + local function sse_stream(chunks) + local stream = { chunks = chunks, current = 0 } + setmetatable(stream, { + __index = { + read = function(self) + self.current = self.current + 1 + return self.chunks[self.current] + end + } + }) + return stream + end + + it("merges request betas after configured betas without duplicates", function() + use({ api_key = "test-key", beta_features = { "prompt-caching-2024-07-31", BINDING_BETA } }) + local sent_beta = nil + claude_client._http_client = { + post = function(url, options) + sent_beta = options.headers["anthropic-beta"] + return { status_code = 200, body = "{}", headers = {} } + end + } + + local _, err = claude_client.request("/v1/messages", {}, { beta_features = { BINDING_BETA, "files-api-2025-04-14" } }) + + test.is_nil(err) + test.eq(sent_beta, "prompt-caching-2024-07-31," .. BINDING_BETA .. ",files-api-2025-04-14") + end) + + it("sends request betas when none are configured", function() + use({ api_key = "test-key" }) + local sent_beta = nil + claude_client._http_client = { + post = function(url, options) + sent_beta = options.headers["anthropic-beta"] + return { status_code = 200, body = "{}", headers = {} } + end + } + + claude_client.request("/v1/messages", {}, { beta_features = { BINDING_BETA } }) + + test.eq(sent_beta, BINDING_BETA) + end) + + it("reports input_transformations of a JSON response in metadata", function() + use({ api_key = "test-key" }) + claude_client._http_client = { + post = function() + return { + status_code = 200, + headers = { ["request-id"] = "req_1" }, + body = json.encode({ + content = {}, + input_transformations = { + { type = "thinking_dropped", path = "messages.1.content.0", reason = "prefix_binding_mismatch" } + } + }) + } + end + } + + local response = claude_client.request("/v1/messages", {}) + + local meta = assert(response.metadata) + test.eq(meta.request_id, "req_1") + test.eq(#meta.input_transformations, 1) + test.eq(meta.input_transformations[1].path, "messages.1.content.0") + end) + + it("reports an empty input_transformations array as empty", function() + use({ api_key = "test-key" }) + claude_client._http_client = { + post = function() + return { status_code = 200, headers = {}, body = '{"content":[],"input_transformations":[]}' } + end + } + + local response = claude_client.request("/v1/messages", {}) + + test.not_nil(response.metadata.input_transformations) + test.eq(#response.metadata.input_transformations, 0) + end) + + it("leaves input_transformations unset when the response carries none", function() + use({ api_key = "test-key" }) + claude_client._http_client = { + post = function() + return { status_code = 200, headers = {}, body = '{"content":[]}' } + end + } + + local response = claude_client.request("/v1/messages", {}) + + test.is_nil(response.metadata.input_transformations) + end) + + it("reports input_transformations from message_start in stream metadata", function() + local stream_response = { + stream = sse_stream({ + 'event: message_start\ndata: {"type":"message_start","message":{"usage":{"input_tokens":5},"input_transformations":[{"type":"thinking_dropped","path":"messages.3.content.0","reason":"model_binding_mismatch"}]}}\n\n', + 'event: message_stop\ndata: {"type":"message_stop"}\n\n' + }), + metadata = { request_id = "req_stream" } + } + + local _, err, result = claude_client.process_stream(stream_response, {}) + + test.is_nil(err) + local res = assert(result) + test.eq(res.metadata.request_id, "req_stream") + test.eq(#res.metadata.input_transformations, 1) + test.eq(res.metadata.input_transformations[1].reason, "model_binding_mismatch") + end) + + it("keeps streamed redacted and signed thinking blocks in response order", function() + local stream_response = { + stream = sse_stream({ + 'event: message_start\ndata: {"type":"message_start","message":{"usage":{"input_tokens":5}}}\n\n', + 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}\n\n', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"first"}}\n\n', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"sig-0"}}\n\n', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n', + 'event: content_block_start\ndata: {"type":"content_block_start","index":1,"content_block":{"type":"redacted_thinking","data":"opaque-1"}}\n\n', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":1}\n\n', + 'event: content_block_start\ndata: {"type":"content_block_start","index":2,"content_block":{"type":"thinking","thinking":""}}\n\n', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":2,"delta":{"type":"thinking_delta","thinking":"third"}}\n\n', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":2,"delta":{"type":"signature_delta","signature":"sig-2"}}\n\n', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":2}\n\n', + 'event: message_stop\ndata: {"type":"message_stop"}\n\n' + }), + metadata = {} + } + + local _, err, result = claude_client.process_stream(stream_response, {}) + + test.is_nil(err) + local blocks = assert(result).thinking + test.eq(#blocks, 3) + test.eq(blocks[1].type, "thinking") + test.eq(blocks[1].thinking, "first") + test.eq(blocks[1].signature, "sig-0") + test.eq(blocks[2].type, "redacted_thinking") + test.eq(blocks[2].data, "opaque-1") + test.is_nil(blocks[2].thinking) + test.is_nil(blocks[2].signature) + test.eq(blocks[3].type, "thinking") + test.eq(blocks[3].thinking, "third") + test.eq(blocks[3].signature, "sig-2") + end) + end) + + describe("Stream block order", function() + it("returns streamed tool calls in response order", function() + local chunks = { + 'event: message_start\ndata: {"type":"message_start","message":{"usage":{"input_tokens":5}}}\n\n', + 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"call_a","name":"first"}}\n\n', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n', + 'event: content_block_start\ndata: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"call_b","name":"second"}}\n\n', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":1}\n\n', + 'event: message_stop\ndata: {"type":"message_stop"}\n\n' + } + local stream = { current = 0 } + function stream:read() + self.current = self.current + 1 + return chunks[self.current] + end + + local _, err, result = claude_client.process_stream({ stream = stream, metadata = {} }, {}) + + test.is_nil(err) + local calls = assert(result).tool_calls + test.eq(#calls, 2) + test.eq(calls[1].id, "call_a") + test.eq(calls[2].id, "call_b") + end) + end) + describe("Stream Error Handling", function() it("should handle stream read errors", function() local mock_stream = { diff --git a/src/llm/test/claude/generate_test.lua b/src/llm/test/claude/generate_test.lua index d68f84d7..ecddfc6b 100644 --- a/src/llm/test/claude/generate_test.lua +++ b/src/llm/test/claude/generate_test.lua @@ -1,6 +1,47 @@ local generate_handler = require("generate_handler") +local claude_client = require("claude_client") local json = require("json") +local BINDING_BETA = "thinking-binding-controls-2026-08-01" + +local SIGNED_THINKING = { type = "thinking", thinking = "Weighing the routes.", signature = "sig-original-1" } +local REDACTED_THINKING = { type = "redacted_thinking", data = "opaque-redacted-1" } + +local function history_with_thinking(blocks) + return { + { role = "system", content = "You are the booking agent." }, + { role = "user", content = { { type = "text", text = "Plan the trip" } } }, + { + role = "function_call", + function_call = { id = "call_1", name = "search", arguments = { q = "flights" } }, + metadata = { thinking_blocks = blocks } + }, + { role = "function_result", function_call_id = "call_1", content = "3 flights" }, + { + role = "assistant", + content = { { type = "text", text = "Here is the plan." } }, + metadata = { thinking_blocks = blocks } + }, + { role = "user", content = { { type = "text", text = "Now book it" } } } + } +end + +local function capturing_client(captured, response) + return { + ENDPOINTS = { MESSAGES = "/v1/messages" }, + request = function(endpoint, payload, options) + captured.payload = payload + captured.options = options + return response or { + content = { { type = "text", text = "Booked." } }, + stop_reason = "end_turn", + usage = { input_tokens = 30, output_tokens = 3 }, + metadata = {} + } + end + } +end + local function define_tests() describe("Claude Generate Handler", function() after_each(function() @@ -1480,6 +1521,305 @@ local function define_tests() end) end) + describe("Thinking block binding", function() + it("binds replayed blocks with drop_block under adaptive thinking", function() + local captured = {} + generate_handler._client = capturing_client(captured) + + local response = generate_handler.handler({ + model = "claude-opus-5-5", + messages = history_with_thinking({ SIGNED_THINKING }), + accepts = { thinking = "adaptive" }, + options = { thinking_effort = 50 } + }) + + test.is_true(response.success) + test.eq(captured.payload.thinking.type, "adaptive") + test.eq(captured.payload.thinking.block_binding.prefix_mismatch_behavior, "drop_block") + test.eq(#captured.options.beta_features, 1) + test.eq(captured.options.beta_features[1], BINDING_BETA) + end) + + it("binds a history whose only replayed block is redacted", function() + local captured = {} + generate_handler._client = capturing_client(captured) + + generate_handler.handler({ + model = "claude-opus-5-5", + messages = history_with_thinking({ REDACTED_THINKING }), + accepts = { thinking = "adaptive" }, + options = { thinking_effort = 50 } + }) + + test.eq(captured.payload.thinking.block_binding.prefix_mismatch_behavior, "drop_block") + test.eq(captured.options.beta_features[1], BINDING_BETA) + end) + + it("binds replayed blocks under budget thinking", function() + local captured = {} + generate_handler._client = capturing_client(captured) + + generate_handler.handler({ + model = "claude-sonnet-4-5", + messages = history_with_thinking({ SIGNED_THINKING }), + options = { thinking_effort = 50 } + }) + + test.eq(captured.payload.thinking.type, "enabled") + test.not_nil(captured.payload.thinking.budget_tokens) + test.eq(captured.payload.thinking.block_binding.prefix_mismatch_behavior, "drop_block") + test.eq(captured.options.beta_features[1], BINDING_BETA) + end) + + it("sends no binding when the history replays no thinking blocks", function() + local captured = {} + generate_handler._client = capturing_client(captured) + + generate_handler.handler({ + model = "claude-opus-5-5", + messages = history_with_thinking({}), + accepts = { thinking = "adaptive" }, + options = { thinking_effort = 50 } + }) + + test.eq(captured.payload.thinking.type, "adaptive") + test.is_nil(captured.payload.thinking.block_binding) + test.is_nil(captured.options.beta_features) + end) + + it("never adds a thinking object to carry the binding when the request sends none", function() + local captured = {} + generate_handler._client = capturing_client(captured) + + generate_handler.handler({ + model = "claude-opus-4-8", + messages = history_with_thinking({ SIGNED_THINKING }), + accepts = { thinking = "adaptive" }, + options = { thinking_effort = 0 } + }) + + test.is_nil(captured.payload.thinking) + test.is_nil(captured.options.beta_features) + local replayed = captured.payload.messages[2].content[1] + test.eq(replayed.type, "thinking") + test.eq(replayed.signature, SIGNED_THINKING.signature) + end) + + it("never adds a thinking object on a route that declares no thinking", function() + local captured = {} + generate_handler._client = capturing_client(captured) + + generate_handler.handler({ + model = "claude-sonnet-4-5", + messages = history_with_thinking({ SIGNED_THINKING }), + accepts = { thinking = "none" }, + options = {} + }) + + test.is_nil(captured.payload.thinking) + test.is_nil(captured.options.beta_features) + end) + + it("replays every thinking block unchanged when the conversation switches models", function() + local captured = {} + generate_handler._client = capturing_client(captured) + + generate_handler.handler({ + model = "claude-haiku-5-5", + messages = history_with_thinking({ SIGNED_THINKING, REDACTED_THINKING }), + accepts = { thinking = "adaptive" }, + options = { thinking_effort = 50 } + }) + + local messages = captured.payload.messages + for _, index in ipairs({ 2, 4 }) do + local content = messages[index].content + test.eq(messages[index].role, "assistant") + test.eq(content[1].type, "thinking") + test.eq(content[1].thinking, SIGNED_THINKING.thinking) + test.eq(content[1].signature, SIGNED_THINKING.signature) + test.eq(content[2].type, "redacted_thinking") + test.eq(content[2].data, REDACTED_THINKING.data) + end + test.eq(captured.payload.thinking.block_binding.prefix_mismatch_behavior, "drop_block") + end) + + it("stores response thinking blocks in the shape the API returned them", function() + generate_handler._client = capturing_client({}, { + content = { + { type = "thinking", thinking = "Weighing the routes.", signature = "sig-new" }, + { type = "redacted_thinking", data = "opaque-new" }, + { type = "text", text = "Booked." } + }, + stop_reason = "end_turn", + usage = { input_tokens = 30, output_tokens = 3 }, + metadata = {} + }) + + local response = generate_handler.handler({ + model = "claude-opus-5-5", + messages = history_with_thinking({}), + accepts = { thinking = "adaptive" }, + options = { thinking_effort = 50 } + }) + + local blocks = response.metadata.thinking_blocks + test.eq(#blocks, 2) + test.eq(blocks[1].type, "thinking") + test.eq(blocks[1].thinking, "Weighing the routes.") + test.eq(blocks[1].signature, "sig-new") + test.is_nil(blocks[1].data) + test.eq(blocks[2].type, "redacted_thinking") + test.eq(blocks[2].data, "opaque-new") + test.is_nil(blocks[2].thinking) + test.is_nil(blocks[2].signature) + end) + + it("reports input_transformations from a non-streaming response in metadata", function() + local captured = {} + generate_handler._client = capturing_client(captured, { + content = { { type = "text", text = "Booked." } }, + stop_reason = "end_turn", + usage = { input_tokens = 30, output_tokens = 3 }, + metadata = { + input_transformations = { + { type = "thinking_dropped", path = "messages.1.content.0", reason = "prefix_binding_mismatch" } + } + } + }) + + local response = generate_handler.handler({ + model = "claude-opus-5-5", + messages = history_with_thinking({ SIGNED_THINKING }), + accepts = { thinking = "adaptive" }, + options = { thinking_effort = 50 } + }) + + local transformations = response.metadata.input_transformations + test.eq(#transformations, 1) + test.eq(transformations[1].type, "thinking_dropped") + test.eq(transformations[1].path, "messages.1.content.0") + test.eq(transformations[1].reason, "prefix_binding_mismatch") + end) + + it("reports input_transformations from a streaming response in metadata", function() + generate_handler._output = { + streamer = function() + return { + buffer_content = function() end, + send_thinking = function() end, + flush = function() end + } + end, + FINISH_REASON = { TOOL_CALL = "tool_call" } + } + generate_handler._client = { + ENDPOINTS = { MESSAGES = "/v1/messages" }, + request = function() + return { stream = {}, metadata = { request_id = "req_stream" } } + end, + process_stream = function(_, callbacks) + callbacks.on_content("Booked.") + local result = { + content = "Booked.", + finish_reason = "end_turn", + usage = { input_tokens = 30, output_tokens = 3 }, + metadata = { + request_id = "req_stream", + input_transformations = { + { type = "thinking_dropped", path = "messages.3.content.0", reason = "model_binding_mismatch" } + } + } + } + callbacks.on_done(result) + return "Booked.", nil, result + end + } + + local response = generate_handler.handler({ + model = "claude-opus-5-5", + messages = history_with_thinking({ SIGNED_THINKING }), + accepts = { thinking = "adaptive" }, + options = { thinking_effort = 50 }, + stream = { reply_to = "test-process", topic = "binding_stream" } + }) + + test.is_true(response.success) + test.eq(response.metadata.request_id, "req_stream") + local transformations = response.metadata.input_transformations + test.eq(#transformations, 1) + test.eq(transformations[1].reason, "model_binding_mismatch") + end) + + describe("through the HTTP client", function() + local saved = {} + + before_each(function() + saved.ctx = claude_client._ctx + saved.env = claude_client._env + saved.http = claude_client._http_client + end) + + after_each(function() + claude_client._ctx = saved.ctx + claude_client._env = saved.env + claude_client._http_client = saved.http + end) + + local function send(configured_betas) + local sent = {} + claude_client._ctx = { + all = function() + return { api_key = "test-key", beta_features = configured_betas } + end + } + claude_client._env = { get = function() return nil end } + claude_client._http_client = { + post = function(url, options) + sent.headers = options.headers + sent.body = json.decode(options.body) + return { + status_code = 200, + headers = { ["request-id"] = "req_bound" }, + body = json.encode({ + content = { { type = "text", text = "Booked." } }, + stop_reason = "end_turn", + usage = { input_tokens = 30, output_tokens = 3 }, + input_transformations = { + { type = "thinking_dropped", path = "messages.1.content.0", reason = "prefix_binding_mismatch" } + } + }) + } + end + } + generate_handler._client = claude_client + + local response = generate_handler.handler({ + model = "claude-opus-5-5", + messages = history_with_thinking({ SIGNED_THINKING }), + accepts = { thinking = "adaptive" }, + options = { thinking_effort = 50 } + }) + return sent, response + end + + it("sends the binding beta once alongside configured betas", function() + local sent, response = send({ "prompt-caching-2024-07-31" }) + + test.eq(sent.headers["anthropic-beta"], "prompt-caching-2024-07-31," .. BINDING_BETA) + test.eq(sent.body.thinking.block_binding.prefix_mismatch_behavior, "drop_block") + test.eq(response.metadata.request_id, "req_bound") + test.eq(response.metadata.input_transformations[1].reason, "prefix_binding_mismatch") + end) + + it("sends the binding beta once when it is already configured", function() + local sent = send({ BINDING_BETA, "prompt-caching-2024-07-31" }) + + test.eq(sent.headers["anthropic-beta"], BINDING_BETA .. ",prompt-caching-2024-07-31") + end) + end) + end) + describe("Stream fallback signals", function() local streamed_args = { model = "claude-3-5-sonnet-20241022", diff --git a/src/llm/test/claude/structured_output_test.lua b/src/llm/test/claude/structured_output_test.lua index 51dc7c66..99f7627a 100644 --- a/src/llm/test/claude/structured_output_test.lua +++ b/src/llm/test/claude/structured_output_test.lua @@ -890,6 +890,68 @@ local function define_tests() end) end) + describe("Thinking block binding", function() + local schema = { + type = "object", + properties = { answer = { type = "string" } }, + required = { "answer" }, + additionalProperties = false + } + + local function send(structured_output_mode) + local sent = {} + structured_output_handler._client._ctx = { all = function() return { api_key = "test-api-key" } end } + structured_output_handler._client._env = { get = function() return nil end } + structured_output_handler._client._http_client = { + post = function(url, options) + sent.headers = options.headers + sent.body = json.decode(tostring(options.body)) + return { + status_code = 200, + headers = {}, + body = json.encode({ + content = { + { type = "text", text = '{"answer":"yes"}' }, + { type = "tool_use", id = "t1", name = "structured_output", input = { answer = "yes" } } + }, + stop_reason = "end_turn", + usage = { input_tokens = 10, output_tokens = 5 }, + input_transformations = { + { type = "thinking_dropped", path = "messages.1.content.0", reason = "prefix_binding_mismatch" } + } + }) + } + end + } + local response = structured_output_handler.handler({ + model = "claude-opus-5-5", + messages = { + { role = "user", content = { { type = "text", text = "Plan" } } }, + { + role = "assistant", + content = { { type = "text", text = "Planned." } }, + metadata = { thinking_blocks = { { type = "thinking", thinking = "t", signature = "sig" } } } + }, + { role = "user", content = { { type = "text", text = "Answer" } } } + }, + schema = schema, + options = { thinking_effort = 50 }, + accepts = { thinking = "adaptive", structured_output = structured_output_mode } + }) + return sent, response + end + + for _, mode in ipairs({ "native", "tool" }) do + it("binds replayed blocks on the " .. mode .. " path", function() + local sent, response = send(mode) + + test.eq(sent.body.thinking.block_binding.prefix_mismatch_behavior, "drop_block") + test.eq(sent.headers["anthropic-beta"], "thinking-binding-controls-2026-08-01") + test.eq(response.metadata.input_transformations[1].reason, "prefix_binding_mismatch") + end) + end + end) + describe("Error Handling", function() it("should handle Claude API errors", function() structured_output_handler._client._ctx = { From 162aeb1a4deee52ccd872163517a904738b211e6 Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 14:33:29 -0500 Subject: [PATCH 02/37] fix(llm): keep thinking binding and replay shapes intact across config and drivers Fold a configured anthropic-beta header into the merged beta list, keep a block_binding the payload already carries and send the beta whenever one is present, append Bedrock betas and response field paths instead of replacing them, replay stored blocks in the API shape of their type, and capture, order and replay Bedrock reasoning and redacted reasoning. --- src/llm/src/README.md | 4 +- src/llm/src/bedrock/_index.yaml | 1 + src/llm/src/bedrock/client.lua | 38 ++++----- src/llm/src/bedrock/mapper.lua | 54 ++++++------ src/llm/src/claude/client.lua | 48 +++++------ src/llm/src/claude/mapper.lua | 36 ++++---- src/llm/src/util/thinking.lua | 14 +-- src/llm/src/util/transport.lua | 29 +++++++ src/llm/test/bedrock/client_test.lua | 45 ++++++++++ src/llm/test/bedrock/mapper_test.lua | 117 ++++++++++++++++++++++++++ src/llm/test/claude/client_test.lua | 36 ++++++++ src/llm/test/claude/generate_test.lua | 27 ++++++ src/llm/test/claude/mapper_test.lua | 41 +++++++++ 13 files changed, 392 insertions(+), 98 deletions(-) diff --git a/src/llm/src/README.md b/src/llm/src/README.md index ff7c13de..1c71c5d3 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -101,9 +101,9 @@ Provider open context composition is unchanged: provider entry `driver.options`, ### Thinking block binding -The Claude driver stores each response's `thinking` and `redacted_thinking` blocks in `metadata.thinking_blocks`, in response order and in the shape the API returned them, and replays them verbatim on later turns; Bedrock does the same for signed `reasoningContent` text. A Claude thinking block is bound to the conversation that produced it: the top-level system prompt, the tools, and every earlier message. After an agent handoff, a prompt or tool change, or a history edit, models that enforce the check reject a request that replays such a block with a 400. +Claude and Bedrock store each response's `thinking` blocks as `{ type, thinking, signature }` and `redacted_thinking` blocks as `{ type, data }` in `metadata.thinking_blocks`, in response order, and replay every stored block on later turns in its position and in the provider shape of its type (Bedrock: `reasoningContent.reasoningText` and `reasoningContent.redactedContent`). A Claude thinking block is bound to the conversation that produced it: the top-level system prompt, the tools, and every earlier message. After an agent handoff, a prompt or tool change, or a history edit, models that enforce the check reject a request that replays such a block with a 400. -When a request replays at least one thinking block and sends a thinking object of type `adaptive` or `enabled` (a positive thinking_effort), the driver sets `thinking.block_binding.prefix_mismatch_behavior = "drop_block"` and adds the `thinking-binding-controls-2026-08-01` beta. The API then drops the first mismatched block and every later thinking block instead of rejecting the request. The Claude driver appends the beta to the `anthropic-beta` header after the configured `beta_features`, each named once. Bedrock sends it in the `anthropic_beta` field of `additionalModelRequestFields` and requests `/input_transformations` through `additionalModelResponseFieldPaths`. Plain generation and structured output behave the same. +When a request replays at least one thinking block and sends a thinking object of type `adaptive` or `enabled` (a positive thinking_effort), the driver sets `thinking.block_binding.prefix_mismatch_behavior = "drop_block"` and adds the `thinking-binding-controls-2026-08-01` beta. The API then drops the first mismatched block and every later thinking block instead of rejecting the request. A `block_binding` the thinking object already carries is kept as set, and any request whose thinking object carries `block_binding` sends the beta. The Claude driver sends one `anthropic-beta` header: the configured `beta_features`, then the betas of a configured `anthropic-beta` context header (any letter case), then request betas, each named once. Bedrock appends the beta to the `anthropic_beta` list of `additionalModelRequestFields` and `/input_transformations` to `additionalModelResponseFieldPaths`, each named once. Plain generation and structured output behave the same. A request without a thinking object (zero or absent effort, or `thinking = "none"`) carries no binding: `block_binding` exists only inside an adaptive or enabled thinking object, and the driver never turns thinking on to carry it, since on several models an omitted thinking object means no thinking. Such a request replays its blocks unbound, so keep that conversation append-only. Claude Sonnet 5.5 `thinking.type = "between_tools"` rejects `block_binding` the same way. diff --git a/src/llm/src/bedrock/_index.yaml b/src/llm/src/bedrock/_index.yaml index f9dda73b..dd95f677 100644 --- a/src/llm/src/bedrock/_index.yaml +++ b/src/llm/src/bedrock/_index.yaml @@ -135,6 +135,7 @@ entries: prompt: wippy.llm:prompt output: wippy.llm:output route: wippy.llm.util:route + transport: wippy.llm.util:transport - name: embed_titan kind: library.lua diff --git a/src/llm/src/bedrock/client.lua b/src/llm/src/bedrock/client.lua index 05d42499..dc9bbc97 100644 --- a/src/llm/src/bedrock/client.lua +++ b/src/llm/src/bedrock/client.lua @@ -438,12 +438,6 @@ function bedrock_client.process_converse_stream(stream_response, callbacks) name = start_block.toolUse.name or "", partial_json = "" } - elseif start_block.reasoningContent then - thinking_blocks[index] = { - type = "thinking", - thinking = "", - signature = "" - } end elseif event.contentBlockDelta then local data = event.contentBlockDelta @@ -455,16 +449,21 @@ function bedrock_client.process_converse_stream(stream_response, callbacks) full_content = full_content .. text_chunk on_content(text_chunk) elseif delta.reasoningContent then + -- ConverseStream opens reasoning blocks with their first delta, not contentBlockStart. local rc = delta.reasoningContent - if rc.text then - on_thinking(tostring(rc.text)) - if thinking_blocks[index] then - thinking_blocks[index].thinking = thinking_blocks[index].thinking .. rc.text + if rc.redactedContent then + local block = thinking_blocks[index] or { type = "redacted_thinking", data = "" } + block.data = block.data .. rc.redactedContent + thinking_blocks[index] = block + else + local block = thinking_blocks[index] or { type = "thinking", thinking = "", signature = "" } + thinking_blocks[index] = block + if rc.text then + on_thinking(tostring(rc.text)) + block.thinking = block.thinking .. rc.text end - end - if rc.signature then - if thinking_blocks[index] then - thinking_blocks[index].signature = thinking_blocks[index].signature .. rc.signature + if rc.signature then + block.signature = block.signature .. rc.signature end end elseif delta.toolUse then @@ -515,23 +514,16 @@ function bedrock_client.process_converse_stream(stream_response, callbacks) end local final_tool_calls = {} - for _, tool_call in pairs(tool_calls) do + for _, tool_call in ipairs(transport.in_index_order(tool_calls)) do if type(tool_call) == "table" and tool_call.id then table.insert(final_tool_calls, tool_call) end end - local final_thinking_blocks = {} - for _, thinking_block in pairs(thinking_blocks) do - if type(thinking_block) == "table" and thinking_block.type == "thinking" then - table.insert(final_thinking_blocks, thinking_block) - end - end - local result = { content = full_content, tool_calls = final_tool_calls, - thinking = final_thinking_blocks, + thinking = transport.in_index_order(thinking_blocks), finish_reason = finish_reason, usage = usage, metadata = metadata diff --git a/src/llm/src/bedrock/mapper.lua b/src/llm/src/bedrock/mapper.lua index 163a6a99..908ba6cd 100644 --- a/src/llm/src/bedrock/mapper.lua +++ b/src/llm/src/bedrock/mapper.lua @@ -3,6 +3,7 @@ local output = require("output") local prompt = require("prompt") local json = require("json") local route = require("route") +local transport = require("transport") local mapper = {} @@ -199,6 +200,16 @@ local function select_cache_positions(system_positions, message_positions) return final_system, final_messages end +-- A stored thinking or redacted_thinking block as a Converse reasoningContent block. +local function reasoning_content(block) + if block.type == "thinking" then + return { reasoningContent = { reasoningText = { text = block.thinking or "", signature = block.signature or "" } } } + elseif block.type == "redacted_thinking" then + return { reasoningContent = { redactedContent = block.data } } + end + return nil +end + -- Map contract messages to Converse format function mapper.map_messages(contract_messages) if not contract_messages or #contract_messages == 0 then @@ -264,15 +275,9 @@ function mapper.map_messages(contract_messages) if msg.metadata and msg.metadata.thinking_blocks then for _, thinking_block in ipairs(msg.metadata.thinking_blocks) do - if thinking_block.type == "thinking" then - table.insert(content_blocks, { - reasoningContent = { - reasoningText = { - text = thinking_block.thinking or "", - signature = thinking_block.signature or "" - } - } - }) + local reasoning = reasoning_content(thinking_block) + if reasoning then + table.insert(content_blocks, reasoning) end end end @@ -294,15 +299,9 @@ function mapper.map_messages(contract_messages) if msg.metadata and msg.metadata.thinking_blocks then for _, thinking_block in ipairs(msg.metadata.thinking_blocks) do - if thinking_block.type == "thinking" then - table.insert(content_blocks, { - reasoningContent = { - reasoningText = { - text = thinking_block.thinking or "", - signature = thinking_block.signature or "" - } - } - }) + local reasoning = reasoning_content(thinking_block) + if reasoning then + table.insert(content_blocks, reasoning) end end end @@ -547,16 +546,21 @@ end -- the anthropic_beta model field and returns input_transformations only when the payload -- names it in additionalModelResponseFieldPaths. function mapper.bind_thinking(payload) - if not replays_reasoning(payload.messages) then + local fields = payload.additionalModelRequestFields + if not fields then return end - local fields = payload.additionalModelRequestFields - local beta = thinking.bind_replayed_blocks(fields and fields.thinking) - if not beta then + if replays_reasoning(payload.messages) then + thinking.bind_replayed_blocks(fields.thinking) + end + if not thinking.is_bound(fields.thinking) then return end - fields.anthropic_beta = { beta } - payload.additionalModelResponseFieldPaths = { "/input_transformations" } + fields.anthropic_beta = transport.merge_unique({ fields.anthropic_beta or {}, { thinking.BINDING_BETA } }) + payload.additionalModelResponseFieldPaths = transport.merge_unique({ + payload.additionalModelResponseFieldPaths or {}, + { "/input_transformations" } + }) end -- Try to parse a text block as an embedded tool call. @@ -655,6 +659,8 @@ function mapper.extract_response_content(converse_response, tool_names) thinking = block.reasoningContent.reasoningText.text or "", signature = block.reasoningContent.reasoningText.signature or "" }) + elseif block.reasoningContent and block.reasoningContent.redactedContent then + table.insert(thinking_blocks, { type = "redacted_thinking", data = block.reasoningContent.redactedContent }) end end diff --git a/src/llm/src/claude/client.lua b/src/llm/src/claude/client.lua index 56857026..cb84c358 100644 --- a/src/llm/src/claude/client.lua +++ b/src/llm/src/claude/client.lua @@ -110,19 +110,24 @@ local function parse_error_response(http_response: transport.HttpResponse): tran return error_info end --- Configured betas first, then request betas, each named once. -local function merge_betas(configured: {string}?, requested: {string}?): {string} - local merged: {string} = {} - local seen: {[string]: boolean} = {} - for _, list in ipairs({ configured or {}, requested or {} }) do - for _, beta in ipairs(list) do - if not seen[beta] then - seen[beta] = true - table.insert(merged, beta) +-- Separates a configured anthropic-beta header, in any letter case, from the other +-- configured headers so its betas join the single merged beta header. +local function split_beta_header(headers: {[string]: string}?): ({[string]: string}, {string}) + local others: {[string]: string} = {} + local betas: {string} = {} + for name, value in pairs(headers or {}) do + if string.lower(tostring(name)) == "anthropic-beta" then + for beta in string.gmatch(tostring(value), "[^,]+") do + local trimmed = beta:match("^%s*(.-)%s*$") + if trimmed ~= "" then + table.insert(betas, trimmed) + end end + else + others[tostring(name)] = tostring(value) end end - return merged + return others, betas end local function prepare_headers(api_key, api_version, beta_features, method, additional_headers): {[string]: string} @@ -162,8 +167,9 @@ function claude_client.request(endpoint_path, payload, options): (any, transport end local full_url = config.base_url .. endpoint_path - local beta_features = merge_betas(config.beta_features, options.beta_features) - local headers: {[string]: string} = prepare_headers(config.api_key, config.api_version, beta_features, method, config.headers) + local configured_headers, header_betas = split_beta_header(config.headers) + local beta_features = transport.merge_unique({ config.beta_features or {}, header_betas, options.beta_features or {} }) + local headers: {[string]: string} = prepare_headers(config.api_key, config.api_version, beta_features, method, configured_headers) local http_options: {[string]: any} = { headers = headers, @@ -217,23 +223,9 @@ function claude_client.request(endpoint_path, payload, options): (any, transport return parsed end --- Stream indices start at 0 and leave gaps for other block types; blocks keep response order. -local function in_index_order(blocks: {[number]: any}): {any} - local indices: {number} = {} - for index in pairs(blocks) do - table.insert(indices, index) - end - table.sort(indices) - local ordered: {any} = {} - for _, index in ipairs(indices) do - table.insert(ordered, blocks[index]) - end - return ordered -end - local function stream_result(full_content, tool_calls, thinking_blocks, finish_reason, usage, metadata) local final_tool_calls = {} - for _, tool_call in ipairs(in_index_order(tool_calls)) do + for _, tool_call in ipairs(transport.in_index_order(tool_calls)) do if type(tool_call) == "table" and tool_call.id then table.insert(final_tool_calls, tool_call) end @@ -242,7 +234,7 @@ local function stream_result(full_content, tool_calls, thinking_blocks, finish_r return { content = full_content, tool_calls = final_tool_calls, - thinking = in_index_order(thinking_blocks), + thinking = transport.in_index_order(thinking_blocks), finish_reason = finish_reason, usage = usage, metadata = metadata diff --git a/src/llm/src/claude/mapper.lua b/src/llm/src/claude/mapper.lua index e827e404..382790ff 100644 --- a/src/llm/src/claude/mapper.lua +++ b/src/llm/src/claude/mapper.lua @@ -217,6 +217,17 @@ function mapper.map_finish_reason(claude_stop_reason) return mapper.FINISH_REASON_MAP[claude_stop_reason] or claude_stop_reason end +-- A thinking or redacted_thinking block in the Messages API shape of its type, for +-- storing a response block and replaying a stored one; other blocks yield nil. +local function thinking_block_shape(block) + if block.type == "thinking" then + return { type = "thinking", thinking = block.thinking or "", signature = block.signature or "" } + elseif block.type == "redacted_thinking" then + return { type = "redacted_thinking", data = block.data } + end + return nil +end + local function extract_thinking_content_from_blocks(thinking_blocks) local thinking_content = "" for _, block in ipairs(thinking_blocks) do @@ -461,7 +472,7 @@ function mapper.map_messages(contract_messages) if msg.metadata and msg.metadata.thinking_blocks then for _, thinking_block in ipairs(msg.metadata.thinking_blocks) do - table.insert(content_blocks, thinking_block) + table.insert(content_blocks, thinking_block_shape(thinking_block) or thinking_block) end end @@ -482,7 +493,7 @@ function mapper.map_messages(contract_messages) if msg.metadata and msg.metadata.thinking_blocks then for _, thinking_block in ipairs(msg.metadata.thinking_blocks) do - table.insert(content_blocks, thinking_block) + table.insert(content_blocks, thinking_block_shape(thinking_block) or thinking_block) end end @@ -694,16 +705,15 @@ local function replays_thinking(messages): boolean end -- Binds the thinking blocks a Messages payload replays to drop_block and returns the --- request betas the binding requires, or nil when the payload carries no binding. +-- request betas its block_binding requires, or nil when the payload carries none. function mapper.bind_thinking(payload): {string}? - if not replays_thinking(payload.messages) then - return nil + if replays_thinking(payload.messages) then + thinking.bind_replayed_blocks(payload.thinking) end - local beta = thinking.bind_replayed_blocks(payload.thinking) - if not beta then + if not thinking.is_bound(payload.thinking) then return nil end - return { beta } + return { thinking.BINDING_BETA } end function mapper.extract_response_content(claude_response) @@ -728,14 +738,8 @@ function mapper.extract_response_content(claude_response) name = block.name or "", arguments = block.input or {} }) - elseif block.type == "thinking" then - table.insert(thinking_blocks, { - type = "thinking", - thinking = block.thinking or "", - signature = block.signature or "" - }) - elseif block.type == "redacted_thinking" then - table.insert(thinking_blocks, { type = "redacted_thinking", data = block.data }) + elseif block.type == "thinking" or block.type == "redacted_thinking" then + table.insert(thinking_blocks, thinking_block_shape(block)) end end diff --git a/src/llm/src/util/thinking.lua b/src/llm/src/util/thinking.lua index 0c9ab009..08f62f31 100644 --- a/src/llm/src/util/thinking.lua +++ b/src/llm/src/util/thinking.lua @@ -9,13 +9,17 @@ local BINDABLE_TYPES: {[string]: boolean} = { adaptive = true, enabled = true } -- Sets drop_block on a thinking object that replays earlier thinking blocks, so the API -- drops blocks bound to a different conversation instead of rejecting the request. --- Returns the beta the request then requires, or nil when the object takes no binding. -function thinking.bind_replayed_blocks(config: {[string]: any}?): string? - if type(config) ~= "table" or not BINDABLE_TYPES[config.type] then - return nil +-- A block_binding the object already carries stays as set. +function thinking.bind_replayed_blocks(config: {[string]: any}?) + if type(config) ~= "table" or not BINDABLE_TYPES[config.type] or config.block_binding ~= nil then + return end config.block_binding = { prefix_mismatch_behavior = "drop_block" } - return thinking.BINDING_BETA +end + +-- Whether a thinking object carries block_binding, which the API admits only with BINDING_BETA. +function thinking.is_bound(config: {[string]: any}?): boolean + return type(config) == "table" and config.block_binding ~= nil end function thinking.effort_level(effort: number): string diff --git a/src/llm/src/util/transport.lua b/src/llm/src/util/transport.lua index f2c13777..ec45a530 100644 --- a/src/llm/src/util/transport.lua +++ b/src/llm/src/util/transport.lua @@ -197,4 +197,33 @@ function transport.health_failure(request_err: RequestError): HealthFailure } end +-- Concatenates string lists in order, naming each value once. +function transport.merge_unique(lists: {{string}}): {string} + local merged: {string} = {} + local seen: {[string]: boolean} = {} + for _, list in ipairs(lists) do + for _, value in ipairs(list) do + if not seen[value] then + seen[value] = true + table.insert(merged, value) + end + end + end + return merged +end + +-- Values of a stream's 0-based, possibly sparse content block index in index order. +function transport.in_index_order(blocks: {[number]: any}): {any} + local indices: {number} = {} + for index in pairs(blocks) do + table.insert(indices, index) + end + table.sort(indices) + local ordered: {any} = {} + for _, index in ipairs(indices) do + table.insert(ordered, blocks[index]) + end + return ordered +end + return transport diff --git a/src/llm/test/bedrock/client_test.lua b/src/llm/test/bedrock/client_test.lua index f3331b3d..3b8627b0 100644 --- a/src/llm/test/bedrock/client_test.lua +++ b/src/llm/test/bedrock/client_test.lua @@ -230,6 +230,51 @@ local function define_tests() return content, err, result, seen, source end + it("keeps streamed reasoning, redacted reasoning and tool calls in response order", function() + local function reasoning(index, rc) + return event("contentBlockDelta", { contentBlockIndex = index, delta = { reasoningContent = rc } }) + end + local function tool(index, id) + return event("contentBlockStart", { + contentBlockIndex = index, + start = { toolUse = { toolUseId = id, name = "search" } } + }) + end + local events = { + event("messageStart", { role = "assistant" }), + reasoning(0, { text = "fir" }), + reasoning(0, { text = "st" }), + reasoning(0, { signature = "sig-0" }), + event("contentBlockStop", { contentBlockIndex = 0 }), + reasoning(1, { redactedContent = "b3BhcXVl" }), + event("contentBlockStop", { contentBlockIndex = 1 }), + reasoning(2, { text = "third" }), + reasoning(2, { signature = "sig-2" }), + event("contentBlockStop", { contentBlockIndex = 2 }), + tool(3, "call_a"), + event("contentBlockStop", { contentBlockIndex = 3 }), + tool(4, "call_b"), + event("contentBlockStop", { contentBlockIndex = 4 }), + event("messageStop", { stopReason = "tool_use" }), + } + + local _, err, result = run(events) + + test.is_nil(err) + local blocks = result.thinking + test.eq(#blocks, 3) + test.eq(blocks[1].type, "thinking") + test.eq(blocks[1].thinking, "first") + test.eq(blocks[1].signature, "sig-0") + test.eq(blocks[2].type, "redacted_thinking") + test.eq(blocks[2].data, "b3BhcXVl") + test.eq(blocks[3].thinking, "third") + test.eq(blocks[3].signature, "sig-2") + test.eq(#result.tool_calls, 2) + test.eq(result.tool_calls[1].id, "call_a") + test.eq(result.tool_calls[2].id, "call_b") + end) + it("reports input_transformations from messageStop in stream metadata", function() local events = conversation() events[6] = event("messageStop", { diff --git a/src/llm/test/bedrock/mapper_test.lua b/src/llm/test/bedrock/mapper_test.lua index a3769683..0cd1e66d 100644 --- a/src/llm/test/bedrock/mapper_test.lua +++ b/src/llm/test/bedrock/mapper_test.lua @@ -696,6 +696,123 @@ local function define_tests() end) end) + describe("Thinking binding", function() + local BINDING_BETA = "thinking-binding-controls-2026-08-01" + + local function payload(fields, response_paths): any + return { + messages = { + { role = "user", content = { { text = "Plan" } } }, + { role = "assistant", content = { { reasoningContent = { reasoningText = { text = "t", signature = "sig" } } } } }, + { role = "user", content = { { text = "Book" } } } + }, + additionalModelRequestFields = fields, + additionalModelResponseFieldPaths = response_paths + } + end + + it("appends the beta and response path to lists the payload already carries", function() + local p: any = payload( + { thinking = { type = "adaptive" }, anthropic_beta = { "context-1m-2025-08-07" } }, + { "/stop_sequence" } + ) + + mapper.bind_thinking(p) + + local fields: any = p.additionalModelRequestFields + test.eq(fields.thinking.block_binding.prefix_mismatch_behavior, "drop_block") + test.eq(#fields.anthropic_beta, 2) + test.eq(fields.anthropic_beta[1], "context-1m-2025-08-07") + test.eq(fields.anthropic_beta[2], BINDING_BETA) + test.eq(#p.additionalModelResponseFieldPaths, 2) + test.eq(p.additionalModelResponseFieldPaths[1], "/stop_sequence") + test.eq(p.additionalModelResponseFieldPaths[2], "/input_transformations") + end) + + it("names the beta and response path once when already present", function() + local p: any = payload( + { thinking = { type = "adaptive" }, anthropic_beta = { BINDING_BETA } }, + { "/input_transformations" } + ) + + mapper.bind_thinking(p) + + test.eq(#p.additionalModelRequestFields.anthropic_beta, 1) + test.eq(#p.additionalModelResponseFieldPaths, 1) + end) + + it("keeps a block_binding the payload already carries", function() + local p: any = payload({ thinking = { type = "adaptive", block_binding = { prefix_mismatch_behavior = "error" } } }) + + mapper.bind_thinking(p) + + local fields: any = p.additionalModelRequestFields + test.eq(fields.thinking.block_binding.prefix_mismatch_behavior, "error") + test.eq(fields.anthropic_beta[1], BINDING_BETA) + end) + + it("requires the beta for a block_binding set without replayed reasoning", function() + local p: any = { + messages = { { role = "user", content = { { text = "Plan" } } } }, + additionalModelRequestFields = { + thinking = { type = "adaptive", block_binding = { prefix_mismatch_behavior = "error" } } + } + } + + mapper.bind_thinking(p) + + test.eq(p.additionalModelRequestFields.anthropic_beta[1], BINDING_BETA) + test.eq(p.additionalModelResponseFieldPaths[1], "/input_transformations") + end) + end) + + describe("Redacted reasoning", function() + it("extracts redacted reasoning in response order", function() + local extracted = mapper.extract_response_content({ + output = { + message = { + role = "assistant", + content = { + { reasoningContent = { reasoningText = { text = "first", signature = "sig-1" } } }, + { reasoningContent = { redactedContent = "b3BhcXVl" } }, + { text = "Answer" } + } + } + } + }) + + test.eq(#extracted.thinking_blocks, 2) + test.eq(extracted.thinking_blocks[1].type, "thinking") + test.eq(extracted.thinking_blocks[1].signature, "sig-1") + test.eq(extracted.thinking_blocks[2].type, "redacted_thinking") + test.eq(extracted.thinking_blocks[2].data, "b3BhcXVl") + test.is_nil(extracted.thinking_blocks[2].thinking) + end) + + it("replays redacted blocks as redactedContent in their position", function() + local blocks = { + { type = "thinking", thinking = "first", signature = "sig-1" }, + { type = "redacted_thinking", thinking = "", data = "b3BhcXVl", signature = "" }, + { type = "thinking", thinking = "third", signature = "sig-3" } + } + local result = mapper.map_messages({ + { role = "user", content = { { type = "text", text = "Hi" } } }, + { role = "function_call", function_call = { id = "call_1", name = "tool", arguments = {} }, metadata = { thinking_blocks = blocks } }, + { role = "function_result", function_call_id = "call_1", content = "ok" }, + { role = "assistant", content = { { type = "text", text = "Done" } }, metadata = { thinking_blocks = blocks } } + }) + + for _, index in ipairs({ 2, 4 }) do + local content = result.messages[index].content :: any + test.eq(content[1].reasoningContent.reasoningText.text, "first") + test.eq(content[1].reasoningContent.reasoningText.signature, "sig-1") + test.eq(content[2].reasoningContent.redactedContent, "b3BhcXVl") + test.is_nil(content[2].reasoningContent.reasoningText) + test.eq(content[3].reasoningContent.reasoningText.text, "third") + end + end) + end) + describe("classify_error", function() it("should classify HTTP status to kind", function() local k, m, d = mapper.classify_error({ status_code = 429, message = "Rate limited" }) diff --git a/src/llm/test/claude/client_test.lua b/src/llm/test/claude/client_test.lua index e5f95ec0..9aed4b77 100644 --- a/src/llm/test/claude/client_test.lua +++ b/src/llm/test/claude/client_test.lua @@ -1223,6 +1223,42 @@ local function define_tests() test.eq(sent_beta, BINDING_BETA) end) + it("folds a configured anthropic-beta header into the merged beta list", function() + use({ + api_key = "test-key", + beta_features = { "prompt-caching-2024-07-31" }, + headers = { ["anthropic-beta"] = "context-1m-2025-08-07, prompt-caching-2024-07-31", ["x-team"] = "core" } + }) + local sent: any = nil + claude_client._http_client = { + post = function(url, options) + sent = options.headers + return { status_code = 200, body = "{}", headers = {} } + end + } + + claude_client.request("/v1/messages", {}, { beta_features = { BINDING_BETA } }) + + test.eq(sent["anthropic-beta"], "prompt-caching-2024-07-31,context-1m-2025-08-07," .. BINDING_BETA) + test.eq(sent["x-team"], "core") + end) + + it("folds a configured beta header of any letter case into one header", function() + use({ api_key = "test-key", headers = { ["Anthropic-Beta"] = "context-1m-2025-08-07" } }) + local sent: any = nil + claude_client._http_client = { + post = function(url, options) + sent = options.headers + return { status_code = 200, body = "{}", headers = {} } + end + } + + claude_client.request("/v1/messages", {}, { beta_features = { BINDING_BETA } }) + + test.eq(sent["anthropic-beta"], "context-1m-2025-08-07," .. BINDING_BETA) + test.is_nil(sent["Anthropic-Beta"]) + end) + it("reports input_transformations of a JSON response in metadata", function() use({ api_key = "test-key" }) claude_client._http_client = { diff --git a/src/llm/test/claude/generate_test.lua b/src/llm/test/claude/generate_test.lua index ecddfc6b..20296758 100644 --- a/src/llm/test/claude/generate_test.lua +++ b/src/llm/test/claude/generate_test.lua @@ -1675,6 +1675,33 @@ local function define_tests() test.is_nil(blocks[2].signature) end) + it("replays stored blocks in the API shape of their type", function() + local captured = {} + generate_handler._client = capturing_client(captured) + + generate_handler.handler({ + model = "claude-opus-5-5", + messages = history_with_thinking({ + { type = "thinking", thinking = "Weighing the routes.", data = "stray", signature = "sig-1" }, + { type = "redacted_thinking", thinking = "", data = "opaque-old", signature = "" } + }), + accepts = { thinking = "adaptive" }, + options = { thinking_effort = 50 } + }) + + for _, index in ipairs({ 2, 4 }) do + local content = captured.payload.messages[index].content + test.eq(content[1].type, "thinking") + test.eq(content[1].thinking, "Weighing the routes.") + test.eq(content[1].signature, "sig-1") + test.is_nil(content[1].data) + test.eq(content[2].type, "redacted_thinking") + test.eq(content[2].data, "opaque-old") + test.is_nil(content[2].thinking) + test.is_nil(content[2].signature) + end + end) + it("reports input_transformations from a non-streaming response in metadata", function() local captured = {} generate_handler._client = capturing_client(captured, { diff --git a/src/llm/test/claude/mapper_test.lua b/src/llm/test/claude/mapper_test.lua index 6d770680..cb46e1f2 100644 --- a/src/llm/test/claude/mapper_test.lua +++ b/src/llm/test/claude/mapper_test.lua @@ -621,6 +621,47 @@ local function define_tests() end) end) + describe("Thinking binding", function() + local BINDING_BETA = "thinking-binding-controls-2026-08-01" + local replayed = { + { role = "user", content = { { type = "text", text = "Plan" } } }, + { role = "assistant", content = { { type = "thinking", thinking = "t", signature = "sig" }, { type = "text", text = "Planned." } } }, + { role = "user", content = { { type = "text", text = "Book" } } } + } + local fresh = { { role = "user", content = { { type = "text", text = "Plan" } } } } + + it("keeps a block_binding the payload already carries", function() + local payload = { + messages = replayed, + thinking = { type = "adaptive", block_binding = { prefix_mismatch_behavior = "error" } } + } + + local betas = mapper.bind_thinking(payload) + + test.eq(payload.thinking.block_binding.prefix_mismatch_behavior, "error") + test.eq(#betas, 1) + test.eq(betas[1], BINDING_BETA) + end) + + it("requires the beta for a block_binding set without replayed blocks", function() + local payload = { + messages = fresh, + thinking = { type = "adaptive", block_binding = { prefix_mismatch_behavior = "error" } } + } + + local betas = mapper.bind_thinking(payload) + + test.eq(betas[1], BINDING_BETA) + end) + + it("requires no beta for a payload without block_binding", function() + local payload = { messages = fresh, thinking = { type = "adaptive" } } + + test.is_nil(mapper.bind_thinking(payload)) + test.is_nil(payload.thinking.block_binding) + end) + end) + describe("Options Mapping", function() it("should map basic options correctly", function() local contract_options = { From d05eb2ff464eefcb829802e49d9819fe58226482 Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 14:50:52 -0500 Subject: [PATCH 03/37] feat(llm): add thinking route fact always for models that always think thinking = always sends the adaptive thinking object at every effort; an absent or zero effort sends no output_config so the model default effort applies. The object carries block_binding at zero effort, so default effort routes survive handoffs. adaptive, budget and none are unchanged; OpenAI, OpenAI-compatible and Google reject always. --- src/llm/src/README.md | 16 +++++++---- src/llm/src/_index.yaml | 10 +++---- src/llm/src/bedrock/mapper.lua | 8 +++++- src/llm/src/claude/mapper.lua | 8 +++++- src/llm/src/llm.lua | 2 +- src/llm/src/util/route.lua | 4 +-- src/llm/test/bedrock/generate_test.lua | 18 ++++++++++++ src/llm/test/bedrock/mapper_test.lua | 22 +++++++++++++++ src/llm/test/claude/generate_test.lua | 19 +++++++++++++ src/llm/test/claude/mapper_test.lua | 28 +++++++++++++++++++ .../test/claude/structured_output_test.lua | 16 ++++++++--- src/llm/test/google/mapper_test.lua | 2 +- src/llm/test/openai/mapper_test.lua | 6 ++++ src/llm/test/openai_compat/mapper_test.lua | 6 ++++ src/llm/test/route_flow_test.lua | 16 +++++++++++ src/llm/test/util/route_test.lua | 11 ++++++++ 16 files changed, 171 insertions(+), 21 deletions(-) diff --git a/src/llm/src/README.md b/src/llm/src/README.md index 1c71c5d3..58619d6b 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -55,7 +55,7 @@ Each provider route declares what the model accepts on the wire. These optional | Fact | Values | Meaning | |---|---|---| -| `thinking` | `adaptive`, `budget`, `none` | How thinking effort is encoded | +| `thinking` | `adaptive`, `budget`, `none`, `always` | How thinking effort is encoded | | `sampling` | `true`, `false` | Whether temperature, top_p and top_k are accepted | | `forced_tool_choice` | `true`, `false` | Whether `tool_choice = "any"` or a named tool is accepted | | `structured_output` | `native`, `tool` | Native JSON schema output or a structured-output tool | @@ -81,6 +81,8 @@ Claude and Bedrock budget thinking never injects a temperature; the provider def Adaptive Claude and Bedrock requests with positive thinking_effort send `thinking.type = "adaptive"` and an effort level in `output_config`; they do not calculate a budget or touch temperature. An absent or zero effort sends no thinking fields. +`always` is for models whose thinking cannot be turned off, where omitting `thinking` and sending `{ type = "adaptive" }` are equivalent: Claude Opus 5.5, Sonnet 5.5, Haiku 5.5, Opus 5, Sonnet 5 and the Fable 5.x models. Claude and Bedrock send the adaptive thinking object on every request. A positive thinking_effort adds its effort level in `output_config`, exactly as `adaptive`; an absent or zero effort sends no `output_config`, so the model's default effort applies. Use `adaptive` only for models where an omitted thinking object means no thinking (Claude Opus 4.6, 4.7, 4.8 and Sonnet 4.6), and never declare `none` for a model that thinks when `thinking` is omitted. OpenAI, OpenAI-compatible and Google reject `always`. Routes for 5.x models that declare `adaptive` keep sending no thinking object at zero effort until they are changed to `always`. + The forced tool choice rule is provider-agnostic: a route with `forced_tool_choice = false` rejects a `tool_choice` of `"any"` or a tool name, for every driver, unless the caller permits `tool_choice_fallback = "auto"`. The caller must enforce tool use itself when accepting that fallback. The response reports `metadata.tool_choice = { requested = "any", sent = "auto" }` independently of `metadata.adjusted`. A route that can only produce structured output by forcing a tool (`structured_output = "tool"`, Bedrock's only mode) rejects `forced_tool_choice = false` the same way. Native Claude structured output requires `additionalProperties: false` on every object in the schema; an open object is rejected with its path. Legacy forms remain supported through the normalizer: @@ -88,7 +90,7 @@ Legacy forms remain supported through the normalizer: | Legacy form | Canonical form / behavior | |---|---| | Route `options.reasoning_model_request = true` | Only on a route whose driver declares it owns this flag (OpenAI, OpenAI-compatible): `thinking: adaptive`, `sampling: false`; false, absent, or a driver that does not declare it derives nothing | -| Route `options.model_profile.thinking_mode = adaptive_only` | `thinking: adaptive` only | +| Route `options.model_profile.thinking_mode = adaptive_only` | `thinking: adaptive` only (not `always`) | | Route `options.model_profile.forced_tool_choice` | `forced_tool_choice`, preserving both true and false | | Route `options.model_profile.structured_output_mode = native` | `structured_output: native` | | Caller `reasoning_model_request = true` | Same driver-declares-it rule as the route form; when it applies, derives adaptive thinking and no sampling unless canonically declared on the route | @@ -103,9 +105,11 @@ Provider open context composition is unchanged: provider entry `driver.options`, Claude and Bedrock store each response's `thinking` blocks as `{ type, thinking, signature }` and `redacted_thinking` blocks as `{ type, data }` in `metadata.thinking_blocks`, in response order, and replay every stored block on later turns in its position and in the provider shape of its type (Bedrock: `reasoningContent.reasoningText` and `reasoningContent.redactedContent`). A Claude thinking block is bound to the conversation that produced it: the top-level system prompt, the tools, and every earlier message. After an agent handoff, a prompt or tool change, or a history edit, models that enforce the check reject a request that replays such a block with a 400. -When a request replays at least one thinking block and sends a thinking object of type `adaptive` or `enabled` (a positive thinking_effort), the driver sets `thinking.block_binding.prefix_mismatch_behavior = "drop_block"` and adds the `thinking-binding-controls-2026-08-01` beta. The API then drops the first mismatched block and every later thinking block instead of rejecting the request. A `block_binding` the thinking object already carries is kept as set, and any request whose thinking object carries `block_binding` sends the beta. The Claude driver sends one `anthropic-beta` header: the configured `beta_features`, then the betas of a configured `anthropic-beta` context header (any letter case), then request betas, each named once. Bedrock appends the beta to the `anthropic_beta` list of `additionalModelRequestFields` and `/input_transformations` to `additionalModelResponseFieldPaths`, each named once. Plain generation and structured output behave the same. +When a request replays at least one thinking block and sends a thinking object of type `adaptive` or `enabled` (a positive thinking_effort, or `thinking = always` at any effort), the driver sets `thinking.block_binding.prefix_mismatch_behavior = "drop_block"` and adds the `thinking-binding-controls-2026-08-01` beta. The API then drops the first mismatched block and every later thinking block instead of rejecting the request. A `block_binding` the thinking object already carries is kept as set, and any request whose thinking object carries `block_binding` sends the beta. The Claude driver sends one `anthropic-beta` header: the configured `beta_features`, then the betas of a configured `anthropic-beta` context header (any letter case), then request betas, each named once. Bedrock appends the beta to the `anthropic_beta` list of `additionalModelRequestFields` and `/input_transformations` to `additionalModelResponseFieldPaths`, each named once. Plain generation and structured output behave the same. + +A request without a thinking object (`adaptive` or `budget` at zero or absent effort, or `thinking = "none"`) carries no binding: `block_binding` exists only inside an adaptive or enabled thinking object, and the driver never turns thinking on to carry it, since on several models an omitted thinking object means no thinking. Such a request replays its blocks unbound, so keep that conversation append-only. `always` is the form that carries the binding at zero effort. -A request without a thinking object (zero or absent effort, or `thinking = "none"`) carries no binding: `block_binding` exists only inside an adaptive or enabled thinking object, and the driver never turns thinking on to carry it, since on several models an omitted thinking object means no thinking. Such a request replays its blocks unbound, so keep that conversation append-only. Claude Sonnet 5.5 `thinking.type = "between_tools"` rejects `block_binding` the same way. +Claude Sonnet 5.5 `thinking.type = "between_tools"` rejects `block_binding`, so a between_tools request has no drop_block recovery. Routes that must survive agent handoffs or history edits use `thinking: always` with a low default effort (`options.thinking_effort: 20`); keep between_tools for append-only routes. between_tools also rejects the `xhigh` level that thinking_effort 80 and above maps to, and per-message effort changes. Responses to a bound request report each dropped block in `metadata.input_transformations`, for example `{ type = "thinking_dropped", path = "messages.1.content.0", reason = "prefix_binding_mismatch" }`. `prefix_binding_mismatch` means the conversation before the block changed; `model_binding_mismatch` means the conversation moved to a model that cannot read the block. An empty array means nothing was dropped. Streaming Claude responses report the array from `message_start`; Bedrock reports it from `messageStop`. Ignore entry types and reasons you do not recognize. @@ -586,7 +590,7 @@ entries: providers: - id: wippy.llm.claude:provider provider_model: claude-sonnet-5 - thinking: adaptive + thinking: always sampling: false forced_tool_choice: true structured_output: native @@ -606,7 +610,7 @@ Additional model entries can use these routes with the same top-level structure. providers: - id: wippy.llm.claude:provider provider_model: claude-opus-5-5 - thinking: adaptive + thinking: always sampling: false forced_tool_choice: false structured_output: native diff --git a/src/llm/src/_index.yaml b/src/llm/src/_index.yaml index 06476780..7b2135b1 100644 --- a/src/llm/src/_index.yaml +++ b/src/llm/src/_index.yaml @@ -255,7 +255,7 @@ entries: "accepts": { "type": "object", "properties": { - "thinking": {"type": "string", "enum": ["adaptive", "budget", "none"]}, + "thinking": {"type": "string", "enum": ["adaptive", "budget", "none", "always"], "description": "always: the model thinks on every request; the driver sends the adaptive thinking object at every effort, 0 leaves the effort at the model default"}, "sampling": {"type": "boolean"}, "forced_tool_choice": {"type": "boolean"}, "structured_output": {"type": "string", "enum": ["native", "tool"]} @@ -276,7 +276,7 @@ entries: "description": "Deprecated legacy route facts; normalized into top-level accepts by llm", "properties": { "forced_tool_choice": {"type": "boolean", "description": "false: the model rejects forced tool choice ('any' or a tool name)"}, - "thinking_mode": {"type": "string", "enum": ["adaptive_only"], "description": "adaptive_only: thinking is always on and thinking_effort maps to an effort level, not a budget"}, + "thinking_mode": {"type": "string", "enum": ["adaptive_only"], "description": "adaptive_only: equals thinking: adaptive; thinking_effort maps to an effort level, not a budget"}, "structured_output_mode": {"type": "string", "enum": ["native"], "description": "native: structured output uses the provider's JSON schema output instead of a forced tool; every object in the schema must set additionalProperties: false"} }, "additionalProperties": false @@ -546,7 +546,7 @@ entries: "accepts": { "type": "object", "properties": { - "thinking": {"type": "string", "enum": ["adaptive", "budget", "none"]}, + "thinking": {"type": "string", "enum": ["adaptive", "budget", "none", "always"], "description": "always: the model thinks on every request; the driver sends the adaptive thinking object at every effort, 0 leaves the effort at the model default"}, "sampling": {"type": "boolean"}, "forced_tool_choice": {"type": "boolean"}, "structured_output": {"type": "string", "enum": ["native", "tool"]} @@ -565,7 +565,7 @@ entries: "deprecated": true, "description": "Deprecated legacy route facts; normalized into top-level accepts by llm", "properties": { - "thinking_mode": {"type": "string", "enum": ["adaptive_only"]}, + "thinking_mode": {"type": "string", "enum": ["adaptive_only"], "description": "adaptive_only: equals thinking: adaptive"}, "forced_tool_choice": {"type": "boolean"}, "structured_output_mode": {"type": "string", "enum": ["native"]} }, @@ -814,7 +814,7 @@ entries: "properties": { "id": {"type": "string"}, "provider_model": {"type": "string"}, - "thinking": {"type": "string", "enum": ["adaptive", "budget", "none"]}, + "thinking": {"type": "string", "enum": ["adaptive", "budget", "none", "always"], "description": "always: the model thinks on every request; the driver sends the adaptive thinking object at every effort, 0 leaves the effort at the model default"}, "sampling": {"type": "boolean"}, "forced_tool_choice": {"type": "boolean"}, "structured_output": {"type": "string", "enum": ["native", "tool"]}, diff --git a/src/llm/src/bedrock/mapper.lua b/src/llm/src/bedrock/mapper.lua index 908ba6cd..af69930d 100644 --- a/src/llm/src/bedrock/mapper.lua +++ b/src/llm/src/bedrock/mapper.lua @@ -16,7 +16,7 @@ mapper.CAPABILITY = { structured_output = "tool" }, supported = { - thinking = { adaptive = true, budget = true, none = true }, + thinking = { adaptive = true, budget = true, none = true, always = true }, structured_output = { tool = true } } } @@ -499,6 +499,12 @@ function mapper.map_options(contract_options, accepts): (table, table, table, st if mode == "adaptive" and contract_options.thinking_effort and contract_options.thinking_effort > 0 then additional_fields.thinking = { type = "adaptive" } additional_fields.output_config = { effort = thinking.effort_level(contract_options.thinking_effort) } + elseif mode == "always" then + -- The model thinks on every request; without an effort it keeps the model default. + additional_fields.thinking = { type = "adaptive" } + if contract_options.thinking_effort and contract_options.thinking_effort > 0 then + additional_fields.output_config = { effort = thinking.effort_level(contract_options.thinking_effort) } + end elseif mode == "budget" and contract_options.thinking_effort and contract_options.thinking_effort > 0 then local thinking_budget = 1024 + (24000 - 1024) * (contract_options.thinking_effort / 100) thinking_budget = math.floor(thinking_budget + 0.5) diff --git a/src/llm/src/claude/mapper.lua b/src/llm/src/claude/mapper.lua index 382790ff..584f2259 100644 --- a/src/llm/src/claude/mapper.lua +++ b/src/llm/src/claude/mapper.lua @@ -16,7 +16,7 @@ mapper.CAPABILITY = { structured_output = "tool" }, supported = { - thinking = { adaptive = true, budget = true, none = true }, + thinking = { adaptive = true, budget = true, none = true, always = true }, structured_output = { native = true, tool = true } } } @@ -667,6 +667,12 @@ function mapper.map_options(contract_options, accepts): (table, table, string?) claude_options.thinking = { type = "adaptive" } claude_options.output_config = { effort = thinking.effort_level(contract_options.thinking_effort) } end + elseif mode == "always" then + -- The model thinks on every request; without an effort it keeps the model default. + claude_options.thinking = { type = "adaptive" } + if contract_options.thinking_effort and contract_options.thinking_effort > 0 then + claude_options.output_config = { effort = thinking.effort_level(contract_options.thinking_effort) } + end elseif mode == "budget" and contract_options.thinking_effort and contract_options.thinking_effort > 0 then local thinking_budget = 1024 + (24000 - 1024) * (contract_options.thinking_effort / 100) thinking_budget = math.floor(thinking_budget + 0.5) diff --git a/src/llm/src/llm.lua b/src/llm/src/llm.lua index 42c8f0d5..9840debb 100644 --- a/src/llm/src/llm.lua +++ b/src/llm/src/llm.lua @@ -110,7 +110,7 @@ type ProviderRef = { id: string, provider_model: string?, context: table?, - thinking: "adaptive" | "budget" | "none" | nil, + thinking: "adaptive" | "budget" | "none" | "always" | nil, sampling: boolean?, forced_tool_choice: boolean?, structured_output: "native" | "tool" | nil, diff --git a/src/llm/src/util/route.lua b/src/llm/src/util/route.lua index 57bcc7a2..3a63c3cf 100644 --- a/src/llm/src/util/route.lua +++ b/src/llm/src/util/route.lua @@ -1,13 +1,13 @@ local route = {} local VALUES = { - thinking = { adaptive = true, budget = true, none = true }, + thinking = { adaptive = true, budget = true, none = true, always = true }, sampling = "boolean", forced_tool_choice = "boolean", structured_output = { native = true, tool = true } } local ALLOWED = { - thinking = "adaptive, budget, none", + thinking = "adaptive, budget, none, always", sampling = "boolean (true, false)", forced_tool_choice = "boolean (true, false)", structured_output = "native, tool" diff --git a/src/llm/test/bedrock/generate_test.lua b/src/llm/test/bedrock/generate_test.lua index 3baa5a37..b2f21d62 100644 --- a/src/llm/test/bedrock/generate_test.lua +++ b/src/llm/test/bedrock/generate_test.lua @@ -352,6 +352,24 @@ local function define_tests() test.eq(fields.anthropic_beta[1], BINDING_BETA) end) + it("binds replayed reasoning on an always route at the model's default effort", function() + local captured = capture() + + generate_handler.handler({ + model = CLAUDE_MODEL, + messages = history_with_thinking({ SIGNED_THINKING }), + accepts = { thinking = "always" }, + options = { thinking_effort = 0 } + }) + + local fields = captured.payload.additionalModelRequestFields + test.eq(fields.thinking.type, "adaptive") + test.is_nil(fields.output_config) + test.eq(fields.thinking.block_binding.prefix_mismatch_behavior, "drop_block") + test.eq(fields.anthropic_beta[1], BINDING_BETA) + test.eq(captured.payload.additionalModelResponseFieldPaths[1], "/input_transformations") + end) + it("sends no binding when the history replays no reasoning", function() local captured = capture() diff --git a/src/llm/test/bedrock/mapper_test.lua b/src/llm/test/bedrock/mapper_test.lua index 0cd1e66d..5deafa3f 100644 --- a/src/llm/test/bedrock/mapper_test.lua +++ b/src/llm/test/bedrock/mapper_test.lua @@ -2,6 +2,13 @@ local mapper = require("mapper") local output = require("output") local json = require("json") +local function same(a: any, b: any): boolean + if type(a) ~= "table" or type(b) ~= "table" then return a == b end + for k, v in pairs(a) do if not same(v, b[k]) then return false end end + for k in pairs(b) do if a[k] == nil then return false end end + return true +end + local function define_tests() describe("Bedrock Converse Mapper", function() @@ -104,6 +111,21 @@ local function define_tests() test.eq(config.temperature, 1) test.is_nil(adjusted.temperature) end) + it("sends the adaptive thinking object without an effort for always at no effort", function() + for _, options in ipairs({ {}, { thinking_effort = 0 } }) do + local config, fields, _, err = mapper.map_options(options, { thinking = "always" }) + test.is_nil(err) + test.eq(fields.thinking.type, "adaptive") + test.is_nil(fields.output_config) + test.is_nil(config.maxTokens) + end + end) + it("encodes always with a positive effort exactly as adaptive", function() + local _, always = mapper.map_options({ thinking_effort = 80 }, { thinking = "always" }) + local _, adaptive = mapper.map_options({ thinking_effort = 80 }, { thinking = "adaptive" }) + test.eq(always.output_config.effort, "xhigh") + test.is_true(same(always, adaptive)) + end) it("omits thinking for none and zero adaptive effort", function() for _, thinking in ipairs({ "none", "adaptive" }) do local config, fields = mapper.map_options({ thinking_effort = thinking == "none" and 50 or 0 }, { thinking = thinking }) diff --git a/src/llm/test/claude/generate_test.lua b/src/llm/test/claude/generate_test.lua index 20296758..abc88ef7 100644 --- a/src/llm/test/claude/generate_test.lua +++ b/src/llm/test/claude/generate_test.lua @@ -1571,6 +1571,25 @@ local function define_tests() test.eq(captured.options.beta_features[1], BINDING_BETA) end) + it("binds replayed blocks on an always route at the model's default effort", function() + local captured = {} + generate_handler._client = capturing_client(captured) + + generate_handler.handler({ + model = "claude-opus-5-5", + messages = history_with_thinking({ SIGNED_THINKING }), + accepts = { thinking = "always" }, + options = {} + }) + + local thinking = captured.payload.thinking + test.eq(thinking.type, "adaptive") + test.eq(thinking.block_binding.prefix_mismatch_behavior, "drop_block") + test.is_nil(captured.payload.output_config) + test.eq(#captured.options.beta_features, 1) + test.eq(captured.options.beta_features[1], BINDING_BETA) + end) + it("sends no binding when the history replays no thinking blocks", function() local captured = {} generate_handler._client = capturing_client(captured) diff --git a/src/llm/test/claude/mapper_test.lua b/src/llm/test/claude/mapper_test.lua index cb46e1f2..b30c48e9 100644 --- a/src/llm/test/claude/mapper_test.lua +++ b/src/llm/test/claude/mapper_test.lua @@ -5,6 +5,13 @@ local json = require("json") local hash = require("hash") local test = require("test") +local function same(a: any, b: any): boolean + if type(a) ~= "table" or type(b) ~= "table" then return a == b end + for k, v in pairs(a) do if not same(v, b[k]) then return false end end + for k in pairs(b) do if a[k] == nil then return false end end + return true +end + local function define_tests() describe("Claude Mapper", function() @@ -724,6 +731,27 @@ local function define_tests() test.is_nil(result.thinking) end) + it("sends the adaptive thinking object without an effort for always at no effort", function() + for _, options in ipairs({ { max_tokens = 1000, temperature = 0.3 }, { max_tokens = 1000, temperature = 0.3, thinking_effort = 0 } }) do + local result, adjusted, err = mapper.map_options(options, { thinking = "always" }) + test.is_nil(err) + test.eq(result.thinking.type, "adaptive") + test.is_nil(next(result.thinking, next(result.thinking))) + test.is_nil(result.output_config) + test.eq(result.temperature, 0.3) + test.eq(result.max_tokens, 1000) + test.is_nil(next(adjusted)) + end + end) + + it("encodes always with a positive effort exactly as adaptive", function() + local always = mapper.map_options({ thinking_effort = 50, max_tokens = 1000 }, { thinking = "always" }) + local adaptive = mapper.map_options({ thinking_effort = 50, max_tokens = 1000 }, { thinking = "adaptive" }) + test.eq(always.thinking.type, "adaptive") + test.eq(always.output_config.effort, "medium") + test.is_true(same(always, adaptive)) + end) + it("should keep the explicit temperature an adaptive-only caller sets", function() local result = mapper.map_options({ temperature = 0.2, thinking_effort = 50 }, { thinking = "adaptive" }) test.eq(result.temperature, 0.2) diff --git a/src/llm/test/claude/structured_output_test.lua b/src/llm/test/claude/structured_output_test.lua index 99f7627a..2d84d88f 100644 --- a/src/llm/test/claude/structured_output_test.lua +++ b/src/llm/test/claude/structured_output_test.lua @@ -898,7 +898,7 @@ local function define_tests() additionalProperties = false } - local function send(structured_output_mode) + local function send(structured_output_mode, thinking_fact, thinking_effort) local sent = {} structured_output_handler._client._ctx = { all = function() return { api_key = "test-api-key" } end } structured_output_handler._client._env = { get = function() return nil end } @@ -935,20 +935,28 @@ local function define_tests() { role = "user", content = { { type = "text", text = "Answer" } } } }, schema = schema, - options = { thinking_effort = 50 }, - accepts = { thinking = "adaptive", structured_output = structured_output_mode } + options = { thinking_effort = thinking_effort }, + accepts = { thinking = thinking_fact, structured_output = structured_output_mode } }) return sent, response end for _, mode in ipairs({ "native", "tool" }) do it("binds replayed blocks on the " .. mode .. " path", function() - local sent, response = send(mode) + local sent, response = send(mode, "adaptive", 50) test.eq(sent.body.thinking.block_binding.prefix_mismatch_behavior, "drop_block") test.eq(sent.headers["anthropic-beta"], "thinking-binding-controls-2026-08-01") test.eq(response.metadata.input_transformations[1].reason, "prefix_binding_mismatch") end) + + it("binds replayed blocks on the " .. mode .. " path of an always route at no effort", function() + local sent = send(mode, "always", nil) + + test.eq(sent.body.thinking.type, "adaptive") + test.eq(sent.body.thinking.block_binding.prefix_mismatch_behavior, "drop_block") + test.eq(sent.headers["anthropic-beta"], "thinking-binding-controls-2026-08-01") + end) end end) diff --git a/src/llm/test/google/mapper_test.lua b/src/llm/test/google/mapper_test.lua index ccb6e769..2d8df25f 100644 --- a/src/llm/test/google/mapper_test.lua +++ b/src/llm/test/google/mapper_test.lua @@ -80,7 +80,7 @@ local function define_tests() describe("Declared thinking", function() it("rejects thinking modes it cannot honor", function() - for _, thinking in ipairs({ "adaptive", "budget" }) do + for _, thinking in ipairs({ "adaptive", "budget", "always" }) do local options, err = mapper.map_options({}, { thinking = thinking }) test.is_nil(options) test.contains(err, "Google") diff --git a/src/llm/test/openai/mapper_test.lua b/src/llm/test/openai/mapper_test.lua index 20e0d769..55ae6771 100644 --- a/src/llm/test/openai/mapper_test.lua +++ b/src/llm/test/openai/mapper_test.lua @@ -111,6 +111,12 @@ local function define_tests() test.contains(err, "budget") end) + it("rejects a declared thinking = always, naming the supported values", function() + local _, err = openai_mapper.map_options({}, { thinking = "always" }) + test.contains(err, "OpenAI does not support thinking = always") + test.contains(err, "supported: adaptive, none") + end) + it("rejects a declared structured_output mode it cannot honor", function() local opts, err = openai_mapper.map_options({}, { structured_output = "tool" }) test.contains(err, "structured_output") diff --git a/src/llm/test/openai_compat/mapper_test.lua b/src/llm/test/openai_compat/mapper_test.lua index b47e7eba..a220b13b 100644 --- a/src/llm/test/openai_compat/mapper_test.lua +++ b/src/llm/test/openai_compat/mapper_test.lua @@ -96,6 +96,12 @@ local function define_tests() test.contains(err, "budget") end) + it("rejects a declared thinking = always, naming the supported values", function() + local _, err = openai_mapper.map_options({}, { thinking = "always" }) + test.contains(err, "OpenAI-compatible does not support thinking = always") + test.contains(err, "supported: adaptive, none") + end) + it("rejects a declared structured_output mode it cannot honor", function() local opts, err = openai_mapper.map_options({}, { structured_output = "tool" }) test.contains(err, "structured_output") diff --git a/src/llm/test/route_flow_test.lua b/src/llm/test/route_flow_test.lua index 57696cab..60f67531 100644 --- a/src/llm/test/route_flow_test.lua +++ b/src/llm/test/route_flow_test.lua @@ -123,6 +123,22 @@ local function define_tests() test.is_nil(sent.payload) end) + it("sends the adaptive thinking object for " .. provider .. " " .. method .. " on an always route without effort", function() + for _, case in ipairs({ { thinking = "always", sends = true }, { thinking = "adaptive", sends = false } }) do + local sent = setup() + local _, err = call(method, { provider_id = "route", model = "wire", accepts = { thinking = case.thinking } }) + test.is_nil(err) + local payload = sent.payload :: table + local fields = provider == "claude" and payload or (payload.additionalModelRequestFields or {}) + if case.sends then + test.eq(fields.thinking.type, "adaptive") + else + test.is_nil(fields.thinking) + end + test.is_nil(fields.output_config) + end + end) + it("encodes " .. provider .. " " .. method .. " adaptive thinking after central sampling removal", function() local sent = setup() local result, err = call(method, { provider_id = "route", model = "wire", diff --git a/src/llm/test/util/route_test.lua b/src/llm/test/util/route_test.lua index 570ffdbe..c1646069 100644 --- a/src/llm/test/util/route_test.lua +++ b/src/llm/test/util/route_test.lua @@ -87,6 +87,17 @@ local function define_tests() end end end) + it("accepts thinking = always in direct and resolved modes", function() + equal(route.accepts({ id = "direct" }, { accepts = { thinking = "always" } }, "direct"), { thinking = "always" }) + equal(route.accepts({ id = "route", thinking = "always" }, {}, "resolved"), { thinking = "always" }) + equal(route.accepts({ options = { model_profile = { thinking_mode = "adaptive_only" } } }, {}, "resolved"), + { thinking = "adaptive" }) + end) + it("lists always among the thinking values a rejection names", function() + local facts, err = route.accepts({ id = "bad-route", thinking = "sometimes" }, {}, "resolved") + test.is_nil(facts) + test.contains(err, "adaptive, budget, none, always") + end) it("rejects non-table accepts and unknown accepts keys", function() for _, value in ipairs({ false, "adaptive", { unexpected = true } }) do local facts, err = route.accepts({ id = "direct" }, { accepts = value }, "direct") From b1cf7f7a765dabd6c343c92554f73fe36d2658c3 Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 15:15:30 -0500 Subject: [PATCH 04/37] fix(llm): bind thinking last, pass replayed block fields through, join redacted bytes Bind thinking as the last change to each driver's wire body so the binding follows the thinking object actually sent. Replay stored Claude thinking blocks unchanged apart from stale fields of an earlier shape. Join Bedrock streamed redactedContent deltas by their decoded bytes. Add thinking helper tests and a shared deep_equal test helper, and document steady-state drop_block behavior and enforcement opt-in. --- src/llm/src/README.md | 8 +-- src/llm/src/bedrock/client.lua | 18 +++++-- src/llm/src/bedrock/generate.lua | 5 +- src/llm/src/bedrock/structured_output.lua | 1 + src/llm/src/claude/generate.lua | 6 ++- src/llm/src/claude/mapper.lua | 28 +++++++--- src/llm/src/claude/structured_output.lua | 14 ++--- src/llm/test/_index.yaml | 21 ++++++++ src/llm/test/bedrock/client_test.lua | 46 ++++++++++++++++ src/llm/test/bedrock/generate_test.lua | 33 ++++++++++++ src/llm/test/bedrock/mapper_test.lua | 7 +-- .../test/bedrock/structured_output_test.lua | 46 ++++++++++++++++ src/llm/test/claude/generate_test.lua | 54 +++++++++++++++++++ src/llm/test/claude/mapper_test.lua | 7 +-- src/llm/test/support/deep_equal.lua | 19 +++++++ src/llm/test/util/thinking_test.lua | 42 +++++++++++++++ 16 files changed, 319 insertions(+), 36 deletions(-) create mode 100644 src/llm/test/support/deep_equal.lua create mode 100644 src/llm/test/util/thinking_test.lua diff --git a/src/llm/src/README.md b/src/llm/src/README.md index 58619d6b..6e5d5645 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -103,17 +103,17 @@ Provider open context composition is unchanged: provider entry `driver.options`, ### Thinking block binding -Claude and Bedrock store each response's `thinking` blocks as `{ type, thinking, signature }` and `redacted_thinking` blocks as `{ type, data }` in `metadata.thinking_blocks`, in response order, and replay every stored block on later turns in its position and in the provider shape of its type (Bedrock: `reasoningContent.reasoningText` and `reasoningContent.redactedContent`). A Claude thinking block is bound to the conversation that produced it: the top-level system prompt, the tools, and every earlier message. After an agent handoff, a prompt or tool change, or a history edit, models that enforce the check reject a request that replays such a block with a 400. +Claude and Bedrock store each response's `thinking` blocks as `{ type, thinking, signature }` and `redacted_thinking` blocks as `{ type, data }` in `metadata.thinking_blocks`, in response order, and replay every stored block on later turns in its position (Bedrock: as `reasoningContent.reasoningText` and `reasoningContent.redactedContent`). Claude replays each block unchanged except for fields an earlier stored shape carried that its type does not accept (`data` on `thinking`, `thinking` and `signature` on `redacted_thinking`). A Claude thinking block is bound to the conversation that produced it: the top-level system prompt, the tools, and every earlier message. After an agent handoff, a prompt or tool change, or a history edit, models that enforce the check reject a request that replays such a block with a 400. -When a request replays at least one thinking block and sends a thinking object of type `adaptive` or `enabled` (a positive thinking_effort, or `thinking = always` at any effort), the driver sets `thinking.block_binding.prefix_mismatch_behavior = "drop_block"` and adds the `thinking-binding-controls-2026-08-01` beta. The API then drops the first mismatched block and every later thinking block instead of rejecting the request. A `block_binding` the thinking object already carries is kept as set, and any request whose thinking object carries `block_binding` sends the beta. The Claude driver sends one `anthropic-beta` header: the configured `beta_features`, then the betas of a configured `anthropic-beta` context header (any letter case), then request betas, each named once. Bedrock appends the beta to the `anthropic_beta` list of `additionalModelRequestFields` and `/input_transformations` to `additionalModelResponseFieldPaths`, each named once. Plain generation and structured output behave the same. +When a request replays at least one thinking block and sends a thinking object of type `adaptive` or `enabled` (a positive thinking_effort, or `thinking = always` at any effort), the driver sets `thinking.block_binding.prefix_mismatch_behavior = "drop_block"` and adds the `thinking-binding-controls-2026-08-01` beta. The API then drops the first mismatched block and every later thinking block instead of rejecting the request. The drop applies to that request only and the history keeps the blocks, so after one change to the system prompt, tools or an earlier message, every later request again drops every thinking block from the first one bound to the old prefix onward, until the history before the change no longer carries thinking blocks; such conversations continue without that earlier reasoning, and `metadata.input_transformations` shows the drops on every response. Setting `prefix_mismatch_behavior` also opts the request into enforcement: on models that enforce the check only for organizations created on or after 2026-08-31, an older organization's mismatched blocks reached the model before and are now dropped. A `block_binding` the thinking object already carries is kept as set, and any request whose thinking object carries `block_binding` sends the beta. The Claude driver sends one `anthropic-beta` header: the configured `beta_features`, then the betas of a configured `anthropic-beta` context header (any letter case), then request betas, each named once. Bedrock appends the beta to the `anthropic_beta` list of `additionalModelRequestFields` and `/input_transformations` to `additionalModelResponseFieldPaths`, each named once. Plain generation and structured output behave the same. A request without a thinking object (`adaptive` or `budget` at zero or absent effort, or `thinking = "none"`) carries no binding: `block_binding` exists only inside an adaptive or enabled thinking object, and the driver never turns thinking on to carry it, since on several models an omitted thinking object means no thinking. Such a request replays its blocks unbound, so keep that conversation append-only. `always` is the form that carries the binding at zero effort. -Claude Sonnet 5.5 `thinking.type = "between_tools"` rejects `block_binding`, so a between_tools request has no drop_block recovery. Routes that must survive agent handoffs or history edits use `thinking: always` with a low default effort (`options.thinking_effort: 20`); keep between_tools for append-only routes. between_tools also rejects the `xhigh` level that thinking_effort 80 and above maps to, and per-message effort changes. +Claude Sonnet 5.5 `thinking.type = "between_tools"` rejects `block_binding`, so a between_tools request has no drop_block recovery. Routes that must survive agent handoffs or history edits use `thinking: always` with a low default effort (`options.thinking_effort: 10`, the `low` level; 20 through 50 map to `medium`); keep between_tools for append-only routes. between_tools also rejects the `xhigh` level that thinking_effort 80 and above maps to, and per-message effort changes. Responses to a bound request report each dropped block in `metadata.input_transformations`, for example `{ type = "thinking_dropped", path = "messages.1.content.0", reason = "prefix_binding_mismatch" }`. `prefix_binding_mismatch` means the conversation before the block changed; `model_binding_mismatch` means the conversation moved to a model that cannot read the block. An empty array means nothing was dropped. Streaming Claude responses report the array from `message_start`; Bedrock reports it from `messageStop`. Ignore entry types and reasons you do not recognize. -Model switches and fallbacks need no client-side handling: the API drops blocks the target model cannot read, so the drivers pass every block back unchanged. +Model switches and fallbacks need no client-side handling: the API drops blocks the target model cannot read, so the drivers pass every block back unchanged. Prefix changes are different: keeping the system prompt, tools and earlier messages append-only is what keeps earlier reasoning in use. The binding controls are a beta on the Claude API, Claude Platform on AWS, Amazon Bedrock and Google Cloud Vertex AI; Microsoft Foundry is unconfirmed. An endpoint that rejects the beta fails the request with its own error. diff --git a/src/llm/src/bedrock/client.lua b/src/llm/src/bedrock/client.lua index dc9bbc97..5c42a188 100644 --- a/src/llm/src/bedrock/client.lua +++ b/src/llm/src/bedrock/client.lua @@ -415,6 +415,7 @@ function bedrock_client.process_converse_stream(stream_response, callbacks) local content_blocks = {} local error_reported = false local metadata = stream_response.metadata or {} + local redacted_bytes: {[number]: string} = {} local function handle_event(event): string? -- Converse stream events use top-level keys instead of a "type" field @@ -452,9 +453,15 @@ function bedrock_client.process_converse_stream(stream_response, callbacks) -- ConverseStream opens reasoning blocks with their first delta, not contentBlockStart. local rc = delta.reasoningContent if rc.redactedContent then - local block = thinking_blocks[index] or { type = "redacted_thinking", data = "" } - block.data = block.data .. rc.redactedContent - thinking_blocks[index] = block + -- Each delta is its own base64 blob; the block's bytes are encoded once at contentBlockStop. + local bytes = base64.decode(tostring(rc.redactedContent)) + if not bytes then + error_reported = true + on_error({ message = "Invalid base64 in redactedContent delta" }) + return "Invalid base64 in redactedContent delta" + end + thinking_blocks[index] = thinking_blocks[index] or { type = "redacted_thinking", data = "" } + redacted_bytes[index] = (redacted_bytes[index] or "") .. bytes else local block = thinking_blocks[index] or { type = "thinking", thinking = "", signature = "" } thinking_blocks[index] = block @@ -474,6 +481,11 @@ function bedrock_client.process_converse_stream(stream_response, callbacks) elseif event.contentBlockStop then local index = (event.contentBlockStop.contentBlockIndex or 0) + if redacted_bytes[index] then + thinking_blocks[index].data = base64.encode(redacted_bytes[index]) + redacted_bytes[index] = nil + end + if tool_calls[index] and tool_calls[index].partial_json then local json_str = tostring(tool_calls[index].partial_json) local arguments = {} diff --git a/src/llm/src/bedrock/generate.lua b/src/llm/src/bedrock/generate.lua index 832623f8..2a627bf6 100644 --- a/src/llm/src/bedrock/generate.lua +++ b/src/llm/src/bedrock/generate.lua @@ -127,8 +127,6 @@ function generate_handler.handler(contract_args) converse_payload.additionalModelRequestFields = additional_fields end - generate_handler._mapper.bind_thinking(converse_payload) - if contract_args.tools and #contract_args.tools > 0 then local converse_tools, name_to_id_map = generate_handler._mapper.map_tools(contract_args.tools) @@ -154,6 +152,9 @@ function generate_handler.handler(contract_args) retry = contract_args.retry } + -- The binding depends on the thinking object as sent, so it is the last change to the payload. + generate_handler._mapper.bind_thinking(converse_payload) + if contract_args.stream and contract_args.stream.reply_to then local stream_response, stream_err = generate_handler._client.converse_stream( contract_args.model, diff --git a/src/llm/src/bedrock/structured_output.lua b/src/llm/src/bedrock/structured_output.lua index 203b1717..8803f6d1 100644 --- a/src/llm/src/bedrock/structured_output.lua +++ b/src/llm/src/bedrock/structured_output.lua @@ -137,6 +137,7 @@ function structured_output_handler.handler(contract_args) converse_payload.additionalModelRequestFields = additional_fields end + -- The binding depends on the thinking object as sent, so it is the last change to the payload. structured_output_handler._mapper.bind_thinking(converse_payload) local response, request_err = structured_output_handler._client.converse( diff --git a/src/llm/src/claude/generate.lua b/src/llm/src/claude/generate.lua index 9efcf9af..57923878 100644 --- a/src/llm/src/claude/generate.lua +++ b/src/llm/src/claude/generate.lua @@ -149,8 +149,7 @@ function generate_handler.handler(contract_args) local request_options = { timeout = contract_args.timeout or 600, - retry = contract_args.retry, - beta_features = generate_handler._mapper.bind_thinking(claude_payload) + retry = contract_args.retry } local stream_config = nil @@ -159,6 +158,9 @@ function generate_handler.handler(contract_args) stream_config = contract_args.stream end + -- The binding depends on the thinking object as sent, so it is the last change to the payload. + request_options.beta_features = generate_handler._mapper.bind_thinking(claude_payload) + local response, request_err = generate_handler._client.request( generate_handler._client.ENDPOINTS.MESSAGES, claude_payload, diff --git a/src/llm/src/claude/mapper.lua b/src/llm/src/claude/mapper.lua index 584f2259..96e4fcfe 100644 --- a/src/llm/src/claude/mapper.lua +++ b/src/llm/src/claude/mapper.lua @@ -217,15 +217,31 @@ function mapper.map_finish_reason(claude_stop_reason) return mapper.FINISH_REASON_MAP[claude_stop_reason] or claude_stop_reason end --- A thinking or redacted_thinking block in the Messages API shape of its type, for --- storing a response block and replaying a stored one; other blocks yield nil. +-- Fields an earlier stored shape carried on a block type the Messages API does not +-- accept them on. +local STALE_THINKING_FIELDS: {[string]: {[string]: boolean}} = { + thinking = { data = true }, + redacted_thinking = { thinking = true, signature = true } +} + +-- A thinking or redacted_thinking block as the API takes it back: every field passes +-- through unchanged except the stale fields of its type; other blocks yield nil. local function thinking_block_shape(block) + local stale = STALE_THINKING_FIELDS[block.type] + if not stale then + return nil + end + local shaped = {} + for key, value in pairs(block) do + if not stale[key] then + shaped[key] = value + end + end if block.type == "thinking" then - return { type = "thinking", thinking = block.thinking or "", signature = block.signature or "" } - elseif block.type == "redacted_thinking" then - return { type = "redacted_thinking", data = block.data } + shaped.thinking = shaped.thinking or "" + shaped.signature = shaped.signature or "" end - return nil + return shaped end local function extract_thinking_content_from_blocks(thinking_blocks) diff --git a/src/llm/src/claude/structured_output.lua b/src/llm/src/claude/structured_output.lua index 427196c4..b4caa897 100644 --- a/src/llm/src/claude/structured_output.lua +++ b/src/llm/src/claude/structured_output.lua @@ -201,14 +201,12 @@ function structured_output_handler.handler(contract_args) end native_payload.output_config = output_config + -- The binding depends on the thinking object as sent, so it is the last change to the payload. + local native_betas = structured_output_handler._mapper.bind_thinking(native_payload) local response, request_err = structured_output_handler._client.request( structured_output_handler._client.ENDPOINTS.MESSAGES, native_payload, - { - timeout = contract_args.timeout, - retry = contract_args.retry, - beta_features = structured_output_handler._mapper.bind_thinking(native_payload) - } + { timeout = contract_args.timeout, retry = contract_args.retry, beta_features = native_betas } ) if request_err then return nil, err:from(request_err):build() @@ -270,10 +268,12 @@ function structured_output_handler.handler(contract_args) local request_options = { timeout = contract_args.timeout, - retry = contract_args.retry, - beta_features = structured_output_handler._mapper.bind_thinking(claude_payload) + retry = contract_args.retry } + -- The binding depends on the thinking object as sent, so it is the last change to the payload. + request_options.beta_features = structured_output_handler._mapper.bind_thinking(claude_payload) + local response, request_err = structured_output_handler._client.request( structured_output_handler._client.ENDPOINTS.MESSAGES, claude_payload, diff --git a/src/llm/test/_index.yaml b/src/llm/test/_index.yaml index 5eb8105b..f4e33752 100644 --- a/src/llm/test/_index.yaml +++ b/src/llm/test/_index.yaml @@ -355,6 +355,7 @@ entries: source: file://bedrock/client_test.lua modules: - json + - base64 imports: bedrock_client: wippy.llm.bedrock:client test: wippy.test:test @@ -375,6 +376,7 @@ entries: modules: - json imports: + deep_equal: app:deep_equal mapper: wippy.llm.bedrock:mapper output: wippy.llm:output test: wippy.test:test @@ -580,6 +582,7 @@ entries: - json - hash imports: + deep_equal: app:deep_equal prompt: wippy.llm:prompt mapper: wippy.llm.claude:mapper output: wippy.llm:output @@ -1427,6 +1430,24 @@ entries: test: wippy.test:test method: run_tests + - name: deep_equal + kind: library.lua + meta: + comment: Structural equality helper shared by tests + source: file://support/deep_equal.lua + + - name: util_thinking_test + kind: function.lua + meta: + name: Thinking Binding Helpers Test + type: test + group: LLM / Utils + source: file://util/thinking_test.lua + imports: + thinking: wippy.llm.util:thinking + test: wippy.test:test + method: run_tests + - name: util_route_test kind: function.lua meta: diff --git a/src/llm/test/bedrock/client_test.lua b/src/llm/test/bedrock/client_test.lua index 3b8627b0..340145a4 100644 --- a/src/llm/test/bedrock/client_test.lua +++ b/src/llm/test/bedrock/client_test.lua @@ -1,5 +1,6 @@ local bedrock_client = require("bedrock_client") local json = require("json") +local base64 = require("base64") local function define_tests() describe("Bedrock Client", function() @@ -152,6 +153,21 @@ local function define_tests() test.eq(#response.metadata.input_transformations, 1) test.eq(response.metadata.input_transformations[1].path, "messages.1.content.0") end) + + it("leaves input_transformations unset when the response carries no model response fields", function() + use_context({}) + bedrock_client._http_client = { + post = function() + return { status_code = 200, headers = {}, body = json.encode({ stopReason = "end_turn" }) } + end + } + + local response, err = bedrock_client.converse("test-model", { messages = {} }) + + test.is_nil(err) + test.not_nil(response.metadata) + test.is_nil(response.metadata.input_transformations) + end) end) describe("ConverseStream", function() @@ -275,6 +291,36 @@ local function define_tests() test.eq(result.tool_calls[2].id, "call_b") end) + it("joins redactedContent deltas by their bytes into one base64 value", function() + local first = base64.encode("ab") + local second = base64.encode("cde") + local events = { + event("messageStart", { role = "assistant" }), + event("contentBlockDelta", { contentBlockIndex = 0, delta = { reasoningContent = { redactedContent = first } } }), + event("contentBlockDelta", { contentBlockIndex = 0, delta = { reasoningContent = { redactedContent = second } } }), + event("contentBlockStop", { contentBlockIndex = 0 }), + event("messageStop", { stopReason = "end_turn" }), + } + + local _, err, result = run(events) + + test.is_nil(err) + test.eq(#result.thinking, 1) + test.eq(result.thinking[1].type, "redacted_thinking") + test.eq(result.thinking[1].data, base64.encode("abcde")) + end) + + it("fails the stream on a redactedContent delta that is not base64", function() + local _, err, _, seen = run({ + event("messageStart", { role = "assistant" }), + event("contentBlockDelta", { contentBlockIndex = 0, delta = { reasoningContent = { redactedContent = "%%not base64%%" } } }), + event("messageStop", { stopReason = "end_turn" }), + }) + + test.contains(err, "redactedContent") + test.eq(#seen.errors, 1) + end) + it("reports input_transformations from messageStop in stream metadata", function() local events = conversation() events[6] = event("messageStop", { diff --git a/src/llm/test/bedrock/generate_test.lua b/src/llm/test/bedrock/generate_test.lua index b2f21d62..7f9e54ef 100644 --- a/src/llm/test/bedrock/generate_test.lua +++ b/src/llm/test/bedrock/generate_test.lua @@ -370,7 +370,40 @@ local function define_tests() test.eq(captured.payload.additionalModelResponseFieldPaths[1], "/input_transformations") end) + it("decides the binding from the thinking object as finally sent", function() + local captured = capture() + local real_mapper = generate_handler._mapper + local fields: any = nil + generate_handler._mapper = setmetatable({ + map_options = function(...) + local config, additional, adjusted, options_err = real_mapper.map_options(...) + fields = additional + return config, additional, adjusted, options_err + end, + map_tools = function(...) + fields.thinking.type = "between_tools" + return real_mapper.map_tools(...) + end + }, { __index = real_mapper }) + + generate_handler.handler({ + model = CLAUDE_MODEL, + messages = history_with_thinking({ SIGNED_THINKING }), + tools = { { name = "search", description = "Search", schema = { type = "object" } } }, + accepts = { thinking = "adaptive" }, + options = { thinking_effort = 50 } + }) + generate_handler._mapper = real_mapper + + local sent = captured.payload.additionalModelRequestFields + test.eq(sent.thinking.type, "between_tools") + test.is_nil(sent.thinking.block_binding) + test.is_nil(sent.anthropic_beta) + test.is_nil(captured.payload.additionalModelResponseFieldPaths) + end) + it("sends no binding when the history replays no reasoning", function() + local captured = capture() generate_handler.handler({ diff --git a/src/llm/test/bedrock/mapper_test.lua b/src/llm/test/bedrock/mapper_test.lua index 5deafa3f..ab80ed1b 100644 --- a/src/llm/test/bedrock/mapper_test.lua +++ b/src/llm/test/bedrock/mapper_test.lua @@ -2,12 +2,7 @@ local mapper = require("mapper") local output = require("output") local json = require("json") -local function same(a: any, b: any): boolean - if type(a) ~= "table" or type(b) ~= "table" then return a == b end - for k, v in pairs(a) do if not same(v, b[k]) then return false end end - for k in pairs(b) do if a[k] == nil then return false end end - return true -end +local same = require("deep_equal") local function define_tests() describe("Bedrock Converse Mapper", function() diff --git a/src/llm/test/bedrock/structured_output_test.lua b/src/llm/test/bedrock/structured_output_test.lua index 7b40e4e2..1aa9091d 100644 --- a/src/llm/test/bedrock/structured_output_test.lua +++ b/src/llm/test/bedrock/structured_output_test.lua @@ -197,7 +197,53 @@ local function define_tests() test.eq(captured_payload.additionalModelResponseFieldPaths[1], "/input_transformations") end) + it("binds replayed reasoning on an always route at effort 0", function() + local captured_payload: any = nil + structured_output_handler._client = { + converse = function(model_id, payload, options) + captured_payload = payload + return { + output = { message = { role = "assistant", content = { + { toolUse = { toolUseId = "c1", name = "structured_output", input = { x = 1 } } } + } } }, + stopReason = "tool_use", + usage = { inputTokens = 10, outputTokens = 5 }, + metadata = {} + } + end + } + + structured_output_handler.handler({ + model = "us.anthropic.claude-opus-5-5-v1:0", + messages = { + { role = "user", content = { { type = "text", text = "Plan" } } }, + { + role = "assistant", + content = { { type = "text", text = "Planned." } }, + metadata = { thinking_blocks = { { type = "thinking", thinking = "t", signature = "sig" } } } + }, + { role = "user", content = { { type = "text", text = "Answer" } } } + }, + schema = { + type = "object", + properties = { x = { type = "number" } }, + required = { "x" }, + additionalProperties = false + }, + options = { thinking_effort = 0 }, + accepts = { thinking = "always" } + }) + + local fields = captured_payload.additionalModelRequestFields + test.eq(fields.thinking.type, "adaptive") + test.is_nil(fields.output_config) + test.eq(fields.thinking.block_binding.prefix_mismatch_behavior, "drop_block") + test.eq(fields.anthropic_beta[1], "thinking-binding-controls-2026-08-01") + test.eq(captured_payload.additionalModelResponseFieldPaths[1], "/input_transformations") + end) + it("should send forced tool_choice in payload", function() + local captured_payload = nil structured_output_handler._client = { converse = function(model_id, payload, options) diff --git a/src/llm/test/claude/generate_test.lua b/src/llm/test/claude/generate_test.lua index abc88ef7..36662339 100644 --- a/src/llm/test/claude/generate_test.lua +++ b/src/llm/test/claude/generate_test.lua @@ -1590,7 +1590,39 @@ local function define_tests() test.eq(captured.options.beta_features[1], BINDING_BETA) end) + it("decides the binding from the thinking object as finally sent", function() + local captured = {} + local real_mapper = generate_handler._mapper + local mapped_options: any = nil + generate_handler._mapper = setmetatable({ + map_options = function(...) + local options, adjusted, options_err = real_mapper.map_options(...) + mapped_options = options + return options, adjusted, options_err + end, + map_tools = function(...) + mapped_options.thinking.type = "between_tools" + return real_mapper.map_tools(...) + end + }, { __index = real_mapper }) + generate_handler._client = capturing_client(captured) + + generate_handler.handler({ + model = "claude-sonnet-5-5", + messages = history_with_thinking({ SIGNED_THINKING }), + tools = { { name = "search", description = "Search", schema = { type = "object" } } }, + accepts = { thinking = "adaptive" }, + options = { thinking_effort = 50 } + }) + generate_handler._mapper = real_mapper + + test.eq(captured.payload.thinking.type, "between_tools") + test.is_nil(captured.payload.thinking.block_binding) + test.is_nil(captured.options.beta_features) + end) + it("sends no binding when the history replays no thinking blocks", function() + local captured = {} generate_handler._client = capturing_client(captured) @@ -1694,7 +1726,29 @@ local function define_tests() test.is_nil(blocks[2].signature) end) + it("passes fields other than the stale ones through on replay", function() + local captured = {} + generate_handler._client = capturing_client(captured) + + generate_handler.handler({ + model = "claude-opus-5-5", + messages = history_with_thinking({ + { type = "thinking", thinking = "Weighing the routes.", signature = "sig-1", cache_control = { type = "ephemeral" } }, + { type = "redacted_thinking", data = "opaque", provenance = "kept" } + }), + accepts = { thinking = "adaptive" }, + options = { thinking_effort = 50 } + }) + + local content = captured.payload.messages[2].content + test.eq(content[1].cache_control.type, "ephemeral") + test.eq(content[1].signature, "sig-1") + test.eq(content[2].provenance, "kept") + test.eq(content[2].data, "opaque") + end) + it("replays stored blocks in the API shape of their type", function() + local captured = {} generate_handler._client = capturing_client(captured) diff --git a/src/llm/test/claude/mapper_test.lua b/src/llm/test/claude/mapper_test.lua index b30c48e9..28c83160 100644 --- a/src/llm/test/claude/mapper_test.lua +++ b/src/llm/test/claude/mapper_test.lua @@ -5,12 +5,7 @@ local json = require("json") local hash = require("hash") local test = require("test") -local function same(a: any, b: any): boolean - if type(a) ~= "table" or type(b) ~= "table" then return a == b end - for k, v in pairs(a) do if not same(v, b[k]) then return false end end - for k in pairs(b) do if a[k] == nil then return false end end - return true -end +local same = require("deep_equal") local function define_tests() describe("Claude Mapper", function() diff --git a/src/llm/test/support/deep_equal.lua b/src/llm/test/support/deep_equal.lua new file mode 100644 index 00000000..e48d384a --- /dev/null +++ b/src/llm/test/support/deep_equal.lua @@ -0,0 +1,19 @@ +-- Structural equality of two values: tables compare key by key, recursively. +local function deep_equal(a: any, b: any): boolean + if type(a) ~= "table" or type(b) ~= "table" then + return a == b + end + for key, value in pairs(a) do + if not deep_equal(value, b[key]) then + return false + end + end + for key in pairs(b) do + if a[key] == nil then + return false + end + end + return true +end + +return deep_equal diff --git a/src/llm/test/util/thinking_test.lua b/src/llm/test/util/thinking_test.lua new file mode 100644 index 00000000..f7e868d8 --- /dev/null +++ b/src/llm/test/util/thinking_test.lua @@ -0,0 +1,42 @@ +local test = require("test") +local thinking = require("thinking") + +local function define_tests() + describe("Thinking binding helpers", function() + it("binds adaptive and enabled thinking objects with drop_block", function() + for _, config in ipairs({ { type = "adaptive" }, { type = "enabled", budget_tokens = 2048 } }) do + thinking.bind_replayed_blocks(config) + test.eq(config.block_binding.prefix_mismatch_behavior, "drop_block") + test.is_true(thinking.is_bound(config)) + end + end) + + it("leaves thinking objects that take no binding unchanged", function() + for _, config in ipairs({ { type = "between_tools" }, { type = "disabled" } }) do + thinking.bind_replayed_blocks(config) + test.is_nil(config.block_binding) + test.is_false(thinking.is_bound(config)) + end + end) + + it("accepts a missing or non-table thinking object", function() + thinking.bind_replayed_blocks(nil) + thinking.bind_replayed_blocks("adaptive" :: any) + test.is_false(thinking.is_bound(nil)) + test.is_false(thinking.is_bound("adaptive" :: any)) + end) + + it("keeps a block_binding the thinking object already carries", function() + local config = { type = "adaptive", block_binding = { prefix_mismatch_behavior = "error" } } + thinking.bind_replayed_blocks(config) + test.eq(config.block_binding.prefix_mismatch_behavior, "error") + test.is_true(thinking.is_bound(config)) + end) + + it("names the binding beta", function() + test.eq(thinking.BINDING_BETA, "thinking-binding-controls-2026-08-01") + end) + end) +end + +return require("test").run_cases(define_tests) From d8a5760902ceeaf1a8b1d87f4d9dab0937f9f592 Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 16:12:31 -0500 Subject: [PATCH 05/37] feat(llm): render positional notes in place and bind thinking blocks as a leading run Positional system/developer notes (metadata.preserve_position) render as text blocks after the preceding user turn on Claude and Bedrock, as user turns on Google, and stay in place on OpenAI and OpenAI-compatible; they never enter the top-level system prompt. A positional note without a preceding user turn is invalid_request. Bedrock developer messages render as user text instead of system blocks. The Google mapper no longer clears caller message metadata. builder:bind_thinking strips thinking blocks from a leading run of non-current messages and rejects middle removal and open tool rounds. Golden tests pin message-derived bodies on all five drivers. --- src/llm/README.md | 10 +- src/llm/src/README.md | 30 +++++ src/llm/src/bedrock/generate.lua | 5 +- src/llm/src/bedrock/mapper.lua | 72 ++++++------ src/llm/src/bedrock/structured_output.lua | 5 +- src/llm/src/claude/generate.lua | 5 +- src/llm/src/claude/mapper.lua | 19 +++- src/llm/src/claude/structured_output.lua | 5 +- src/llm/src/google/_index.yaml | 1 + src/llm/src/google/mapper.lua | 7 +- src/llm/src/openai/_index.yaml | 1 + src/llm/src/openai/mapper.lua | 4 +- src/llm/src/prompt.lua | 56 ++++++++++ src/llm/test/_index.yaml | 22 ++++ src/llm/test/bedrock/generate_test.lua | 21 ++++ src/llm/test/bedrock/mapper_test.lua | 122 ++++++++++++++++++++- src/llm/test/claude/generate_test.lua | 21 ++++ src/llm/test/claude/mapper_test.lua | 116 ++++++++++++++++++-- src/llm/test/google/mapper_test.lua | 43 ++++++++ src/llm/test/openai/mapper_test.lua | 16 +++ src/llm/test/openai_compat/mapper_test.lua | 18 +++ src/llm/test/prompt_test.lua | 110 +++++++++++++++++++ src/llm/test/request_golden_test.lua | 79 +++++++++++++ 23 files changed, 728 insertions(+), 60 deletions(-) create mode 100644 src/llm/test/request_golden_test.lua diff --git a/src/llm/README.md b/src/llm/README.md index dc03fff0..3499750b 100644 --- a/src/llm/README.md +++ b/src/llm/README.md @@ -60,8 +60,8 @@ See [LICENSE](LICENSE) for details. [wippy-documentation]: https://docs.wippy.ai [wippy-framework]: https://github.com/wippyai/framework -Message metadata can set `preserve_position = true` to retain a message at its -conversation position. Prompt consolidation keeps it separate from adjacent -messages of the same role. OpenAI Responses sends positional system and developer -messages in `input` alongside the conversation; ordinary system and developer -messages retain their existing top-level `instructions` behavior. +Message metadata can set `preserve_position = true` to retain a system or +developer message at its conversation position. Prompt consolidation keeps it +separate from adjacent messages of the same role, and no driver moves it into +the top-level system prompt or instructions. See the Positional Notes section of +[src/README.md](src/README.md). diff --git a/src/llm/src/README.md b/src/llm/src/README.md index ebea2b52..1e6ef39d 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -210,6 +210,36 @@ builder:add_cache_marker("system_cache") builder:add_user("Question") ``` +### Positional Notes + +A system or developer message with `metadata.preserve_position = true` is a positional note. It stays at its conversation position on every driver and is never merged with neighbouring messages: + +- Claude and Bedrock append it as text blocks after the content of the user turn it follows (including a turn of tool results). The top-level system prompt never contains it. A positional note that does not follow a user turn fails with `invalid_request` before dispatch. +- Google sends it as a user turn; `systemInstruction` holds only ordinary system messages. +- OpenAI Responses keeps it in `input`; OpenAI-compatible keeps it as a `system` message in place. + +Per-turn context (current time, recalled memory, lifecycle notes) belongs in positional notes written once and kept in the history. Changing the top-level system prompt, the tools or any earlier message changes the provider prefix, which restarts the prompt cache and invalidates every later thinking block. + +```lua +builder:add_user("What is on my calendar?") +builder:add_system("The current date and time is Tuesday, November 3, 2026 at 09:41 UTC.", + { preserve_position = true }) +``` + +Bedrock renders ordinary developer messages as user text at their position, like Claude. + +### Thinking Blocks + +Assistant and function call messages carry provider reasoning in `metadata.thinking_blocks`. Each block is bound to the prefix before it and chained to the previous block, so blocks can be dropped only from the front of the history. `bind_thinking` removes the blocks of every message the predicate reports as not current and keeps the rest: + +```lua +local bound, err = builder:bind_thinking(function(msg) + return msg.metadata.turn_id == current_turn_id +end) +``` + +It fails, leaving the history unchanged, when a non-current message follows a current one, or when a non-current function call has no result in the history yet. Messages without thinking blocks are not consulted. + ### Builder Operations ```lua diff --git a/src/llm/src/bedrock/generate.lua b/src/llm/src/bedrock/generate.lua index a706f30d..db41b818 100644 --- a/src/llm/src/bedrock/generate.lua +++ b/src/llm/src/bedrock/generate.lua @@ -98,7 +98,10 @@ function generate_handler.handler(contract_args) name_to_id_map = {} } - local mapped = generate_handler._mapper.map_messages(contract_args.messages) + local mapped, map_err = generate_handler._mapper.map_messages(contract_args.messages) + if not mapped then + return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(tostring(map_err)):build() + end local inference_config, additional_fields, adjusted, options_err = generate_handler._mapper.map_options(contract_args.options or {}, contract_args.accepts) if options_err then return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(options_err):build() diff --git a/src/llm/src/bedrock/mapper.lua b/src/llm/src/bedrock/mapper.lua index 486c3dfc..dcf7330d 100644 --- a/src/llm/src/bedrock/mapper.lua +++ b/src/llm/src/bedrock/mapper.lua @@ -140,6 +140,30 @@ local function convert_document_to_converse(content_part) return nil end +local function user_content_blocks(content) + local content_blocks = {} + if type(content) == "string" then + table.insert(content_blocks, { text = content }) + elseif type(content) == "table" then + for _, part in ipairs(content) do + if part.type == "text" and part.text then + table.insert(content_blocks, { text = part.text }) + elseif part.type == "image" then + local img = convert_image_to_converse(part) + if img then + table.insert(content_blocks, img) + end + elseif part.type == "document" then + local doc = convert_document_to_converse(part) + if doc then + table.insert(content_blocks, doc) + end + end + end + end + return content_blocks +end + local function normalize_tool_arguments(raw_arguments) local arguments = raw_arguments if type(arguments) == "string" then @@ -213,8 +237,19 @@ function mapper.map_messages(contract_messages) local system_cache_positions = {} local message_cache_positions = {} - for _, msg in ipairs(contract_messages) do - if msg.role == prompt.ROLE.SYSTEM then + for i, msg in ipairs(contract_messages) do + if prompt.is_positional_note(msg) then + -- A positional note joins the user turn it follows, as blocks after + -- that turn's content, so the system blocks and every earlier + -- block stay byte-identical when the note changes. + local last_msg = converse_messages[#converse_messages] + if not last_msg or last_msg.role ~= "user" then + return nil, string.format("positional %s note at message %d must follow a user message", msg.role, i) + end + for _, block in ipairs(user_content_blocks(msg.content)) do + table.insert(last_msg.content, block) + end + elseif msg.role == prompt.ROLE.SYSTEM then if type(msg.content) == "string" then table.insert(system_blocks, { text = msg.content }) elseif type(msg.content) == "table" then @@ -238,13 +273,6 @@ function mapper.map_messages(contract_messages) message_cache_positions[#message_cache_positions + 1] = position end end - elseif msg.role == prompt.ROLE.DEVELOPER then - local dev_text = type(msg.content) == "string" and msg.content or - (type(msg.content) == "table" and msg.content[1] and msg.content[1].text) or "" - - if dev_text ~= "" then - table.insert(system_blocks, { text = dev_text }) - end elseif msg.role == prompt.ROLE.FUNCTION_RESULT then local result_text = type(msg.content) == "string" and msg.content or (type(msg.content) == "table" and msg.content[1] and msg.content[1].text) or "" @@ -349,29 +377,9 @@ function mapper.map_messages(contract_messages) }) end else - -- User messages - local content_blocks = {} - local content = msg.content - if type(content) == "string" then - table.insert(content_blocks, { text = content }) - elseif type(content) == "table" then - for _, part in ipairs(content) do - if part.type == "text" and part.text then - table.insert(content_blocks, { text = part.text }) - elseif part.type == "image" then - local img = convert_image_to_converse(part) - if img then - table.insert(content_blocks, img) - end - elseif part.type == "document" then - local doc = convert_document_to_converse(part) - if doc then - table.insert(content_blocks, doc) - end - end - end - end - + -- User messages and developer guidance, which renders as user text + -- so per-step guidance never rewrites the system blocks. + local content_blocks = user_content_blocks(msg.content) if #content_blocks > 0 then table.insert(converse_messages, { role = "user", diff --git a/src/llm/src/bedrock/structured_output.lua b/src/llm/src/bedrock/structured_output.lua index 7a5d3d0e..b00b9819 100644 --- a/src/llm/src/bedrock/structured_output.lua +++ b/src/llm/src/bedrock/structured_output.lua @@ -91,7 +91,10 @@ function structured_output_handler.handler(contract_args) return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(forced_err):build() end - local mapped = structured_output_handler._mapper.map_messages(contract_args.messages) + local mapped, map_err = structured_output_handler._mapper.map_messages(contract_args.messages) + if not mapped then + return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(tostring(map_err)):build() + end local inference_config, additional_fields, adjusted, options_err = structured_output_handler._mapper.map_options(contract_args.options or {}, contract_args.accepts) if options_err then return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(options_err):build() diff --git a/src/llm/src/claude/generate.lua b/src/llm/src/claude/generate.lua index 840d5377..3c1aeaa6 100644 --- a/src/llm/src/claude/generate.lua +++ b/src/llm/src/claude/generate.lua @@ -100,7 +100,10 @@ function generate_handler.handler(contract_args) name_to_id_map = {} } - local mapped_messages = generate_handler._mapper.map_messages(contract_args.messages) + local mapped_messages, map_err = generate_handler._mapper.map_messages(contract_args.messages) + if not mapped_messages then + return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(tostring(map_err)):build() + end local mapped_options, adjusted, options_err = generate_handler._mapper.map_options(contract_args.options or {}, contract_args.accepts) if options_err then return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(options_err):build() diff --git a/src/llm/src/claude/mapper.lua b/src/llm/src/claude/mapper.lua index 7e9a70a0..31ab69ee 100644 --- a/src/llm/src/claude/mapper.lua +++ b/src/llm/src/claude/mapper.lua @@ -350,7 +350,24 @@ function mapper.map_messages(contract_messages) local in_system_phase = true for i, msg in ipairs(contract_messages) do - if msg.role == prompt.ROLE.SYSTEM then + if prompt.is_positional_note(msg) then + -- A positional note joins the user turn it follows, as blocks after + -- that turn's content, so the system prompt and every earlier + -- block stay byte-identical when the note changes. + in_system_phase = false + local last_msg = claude_messages[#claude_messages] + if not last_msg or last_msg.role ~= "user" then + return nil, string.format("positional %s note at message %d must follow a user message", msg.role, i) + end + local note_content: any = process_content_array(msg.content) + if type(note_content) == "string" then + table.insert(last_msg.content, { type = "text", text = note_content }) + elseif type(note_content) == "table" then + for _, part in ipairs(note_content) do + table.insert(last_msg.content, part) + end + end + elseif msg.role == prompt.ROLE.SYSTEM then if type(msg.content) == "string" then table.insert(system_blocks, { type = "text", diff --git a/src/llm/src/claude/structured_output.lua b/src/llm/src/claude/structured_output.lua index 353ab5c2..80d02108 100644 --- a/src/llm/src/claude/structured_output.lua +++ b/src/llm/src/claude/structured_output.lua @@ -173,7 +173,10 @@ function structured_output_handler.handler(contract_args) end end - local mapped_messages = structured_output_handler._mapper.map_messages(contract_args.messages) + local mapped_messages, map_err = structured_output_handler._mapper.map_messages(contract_args.messages) + if not mapped_messages then + return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(tostring(map_err)):build() + end local mapped_options, adjusted, options_err = structured_output_handler._mapper.map_options(contract_args.options or {}, contract_args.accepts) if options_err then return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(options_err):build() diff --git a/src/llm/src/google/_index.yaml b/src/llm/src/google/_index.yaml index 2470573b..50a3de85 100644 --- a/src/llm/src/google/_index.yaml +++ b/src/llm/src/google/_index.yaml @@ -165,6 +165,7 @@ entries: imports: output: wippy.llm:output route: wippy.llm.util:route + prompt: wippy.llm:prompt # wippy.llm.google:status - name: status diff --git a/src/llm/src/google/mapper.lua b/src/llm/src/google/mapper.lua index c3db8fa9..4677b346 100644 --- a/src/llm/src/google/mapper.lua +++ b/src/llm/src/google/mapper.lua @@ -2,6 +2,7 @@ local json = require("json") local output = require("output") local uuid = require("uuid") local route = require("route") +local prompt = require("prompt") local mapper = {} @@ -166,15 +167,15 @@ function mapper.map_messages(contract_messages, options) while i <= #contract_messages do local msg = contract_messages[i] - msg.metadata = nil + local positional = prompt.is_positional_note(msg) if msg.role ~= "function_call" and msg.role ~= "function_result" and msg.role ~= "cache_marker" then flush_tool_round() end - if msg.role == "system" then + if msg.role == "system" and not positional then table.insert(system_instructions, { text = mapper.standardize_content(msg.content) }) i = i + 1 - elseif msg.role == "user" or msg.role == "developer" then + elseif msg.role == "user" or msg.role == "developer" or positional then table.insert(processed_messages, { role = "user", parts = normalize_content(msg.content) }) i = i + 1 elseif msg.role == "assistant" then diff --git a/src/llm/src/openai/_index.yaml b/src/llm/src/openai/_index.yaml index fbb417b7..f23a4a9b 100644 --- a/src/llm/src/openai/_index.yaml +++ b/src/llm/src/openai/_index.yaml @@ -156,6 +156,7 @@ entries: imports: output: wippy.llm:output route: wippy.llm.util:route + prompt: wippy.llm:prompt # wippy.llm.openai:organization - name: organization diff --git a/src/llm/src/openai/mapper.lua b/src/llm/src/openai/mapper.lua index de8e4c2e..27da4494 100644 --- a/src/llm/src/openai/mapper.lua +++ b/src/llm/src/openai/mapper.lua @@ -1,6 +1,7 @@ local json = require("json") local output = require("output") local route = require("route") +local prompt = require("prompt") local openai_mapper = {} @@ -177,8 +178,7 @@ function openai_mapper.extract_instructions(contract_messages) local parts = {} local consumed = {} for i, msg in ipairs(contract_messages) do - if (msg.role == "system" or msg.role == "developer") - and not (msg.metadata and msg.metadata.preserve_position == true) then + if (msg.role == "system" or msg.role == "developer") and not prompt.is_positional_note(msg) then local text = extract_text(msg.content) if text and text ~= "" then table.insert(parts, text) diff --git a/src/llm/src/prompt.lua b/src/llm/src/prompt.lua index 9ab61b7d..33308fb6 100644 --- a/src/llm/src/prompt.lua +++ b/src/llm/src/prompt.lua @@ -52,6 +52,7 @@ type PromptBuilder = { add_function_call: (self: any, function_name: string, arguments: string, function_call_id: string?, options: FunctionCallOptions?) -> PromptBuilder, add_function_result: (self: any, name: string, content: any, function_call_id: string?, options: FunctionResultOptions?) -> PromptBuilder, add_cache_marker: (self: any, marker_id: string?) -> PromptBuilder, + bind_thinking: (self: any, is_current: (msg: Message) -> boolean) -> (PromptBuilder?, string?), get_messages: (self: any) -> {Message}, build: (self: any) -> {messages: {Message}}, clone: (self: any) -> PromptBuilder, @@ -132,6 +133,14 @@ function prompt.document_base64(mime_type: string, data: string): ContentPart } end +-- A positional note is a system or developer message marked with +-- metadata.preserve_position. Drivers render it at its conversation position +-- instead of folding it into the top-level instructions. +function prompt.is_positional_note(msg: Message): boolean + return (msg.role == prompt.ROLE.SYSTEM or msg.role == prompt.ROLE.DEVELOPER) + and type(msg.metadata) == "table" and msg.metadata.preserve_position == true +end + --------------------------- -- Helper Functions --------------------------- @@ -363,6 +372,53 @@ function prompt.new(messages: {Message}?) return self end + -- Remove thinking blocks the current request can no longer replay. + -- Providers bind each block to the exact prefix before it and chain it to + -- the previous block, so only a leading run of blocks may be dropped, and + -- an assistant turn whose tool calls still await results keeps its blocks. + builder.bind_thinking = function(self: any, is_current: (msg: Message) -> boolean): (PromptBuilder?, string?) + local messages: {Message} = self.messages :: {Message} + local resolved: {[string]: boolean} = {} + for _, msg in ipairs(messages) do + if msg.role == prompt.ROLE.FUNCTION_RESULT and msg.function_call_id then + resolved[msg.function_call_id] = true + end + end + + local stale: {Message} = {} + local kept_index: integer? = nil + for index, msg in ipairs(messages) do + local carries = (msg.role == prompt.ROLE.ASSISTANT or msg.role == prompt.ROLE.FUNCTION_CALL) + and msg.metadata ~= nil and msg.metadata.thinking_blocks ~= nil + if carries then + if is_current(msg) then + kept_index = kept_index or index + elseif kept_index then + return nil, string.format( + "thinking blocks can be removed only as a leading run: message %d is not current but follows current message %d", + index, kept_index) + else + local call_id = msg.function_call and msg.function_call.id + if msg.role == prompt.ROLE.FUNCTION_CALL and not (call_id and resolved[call_id]) then + return nil, string.format( + "thinking blocks of open tool call %s must stay until its result is present", + tostring(call_id)) + end + table.insert(stale, msg) + end + end + end + + for _, msg in ipairs(stale) do + local metadata = {} + for key, value in pairs(msg.metadata :: table) do + if key ~= "thinking_blocks" then metadata[key] = value end + end + msg.metadata = metadata + end + return self, nil + end + -- Get all messages in the current builder (with image processing) builder.get_messages = function(self: any): {Message} local processed_messages: {Message} = {} diff --git a/src/llm/test/_index.yaml b/src/llm/test/_index.yaml index bbb5b508..54d9b3d7 100644 --- a/src/llm/test/_index.yaml +++ b/src/llm/test/_index.yaml @@ -280,6 +280,28 @@ entries: test: wippy.test:test method: run_tests + - name: request_golden_test + kind: function.lua + meta: + name: Request Body Golden Test + type: test + comment: Pins message-derived request body sections on every driver + group: LLM / General + tags: + - llm + - mapper + - cache + source: file://request_golden_test.lua + imports: + claude_mapper: wippy.llm.claude:mapper + bedrock_mapper: wippy.llm.bedrock:mapper + google_mapper: wippy.llm.google:mapper + openai_mapper: wippy.llm.openai:mapper + compat_mapper: wippy.llm.openai_compat:mapper + codec: wippy.llm:context_attention_codec_v2 + test: wippy.test:test + method: run_tests + # app:prompt_test - name: prompt_test kind: function.lua diff --git a/src/llm/test/bedrock/generate_test.lua b/src/llm/test/bedrock/generate_test.lua index a6857e97..2b5bd3fe 100644 --- a/src/llm/test/bedrock/generate_test.lua +++ b/src/llm/test/bedrock/generate_test.lua @@ -9,6 +9,27 @@ local function define_tests() end) describe("Contract Validation", function() + it("should reject a positional note without a preceding user message before dispatch", function() + generate_handler._client = { + ENDPOINTS = { MESSAGES = "/v1/messages" }, + request = function() error("request must not be dispatched") end, + converse = function() error("request must not be dispatched") end, + } + local response, err = generate_handler.handler({ + model = "us.anthropic.claude-haiku-4-5-20251001-v1:0", + messages = { + { role = "user", content = { { type = "text", text = "Question" } } }, + { role = "assistant", content = { { type = "text", text = "Answer" } } }, + { role = "system", content = { { type = "text", text = "Note" } }, + metadata = { preserve_position = true } }, + } + }) + test.is_nil(response) + test.not_nil(err) + test.eq(err:kind(), "Invalid") + test.contains(err:message(), "positional") + end) + it("should require model parameter", function() local response, err = generate_handler.handler({ messages = { { role = "user", content = { { type = "text", text = "Test" } } } } diff --git a/src/llm/test/bedrock/mapper_test.lua b/src/llm/test/bedrock/mapper_test.lua index a3769683..02896f43 100644 --- a/src/llm/test/bedrock/mapper_test.lua +++ b/src/llm/test/bedrock/mapper_test.lua @@ -5,6 +5,119 @@ local json = require("json") local function define_tests() describe("Bedrock Converse Mapper", function() + describe("Positional notes", function() + local function text(value) + return { { type = "text", text = value } } + end + + local function note(role, value) + return { role = role, content = text(value), metadata = { preserve_position = true } } + end + + it("renders a positional note as a text block after the preceding user content", function() + for _, role in ipairs({ "system", "developer" }) do + local mapped, err = mapper.map_messages({ + { role = "system", content = text("Base") }, + { role = "user", content = text("Question") }, + note(role, "The current date and time is Tuesday."), + }) + test.is_nil(err) + assert(mapped) + test.eq(#mapped.system, 1) + test.eq(mapped.system[1].text, "Base") + test.eq(#mapped.messages, 1) + local content = mapped.messages[1].content + test.eq(#content, 2) + test.eq(content[1].text, "Question") + test.eq(content[2].text, "The current date and time is Tuesday.") + end + end) + + it("appends a positional note after the tool results of the same user turn", function() + local mapped, err = mapper.map_messages({ + { role = "user", content = text("Question") }, + { role = "function_call", content = {}, function_call = { id = "call_1", name = "lookup", arguments = "{}" } }, + { role = "function_result", name = "lookup", function_call_id = "call_1", content = text("found") }, + note("system", "Reminder"), + }) + test.is_nil(err) + assert(mapped) + test.is_nil(mapped.system) + test.eq(#mapped.messages, 3) + local content = mapped.messages[3].content + test.eq(#content, 2) + test.not_nil(content[1].toolResult) + test.eq(content[2].text, "Reminder") + end) + + it("keeps the system blocks identical when the note text changes", function() + local function build(note_text) + return assert(mapper.map_messages({ + { role = "system", content = text("Base") }, + { role = "cache_marker", marker_id = "agent" }, + { role = "user", content = text("Question") }, + note("system", note_text), + })) + end + local first, second = build("Monday 09:00"), build("Tuesday 10:30") + test.eq(#first.system, 2) + test.eq(#second.system, 2) + test.eq(second.system[1].text, first.system[1].text) + test.eq(second.system[2].cachePoint.type, first.system[2].cachePoint.type) + test.eq(first.messages[1].content[2].text, "Monday 09:00") + test.eq(second.messages[1].content[2].text, "Tuesday 10:30") + end) + + it("keeps a cache point on the user block before the note", function() + local mapped = assert(mapper.map_messages({ + { role = "system", content = text("Base") }, + { role = "user", content = text("Question") }, + { role = "cache_marker", marker_id = "conversation_tail" }, + note("developer", "Recalled memory"), + })) + local content = mapped.messages[1].content + test.eq(#content, 3) + test.eq(content[1].text, "Question") + test.eq(content[2].cachePoint.type, "default") + test.eq(content[3].text, "Recalled memory") + end) + + it("rejects a positional note that has no preceding user message", function() + for _, history in ipairs({ + { { role = "system", content = text("Base") }, note("system", "Orphan") }, + { + { role = "user", content = text("Question") }, + { role = "assistant", content = text("Answer") }, + note("developer", "After the answer"), + }, + }) do + local mapped, err = mapper.map_messages(history) + test.is_nil(mapped) + test.contains(tostring(err), "positional") + end + end) + + it("renders developer guidance as user text instead of system blocks", function() + local function build(guidance) + return assert(mapper.map_messages({ + { role = "system", content = text("Base") }, + { role = "user", content = text("Question") }, + { role = "developer", content = text(guidance) }, + })) + end + local first, second = build("Step one guidance"), build("Step two guidance") + for _, mapped in ipairs({ first, second }) do + test.eq(#mapped.system, 1) + test.eq(mapped.system[1].text, "Base") + test.eq(#mapped.messages, 1) + test.eq(mapped.messages[1].role, "user") + test.eq(mapped.messages[1].content[1].text, "Question") + end + test.eq(first.messages[1].content[2].text, "Step one guidance") + test.eq(second.messages[1].content[2].text, "Step two guidance") + end) + end) + describe("Payment and quota classification", function() it("classifies HTTP 402 as non-retryable payment_required", function() for _, message in ipairs({ "Payment required", "Network quota unavailable", "Maximum allowed tokens" }) do @@ -296,7 +409,7 @@ local function define_tests() test.eq((result.messages[1].content[1] :: any).text, "Plain text") end) - it("should map developer messages to system blocks", function() + it("should map developer messages to user text at their position", function() local result = mapper.map_messages({ { role = "system", content = "System prompt" }, { role = "developer", content = "Developer instruction" }, @@ -304,9 +417,12 @@ local function define_tests() }) test.not_nil(result.system) - test.eq(#result.system, 2) + test.eq(#result.system, 1) test.eq((result.system[1] :: any).text, "System prompt") - test.eq((result.system[2] :: any).text, "Developer instruction") + test.eq(#result.messages, 1) + test.eq(result.messages[1].role, "user") + test.eq(result.messages[1].content[1].text, "Developer instruction") + test.eq(result.messages[1].content[2].text, "Hello") end) it("should preserve thinking blocks in function_call messages", function() diff --git a/src/llm/test/claude/generate_test.lua b/src/llm/test/claude/generate_test.lua index d68f84d7..0b6bd079 100644 --- a/src/llm/test/claude/generate_test.lua +++ b/src/llm/test/claude/generate_test.lua @@ -10,6 +10,27 @@ local function define_tests() end) describe("Contract Validation", function() + it("should reject a positional note without a preceding user message before dispatch", function() + generate_handler._client = { + ENDPOINTS = { MESSAGES = "/v1/messages" }, + request = function() error("request must not be dispatched") end, + converse = function() error("request must not be dispatched") end, + } + local response, err = generate_handler.handler({ + model = "claude-3-5-sonnet-20241022", + messages = { + { role = "user", content = { { type = "text", text = "Question" } } }, + { role = "assistant", content = { { type = "text", text = "Answer" } } }, + { role = "system", content = { { type = "text", text = "Note" } }, + metadata = { preserve_position = true } }, + } + }) + test.is_nil(response) + test.not_nil(err) + test.eq(err:kind(), "Invalid") + test.contains(err:message(), "positional") + end) + it("should require model parameter", function() local response, err = generate_handler.handler({ messages = { { role = "user", content = { { type = "text", text = "Test" } } } } diff --git a/src/llm/test/claude/mapper_test.lua b/src/llm/test/claude/mapper_test.lua index 6d770680..091b3a02 100644 --- a/src/llm/test/claude/mapper_test.lua +++ b/src/llm/test/claude/mapper_test.lua @@ -8,6 +8,102 @@ local test = require("test") local function define_tests() describe("Claude Mapper", function() + describe("Positional notes", function() + local function text(value) + return { { type = "text", text = value } } + end + + local function note(role, value) + return { role = role, content = text(value), metadata = { preserve_position = true } } + end + + it("renders a positional note as a text block after the preceding user content", function() + for _, role in ipairs({ "system", "developer" }) do + local mapped, err = mapper.map_messages({ + { role = "system", content = text("Base") }, + { role = "user", content = text("Question") }, + note(role, "The current date and time is Tuesday."), + }) + test.is_nil(err) + assert(mapped) + test.eq(#mapped.system, 1) + test.eq(mapped.system[1].text, "Base") + test.eq(#mapped.messages, 1) + local content = mapped.messages[1].content + test.eq(#content, 2) + test.eq(content[1].text, "Question") + test.eq(content[2].type, "text") + test.eq(content[2].text, "The current date and time is Tuesday.") + end + end) + + it("appends a positional note after the tool results of the same user turn", function() + local mapped, err = mapper.map_messages({ + { role = "user", content = text("Question") }, + { role = "function_call", content = {}, function_call = { id = "call_1", name = "lookup", arguments = "{}" } }, + { role = "function_result", name = "lookup", function_call_id = "call_1", content = text("found") }, + note("system", "Reminder"), + }) + test.is_nil(err) + assert(mapped) + test.is_nil(mapped.system) + test.eq(#mapped.messages, 3) + local content = mapped.messages[3].content + test.eq(#content, 2) + test.eq(content[1].type, "tool_result") + test.eq(content[2].type, "text") + test.eq(content[2].text, "Reminder") + end) + + it("keeps the top-level system identical when the note text changes", function() + local function build(note_text) + local mapped = mapper.map_messages({ + { role = "system", content = text("Base") }, + { role = "cache_marker", marker_id = "agent" }, + { role = "user", content = text("Question") }, + note("system", note_text), + }) + return assert(mapped) + end + local first, second = build("Monday 09:00"), build("Tuesday 10:30") + test.eq(#first.system, 1) + test.eq(#second.system, 1) + test.eq(second.system[1].text, first.system[1].text) + test.eq(second.system[1].type, first.system[1].type) + test.eq(second.system[1].cache_control.type, first.system[1].cache_control.type) + test.eq(first.messages[1].content[2].text, "Monday 09:00") + test.eq(second.messages[1].content[2].text, "Tuesday 10:30") + end) + + it("keeps a cache breakpoint on the user block before the note", function() + local mapped = assert(mapper.map_messages({ + { role = "system", content = text("Base") }, + { role = "user", content = text("Question") }, + { role = "cache_marker", marker_id = "conversation_tail" }, + note("developer", "Recalled memory"), + })) + local content = mapped.messages[1].content + test.eq(content[1].cache_control.type, "ephemeral") + test.eq(content[2].text, "Recalled memory") + test.is_nil(content[2].cache_control) + end) + + it("rejects a positional note that has no preceding user message", function() + for _, history in ipairs({ + { { role = "system", content = text("Base") }, note("system", "Orphan") }, + { + { role = "user", content = text("Question") }, + { role = "assistant", content = text("Answer") }, + note("developer", "After the answer"), + }, + }) do + local mapped, err = mapper.map_messages(history) + test.is_nil(mapped) + test.contains(tostring(err), "positional") + end + end) + end) + describe("Payment and quota classification", function() it("classifies HTTP 402 as non-retryable payment_required", function() for _, message in ipairs({ "Payment required", "Network quota unavailable", "Maximum allowed tokens" }) do @@ -74,14 +170,14 @@ local function define_tests() end) end) it("preserves distinct valid call IDs and their result pairings", function() - local mapped = mapper.map_messages({ + local mapped = assert(mapper.map_messages({ { role = "assistant", content = { { type = "function_call", id = "call_one", name = "lookup", arguments = "{}" }, { type = "function_call", id = "call-one", name = "lookup", arguments = "{}" }, } }, { role = "function_result", function_call_id = "call_one", content = "one" }, { role = "function_result", function_call_id = "call-one", content = "two" }, - }) + })) test.eq(mapped.messages[1].content[1].id, "call_one") test.eq(mapped.messages[1].content[2].id, "call-one") test.eq(mapped.messages[2].content[1].tool_use_id, "call_one") @@ -99,8 +195,8 @@ local function define_tests() { role = "function_result", function_call_id = foreign, content = "foreign" }, { role = "function_result", function_call_id = native, content = "native" }, } - local first = mapper.map_messages(history) - local second = mapper.map_messages(history) + local first = assert(mapper.map_messages(history)) + local second = assert(mapper.map_messages(history)) local translated = first.messages[1].content[1].id test.is_true(translated:match("^[a-zA-Z0-9_-]+$") ~= nil) test.is_false(translated == native) @@ -113,8 +209,8 @@ local function define_tests() it("marks only explicit tool failures, preserving unflagged outputs", function() for _, failed in ipairs({ false, true }) do - local mapped = mapper.map_messages({{ role = "function_result", name = "lookup", - function_call_id = "call-1", content = "denied", is_error = failed }}) + local mapped = assert(mapper.map_messages({{ role = "function_result", name = "lookup", + function_call_id = "call-1", content = "denied", is_error = failed }})) local result = mapped.messages[1].content[1] test.eq(result.tool_use_id, "call-1") test.eq(result.content, "denied") @@ -507,15 +603,15 @@ local function define_tests() end) it("should keep the marked prefix stable when a dynamic developer note follows", function() - local first = mapper.map_messages({ + local first = assert(mapper.map_messages({ { role = "user", content = {{ type = "text", text = "Question" }} }, { role = "cache_marker", marker_id = "history_tail" } - }) - local second = mapper.map_messages({ + })) + local second = assert(mapper.map_messages({ { role = "user", content = {{ type = "text", text = "Question" }} }, { role = "cache_marker", marker_id = "history_tail" }, { role = "developer", content = "Changing memory recall" } - }) + })) test.eq(second.messages[1].content[1].text, first.messages[1].content[1].text) test.not_nil((second.messages[1].content[1] :: any).cache_control) diff --git a/src/llm/test/google/mapper_test.lua b/src/llm/test/google/mapper_test.lua index ccb6e769..fbe53664 100644 --- a/src/llm/test/google/mapper_test.lua +++ b/src/llm/test/google/mapper_test.lua @@ -100,6 +100,49 @@ local function define_tests() end) end) + describe("Positional notes", function() + it("renders a positional system note as user text in place", function() + local function build(note_text) + return mapper.map_messages({ + { role = "system", content = { { type = "text", text = "Base" } } }, + { role = "user", content = { { type = "text", text = "Question" } } }, + { role = "system", content = { { type = "text", text = note_text } }, + metadata = { preserve_position = true } }, + { role = "user", content = { { type = "text", text = "Follow-up" } } }, + }, { model = "gemini-2.5-pro" }) + end + local first_contents, first_system = build("Monday 09:00") + local second_contents, second_system = build("Tuesday 10:30") + for _, system in ipairs({ first_system, second_system }) do + tests.eq(#system, 1) + tests.eq(system[1].text, "Base") + end + for _, contents in ipairs({ first_contents, second_contents }) do + tests.eq(#contents, 3) + tests.eq(contents[1].parts[1].text, "Question") + tests.eq(contents[2].role, "user") + tests.eq(contents[3].parts[1].text, "Follow-up") + end + tests.eq(first_contents[2].parts[1].text, "Monday 09:00") + tests.eq(second_contents[2].parts[1].text, "Tuesday 10:30") + end) + + it("leaves the caller's message metadata intact for later routes", function() + local note = { role = "system", content = { { type = "text", text = "Note" } }, + metadata = { preserve_position = true } } + local answer = { role = "assistant", content = { { type = "text", text = "Answer" } }, + metadata = { thinking_blocks = { { type = "thinking", thinking = "t", signature = "s" } } } } + mapper.map_messages({ + { role = "user", content = { { type = "text", text = "Question" } } }, + answer, + { role = "user", content = { { type = "text", text = "Again" } } }, + note, + }, { model = "gemini-2.5-pro" }) + tests.is_true(note.metadata.preserve_position) + tests.eq(#answer.metadata.thinking_blocks, 1) + end) + end) + describe("Message Mapping", function() it("keeps tool-only rounds separate while cache markers preserve parallel pairing", function() local mapped = mapper.map_messages({ diff --git a/src/llm/test/openai/mapper_test.lua b/src/llm/test/openai/mapper_test.lua index 20e0d769..2ac7d3e3 100644 --- a/src/llm/test/openai/mapper_test.lua +++ b/src/llm/test/openai/mapper_test.lua @@ -72,6 +72,22 @@ local function define_tests() end end) end) + it("keeps instructions identical when positional note text changes", function() + local function build(note_text) + local messages = { + { role = "system", content = "Global guidance" }, + { role = "user", content = "Request" }, + { role = "system", content = note_text, metadata = { preserve_position = true } }, + } + return openai_mapper.extract_instructions(messages), openai_mapper.map_messages(messages) + end + local first_instructions, first_input = build("Monday 09:00") + local second_instructions, second_input = build("Tuesday 10:30") + test.eq(first_instructions, "Global guidance") + test.eq(second_instructions, first_instructions) + test.eq(first_input[2].content[1].text, "Monday 09:00") + test.eq(second_input[2].content[1].text, "Tuesday 10:30") + end) it("keeps positional instructions in the Responses input stream", function() for _, role in ipairs({ "system", "developer" }) do local messages = { diff --git a/src/llm/test/openai_compat/mapper_test.lua b/src/llm/test/openai_compat/mapper_test.lua index b47e7eba..0419f524 100644 --- a/src/llm/test/openai_compat/mapper_test.lua +++ b/src/llm/test/openai_compat/mapper_test.lua @@ -104,6 +104,24 @@ local function define_tests() end) describe("Message Mapping", function() + it("keeps positional system and developer notes at their conversation position", function() + for _, role in ipairs({ "system", "developer" }) do + local mapped = openai_mapper.map_messages({ + { role = "system", content = "Base" }, + { role = "user", content = "Question" }, + { role = role, content = "Note", metadata = { preserve_position = true } }, + { role = "user", content = "Follow-up" }, + }, { model = "gpt-5" }) + test.eq(#mapped, 4) + test.eq(mapped[1].role, "system") + test.eq(mapped[1].content[1].text, "Base") + test.eq(mapped[2].role, "user") + test.eq(mapped[3].role, "system") + test.eq(mapped[3].content[1].text, "Note") + test.eq(mapped[4].role, "user") + end + end) + it("should map standard user, assistant, system messages", function() local contract_messages = { { diff --git a/src/llm/test/prompt_test.lua b/src/llm/test/prompt_test.lua index 1750aa8e..420c554d 100644 --- a/src/llm/test/prompt_test.lua +++ b/src/llm/test/prompt_test.lua @@ -18,6 +18,116 @@ local function define_tests() end end end) + it("keeps positional notes and thinking metadata through clone", function() + local blocks = { { type = "thinking", thinking = "plan", signature = "sig" } } + local builder = prompt.new() + builder:add_user("question") + builder:add_system("note", { preserve_position = true }) + builder:add_developer("guidance", { preserve_position = true }) + builder:add_function_call("lookup", "{}", "call_1") + builder.messages[#builder.messages].metadata = { thinking_blocks = blocks } + local cloned: any = builder:clone() + local messages = cloned:get_messages() + test.eq(#messages, 4) + test.eq(messages[2].role, prompt.ROLE.SYSTEM) + test.is_true(messages[2].metadata.preserve_position) + test.eq(messages[3].role, prompt.ROLE.DEVELOPER) + test.is_true(messages[3].metadata.preserve_position) + test.eq(#messages[4].metadata.thinking_blocks, 1) + test.eq(messages[4].metadata.thinking_blocks[1].signature, "sig") + end) + + describe("bind_thinking", function() + local function block(name) + return { { type = "thinking", thinking = name, signature = "sig-" .. name } } + end + + local function history() + local builder = prompt.new() + builder:add_user("first") + builder:add_function_call("lookup", "{}", "call_1") + builder.messages[#builder.messages].metadata = { thinking_blocks = block("a"), turn = 1 } + builder:add_function_result("lookup", "found", "call_1") + builder:add_message(prompt.ROLE.ASSISTANT, { prompt.text("answer one") }, nil, + { thinking_blocks = block("b"), turn = 1 }) + builder:add_user("second") + builder:add_message(prompt.ROLE.ASSISTANT, { prompt.text("answer two") }, nil, + { thinking_blocks = block("c"), turn = 2 }) + return builder + end + + it("strips thinking blocks from a leading run of older messages", function() + local builder = history() + local bound, err = builder:bind_thinking(function(msg) return msg.metadata.turn >= 2 end) + test.is_nil(err) + test.eq(bound, builder) + local messages = builder:get_messages() + test.is_nil(messages[2].metadata.thinking_blocks) + test.eq(messages[2].metadata.turn, 1) + test.eq(messages[2].function_call.id, "call_1") + test.is_nil(messages[4].metadata.thinking_blocks) + test.eq(messages[4].content[1].text, "answer one") + test.eq(messages[6].metadata.thinking_blocks[1].thinking, "c") + end) + + it("keeps every block when all messages are current", function() + local builder = history() + local _, err = builder:bind_thinking(function() return true end) + test.is_nil(err) + local messages = builder:get_messages() + test.eq(messages[2].metadata.thinking_blocks[1].thinking, "a") + test.eq(messages[4].metadata.thinking_blocks[1].thinking, "b") + test.eq(messages[6].metadata.thinking_blocks[1].thinking, "c") + end) + + it("rejects removal from the middle and leaves the history unchanged", function() + local builder = history() + local bound, err = builder:bind_thinking(function(msg) + return msg.metadata.thinking_blocks[1].thinking ~= "b" + end) + test.is_nil(bound) + test.contains(tostring(err), "leading run") + local messages = builder:get_messages() + test.eq(messages[2].metadata.thinking_blocks[1].thinking, "a") + test.eq(messages[4].metadata.thinking_blocks[1].thinking, "b") + test.eq(messages[6].metadata.thinking_blocks[1].thinking, "c") + end) + + it("rejects a boundary that splits an open tool round", function() + local builder = prompt.new() + builder:add_user("first") + builder:add_message(prompt.ROLE.ASSISTANT, { prompt.text("old") }, nil, { thinking_blocks = block("a") }) + builder:add_user("second") + builder:add_function_call("lookup", "{}", "call_open") + builder.messages[#builder.messages].metadata = { thinking_blocks = block("b") } + local bound, err = builder:bind_thinking(function() return false end) + test.is_nil(bound) + test.contains(tostring(err), "call_open") + local messages = builder:get_messages() + test.eq(messages[2].metadata.thinking_blocks[1].thinking, "a") + test.eq(messages[4].metadata.thinking_blocks[1].thinking, "b") + end) + + it("is a no-op when no message carries thinking blocks", function() + local builder = prompt.new() + builder:add_user("hello") + builder:add_assistant("hi", { turn = 1 }) + builder:add_function_call("lookup", "{}", "call_1") + local consulted = false + local bound, err = builder:bind_thinking(function() + consulted = true + return false + end) + test.is_nil(err) + test.eq(bound, builder) + test.is_false(consulted) + local messages = builder:get_messages() + test.eq(#messages, 3) + test.eq(messages[2].metadata.turn, 1) + test.is_nil(messages[3].metadata) + end) + end) + it("preserves explicit result failures through build and clone without inspecting success payloads", function() local builder = prompt.new() builder:add_function_result("lookup", "denied", "failed", { is_error = true }) diff --git a/src/llm/test/request_golden_test.lua b/src/llm/test/request_golden_test.lua new file mode 100644 index 00000000..693160d4 --- /dev/null +++ b/src/llm/test/request_golden_test.lua @@ -0,0 +1,79 @@ +local claude_mapper = require("claude_mapper") +local bedrock_mapper = require("bedrock_mapper") +local google_mapper = require("google_mapper") +local openai_mapper = require("openai_mapper") +local compat_mapper = require("compat_mapper") +local codec = require("codec") + +-- Message-derived request body sections, recorded from the drivers before +-- positional notes existed. Prompts without positional notes must keep these +-- bytes exactly; any difference changes the provider prefix and its cache. +local GOLDEN = { + claude = "{\"messages\":[{\"content\":[{\"text\":\"Hello\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"signature\":\"sig-greet\",\"thinking\":\"greet\",\"type\":\"thinking\"},{\"text\":\"Hi\",\"type\":\"text\"}],\"role\":\"assistant\"},{\"content\":[{\"text\":\"Look it up\\n\\u003cdeveloper-instruction\\u003ePrefer cached data\\u003c/developer-instruction\\u003e\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"signature\":\"sig-lookup\",\"thinking\":\"lookup\",\"type\":\"thinking\"},{\"id\":\"call_1\",\"input\":{\"q\":\"x\"},\"name\":\"lookup\",\"type\":\"tool_use\"}],\"role\":\"assistant\"},{\"content\":[{\"cache_control\":{\"type\":\"ephemeral\"},\"content\":\"found\",\"tool_use_id\":\"call_1\",\"type\":\"tool_result\"}],\"role\":\"user\"},{\"content\":[{\"text\":\"Answer briefly\",\"type\":\"text\"}],\"role\":\"user\"}],\"system\":[{\"cache_control\":{\"type\":\"ephemeral\"},\"text\":\"You are concise.\",\"type\":\"text\"},{\"text\":\"Mode: review\",\"type\":\"text\"}]}", + bedrock = "{\"messages\":[{\"content\":[{\"text\":\"Hello\"}],\"role\":\"user\"},{\"content\":[{\"reasoningContent\":{\"reasoningText\":{\"signature\":\"sig-greet\",\"text\":\"greet\"}}},{\"text\":\"Hi\"}],\"role\":\"assistant\"},{\"content\":[{\"text\":\"Look it up\"}],\"role\":\"user\"},{\"content\":[{\"reasoningContent\":{\"reasoningText\":{\"signature\":\"sig-lookup\",\"text\":\"lookup\"}}},{\"toolUse\":{\"input\":{\"q\":\"x\"},\"name\":\"lookup\",\"toolUseId\":\"call_1\"}}],\"role\":\"assistant\"},{\"content\":[{\"toolResult\":{\"content\":[{\"text\":\"found\"}],\"status\":\"success\",\"toolUseId\":\"call_1\"}},{\"cachePoint\":{\"type\":\"default\"}}],\"role\":\"user\"}],\"system\":[{\"text\":\"You are concise.\"},{\"cachePoint\":{\"type\":\"default\"}},{\"text\":\"Mode: review\"}]}", + google = "{\"contents\":[{\"parts\":[{\"text\":\"Hello\"}],\"role\":\"user\"},{\"parts\":[{\"text\":\"Hi\"}],\"role\":\"model\"},{\"parts\":[{\"text\":\"Look it up\"}],\"role\":\"user\"},{\"parts\":[{\"text\":\"Prefer cached data\"}],\"role\":\"user\"},{\"parts\":[{\"functionCall\":{\"args\":{\"q\":\"x\"},\"name\":\"lookup\"}}],\"role\":\"model\"},{\"parts\":[{\"functionResponse\":{\"name\":\"lookup\",\"response\":{\"content\":\"found\"}}}],\"role\":\"user\"},{\"parts\":[{\"text\":\"Answer briefly\"}],\"role\":\"user\"}],\"systemInstruction\":[{\"text\":\"You are concise.\"},{\"text\":\"Mode: review\"}]}", + openai = "{\"input\":[{\"content\":[{\"text\":\"Hello\",\"type\":\"input_text\"}],\"role\":\"user\",\"type\":\"message\"},{\"content\":[{\"text\":\"Hi\",\"type\":\"output_text\"}],\"role\":\"assistant\",\"type\":\"message\"},{\"content\":[{\"text\":\"Look it up\",\"type\":\"input_text\"}],\"role\":\"user\",\"type\":\"message\"},{\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\",\"call_id\":\"call_1\",\"name\":\"lookup\",\"type\":\"function_call\"},{\"call_id\":\"call_1\",\"output\":\"found\",\"type\":\"function_call_output\"}],\"instructions\":\"You are concise.\\n\\nPrefer cached data\\n\\nMode: review\\n\\nAnswer briefly\"}", + openai_compat = "[{\"content\":[{\"text\":\"You are concise.\",\"type\":\"text\"}],\"role\":\"system\"},{\"content\":[{\"text\":\"Hello\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":\"Hi\",\"role\":\"assistant\"},{\"content\":[{\"text\":\"Look it up\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"text\":\"Prefer cached data\",\"type\":\"text\"}],\"role\":\"system\"},{\"content\":\"\",\"role\":\"assistant\",\"tool_calls\":[{\"function\":{\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\",\"name\":\"lookup\"},\"id\":\"call_1\",\"type\":\"function\"}]},{\"content\":\"found\",\"name\":\"lookup\",\"role\":\"tool\",\"tool_call_id\":\"call_1\"},{\"content\":[{\"text\":\"Mode: review\",\"type\":\"text\"}],\"role\":\"system\"},{\"content\":[{\"text\":\"Answer briefly\",\"type\":\"text\"}],\"role\":\"system\"}]", +} + +local function thinking(name) + return { { type = "thinking", thinking = name, signature = "sig-" .. name } } +end + +local function text(value) + return { { type = "text", text = value } } +end + +local function conversation(with_developer) + local messages = { + { role = "system", content = text("You are concise.") }, + { role = "cache_marker", marker_id = "agent" }, + { role = "user", content = text("Hello") }, + { role = "assistant", content = text("Hi"), metadata = { thinking_blocks = thinking("greet") } }, + { role = "user", content = text("Look it up") }, + } + if with_developer then + table.insert(messages, { role = "developer", content = text("Prefer cached data") }) + end + table.insert(messages, { + role = "function_call", + content = {}, + function_call = { id = "call_1", name = "lookup", arguments = '{"q":"x"}' }, + metadata = { thinking_blocks = thinking("lookup") }, + }) + table.insert(messages, { role = "function_result", name = "lookup", function_call_id = "call_1", content = text("found") }) + table.insert(messages, { role = "cache_marker", marker_id = "conversation_tail" }) + table.insert(messages, { role = "system", content = text("Mode: review") }) + if with_developer then + table.insert(messages, { role = "developer", content = text("Answer briefly") }) + end + return messages +end + +local function bodies() + local google_contents, google_system = google_mapper.map_messages(conversation(true), { model = "gemini-2.5-pro" }) + return { + claude = codec.canonical_json(claude_mapper.map_messages(conversation(true))), + bedrock = codec.canonical_json(bedrock_mapper.map_messages(conversation(false))), + google = codec.canonical_json({ contents = google_contents, systemInstruction = google_system }), + openai = codec.canonical_json({ + instructions = openai_mapper.extract_instructions(conversation(true)), + input = openai_mapper.map_messages(conversation(true), { model = "gpt-5" }), + }), + openai_compat = codec.canonical_json(compat_mapper.map_messages(conversation(true), { model = "gpt-5" })), + } +end + +local function define_tests() + describe("Request body golden sections", function() + for _, driver in ipairs({ "claude", "bedrock", "google", "openai", "openai_compat" }) do + it("keeps the " .. driver .. " body byte-identical for prompts without positional notes", function() + local body = bodies()[driver] + test.not_nil(body) + test.eq(body, GOLDEN[driver]) + end) + end + end) +end + +return require("test").run_cases(define_tests) From 664bc4bf7fda58be33477b5f2fac49f2ab2e6858 Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 16:12:31 -0500 Subject: [PATCH 06/37] feat(agent): keep the request prefix stable across steps time_aware contributes agent_options.turn_context instead of prompt text; hosts render the note with wippy.agent:turn_context and persist it per user turn. Memory recall runs only for hosts that set persist_context_notes and is returned as positional context_notes the host persists before the step output. The tail cache marker skips trailing positional notes. Each step returns prefix_fingerprint over the system prompt, tools and model. Sorted serialization moves to wippy.agent:canonical and is shared with attention_read. --- src/agent/src/README.md | 77 +++++- src/agent/src/_index.yaml | 22 ++ src/agent/src/agent.lua | 49 +++- src/agent/src/canonical.lua | 21 ++ src/agent/src/tools/_index.yaml | 2 + src/agent/src/tools/attention_read.lua | 14 +- src/agent/src/traits/_index.yaml | 9 +- src/agent/src/traits/init_time_aware.lua | 51 ++-- src/agent/src/turn_context.lua | 37 +++ src/agent/test/_index.yaml | 24 +- src/agent/test/agent_memory_test.lua | 94 ++++--- src/agent/test/agent_test.lua | 255 +++++++++++++++++- src/agent/test/discovery/traits_test.lua | 4 +- .../test/traits/init_time_aware_test.lua | 76 +----- src/agent/test/turn_context_test.lua | 53 ++++ 15 files changed, 587 insertions(+), 201 deletions(-) create mode 100644 src/agent/src/canonical.lua create mode 100644 src/agent/src/turn_context.lua create mode 100644 src/agent/test/turn_context_test.lua diff --git a/src/agent/src/README.md b/src/agent/src/README.md index 83786a3a..c67cd6cd 100644 --- a/src/agent/src/README.md +++ b/src/agent/src/README.md @@ -224,6 +224,7 @@ local response, err = runner:step(builder, { stream_target = { reply_to = process.self(), topic = "stream" }, tool_call = "auto", disable_memory_recall = false, + persist_context_notes = true, -- the host persists response.context_notes in place previous_memory_ids = { "mem_1", "mem_2" }, stop_check = function() return stop_requested end, cancel = turn_cancel, @@ -275,11 +276,37 @@ When the model's first route fails and `llm` answers from a fallback model, the finish_reason = "stop", route_pin = { model = "...", provider_id = "...", provider_model = "..." }, -- nil until a step fell back memory_recall = { memory_ids = { "mem_3" }, count = 1 }, - memory_prompt = { ... }, + context_notes = { ... }, -- positional notes sent after the conversation; nil if none + prefix_fingerprint = "9f2c...", -- sha256 of system prompt, tools and model memory_recall_error = nil -- string when a configured recall failed } ``` +### Context notes + +A step can send notes after the conversation, such as recalled memory. Each +note is a developer message with `metadata.preserve_position = true` and is +returned in `context_notes`. The host persists the notes, in order, after the +messages it passed to the step and before the step output (assistant text or +function calls). The next step then replays them at the position they were sent, +so the provider prefix, the prompt cache and earlier thinking blocks stay valid. +When the host discards the step output, it discards the notes with it. + +Memory recall runs only when the host sets `persist_context_notes = true`. A host +that does not persist the notes would send a note once and drop it from the next +request, which edits the history before the output it produced; such a host gets +no recall. The recall cooldown reads `metadata.memory_ids` from the persisted +notes. + +### Prefix fingerprint + +`prefix_fingerprint` identifies what providers bind cached prefixes and thinking +blocks to: the final system prompt, the name-sorted tool schemas and the model. +It is the same on every step of one agent configuration and changes when any of +them changes. Hosts store it on persisted rows; a row whose fingerprint differs +from the current step's has thinking blocks the provider will no longer accept, +which `prompt` `builder:bind_thinking` strips as a leading run. + A configured recall that fails (the provider errors, cannot be opened, or returns no valid `memories` list) does not stop the step: generation runs without recalled memory and `memory_recall_error` describes the failure. It is @@ -290,11 +317,39 @@ error. Skipped recall and an empty `memories` list are not errors. ### Prompt caching The runner marks two cache breakpoints: after the system prompt and after the -newest user or tool-result message. Trailing developer messages in the builder -(lifecycle messages the host appends for the current step) and the memory -recall are placed after the conversation breakpoint, so the next step reuses -the cached prefix. Prompt function output is part of the system prompt and must -be deterministic across steps to keep the prefix cacheable. +newest user or tool-result message. Trailing developer messages and positional +notes in the builder (lifecycle messages, turn context) and the memory recall +are placed after the conversation breakpoint, so the next step reuses the cached +prefix. + +Prompt function output is part of the system prompt. Prompt functions must +return the same text on every step of a conversation: anything that changes per +step or per turn (the current time, recalled facts, status) changes the prefix +before every earlier message. Put such context in positional notes the host +persists instead (see Turn context). + +### Turn context + +`agent_options.turn_context` declares per-turn context the host adds to each +user turn. The `time_aware` trait sets +`turn_context.time = { timezone = , format = }`. +When a user message arrives, the host renders the note once and persists it as a +positional note right after that message: + +```lua +local turn_context = require("turn_context") -- wippy.agent:turn_context + +local note, err = turn_context.render(runner.agent_options, time.now()) +-- "The current date and time is Tuesday, November 3, 2026 at 09:41 UTC." +if note then + builder:add_system(note, { preserve_position = true }) +end +``` + +`render` returns nil when the agent declares no time context and an error for an +unknown timezone. `format` is an optional Go time layout that replaces the +default wording after "The current date and time is". The note carries the exact +minute and is never rebuilt; the framework does not inject it per step. ### Token Tracking @@ -357,7 +412,6 @@ entries: context: timezone: UTC context: - time_interval: 15 timezone: UTC ``` @@ -483,23 +537,24 @@ local function execute(base_prompt, context) prompt = "Additional prompt text...", tools = { { id = "my_ns:dynamic_tool" } }, delegates = { { id = "other_ns:agent", name = "helper" } }, - context = { extra_key = "value" } + context = { extra_key = "value" }, + agent_options = { turn_context = { time = { timezone = "UTC" } } } } end return { execute = execute } ``` ```lua --- Prompt function: called at each agent:step(), returns dynamic prompt text +-- Prompt function: called at each agent:step(); must return the same text on every step local function execute(base_prompt) - return "Current time: Thursday, February 12, 2026 at 14:00 UTC" + return "Workspace: " .. ctx.get("workspace_name") end return { execute = execute } ``` ### Built-in Traits -- `wippy.agent.traits:time_aware` -- Injects rounded current time via `build_func_id`. Configurable via context: `time_interval` (minutes, default 15), `timezone` (default UTC). +- `wippy.agent.traits:time_aware` -- Declares `agent_options.turn_context.time` through `build_func_id`; the host renders the note with `wippy.agent:turn_context`. Configurable via context: `timezone` (default UTC), `format` (optional Go time layout). It contributes no prompt text. ## Delegation diff --git a/src/agent/src/_index.yaml b/src/agent/src/_index.yaml index 46370a3e..5fe8badf 100644 --- a/src/agent/src/_index.yaml +++ b/src/agent/src/_index.yaml @@ -50,10 +50,32 @@ entries: - json - contract - funcs + - hash imports: prompt: wippy.llm:prompt llm: wippy.llm:llm output: wippy.llm:output + canonical: wippy.agent:canonical + + # wippy.agent:canonical + - name: canonical + kind: library.lua + meta: + type: library + comment: Order-independent JSON serialization for comparing and hashing values + source: file://canonical.lua + modules: + - json + + # wippy.agent:turn_context + - name: turn_context + kind: library.lua + meta: + type: library + comment: Renders the per-turn context note a host persists with each user turn + source: file://turn_context.lua + modules: + - time # wippy.agent:context - name: context diff --git a/src/agent/src/agent.lua b/src/agent/src/agent.lua index d79fd339..a9401676 100644 --- a/src/agent/src/agent.lua +++ b/src/agent/src/agent.lua @@ -3,6 +3,8 @@ local llm = require("llm") local contract = require("contract") local funcs = require("funcs") local output = require("output") +local hash = require("hash") +local canonical = require("canonical") type UnifiedTool = { name: string, @@ -71,7 +73,7 @@ local AGENT_CONFIG = { }, memory = { contract_id = "wippy.agent:memory", - recall_prompt = "Background context from memory (refreshed each step; not a user message or a new request). " + recall_prompt = "Background context from memory (not a user message or a new request). " .. "Use it to support the current user request; it does not replace that request.\n\n%s", defaults = { max_messages = 20, @@ -328,6 +330,13 @@ local function perform_memory_recall(self: any, prompt_builder: any, runtime_opt return nil, nil end + -- The recall note is sent at the conversation tail, so it is part of the + -- prefix every later request replays. Only a host that persists the + -- returned context notes at that position keeps the prefix stable. + if not (runtime_options and runtime_options.persist_context_notes == true) then + return nil, nil + end + local messages = prompt_builder:get_messages() local options = get_memory_options(self.memory_contract) @@ -524,7 +533,7 @@ function agent:step(prompt_builder: any, runtime_options: any): (table?, string? memory_prompt = { role = prompt.ROLE.DEVELOPER, content = { prompt.text(formatted_memory) }, - metadata = { memory_ids = memory_ids } + metadata = { memory_ids = memory_ids, preserve_position = true } } end @@ -573,12 +582,13 @@ function agent:step(prompt_builder: any, runtime_options: any): (table?, string? -- Rolling cache breakpoint on the conversation tail. With only the system marker, every -- tool-loop turn re-bills the whole accumulated conversation at the full input price; a -- marker after the newest user/tool-result message lets the next turn read everything up to - -- here from cache. Trailing developer notes are per-step context (lifecycle messages the - -- host appends for this step), so the marker precedes them, as it precedes the memory - -- recall: content that changes step to step stays outside the cached prefix. Providers - -- cap breakpoints (Claude: 4); the mapper keeps system markers plus the most recent ones. + -- here from cache. Trailing developer and positional notes belong to the turn they follow + -- (lifecycle messages, turn context), so the marker lands on that turn before them, as it + -- precedes the memory recall. Providers cap breakpoints (Claude: 4); the mapper keeps + -- system markers plus the most recent ones. local tail_index = #conversation_messages - while tail_index > 0 and conversation_messages[tail_index].role == prompt.ROLE.DEVELOPER do + while tail_index > 0 and (conversation_messages[tail_index].role == prompt.ROLE.DEVELOPER + or prompt.is_positional_note(conversation_messages[tail_index])) do tail_index = tail_index - 1 end local tail = conversation_messages[tail_index] @@ -595,7 +605,7 @@ function agent:step(prompt_builder: any, runtime_options: any): (table?, string? end end - -- Append memory recall after conversation + -- The recall note follows the conversation, where the host persists it if memory_prompt then table.insert(final_messages, memory_prompt) end @@ -649,6 +659,17 @@ function agent:step(prompt_builder: any, runtime_options: any): (table?, string? --end + local fingerprint, fingerprint_err = hash.sha256(canonical.encode({ + system = complete_system_prompt, + tools = options.tools or {}, + model = self.model, + })) + if fingerprint_err then + return nil, output.errors.generate():kind(output.ERROR_TYPE.INVALID_REQUEST) + :message("Prefix fingerprint failed: " .. tostring(fingerprint_err)):build() + end + local context_notes = memory_prompt and { memory_prompt } or nil + if runtime_options.stop_check and runtime_options.stop_check() then return { stopped = true, memory_recall_error = memory_recall_error }, nil end @@ -671,7 +692,9 @@ function agent:step(prompt_builder: any, runtime_options: any): (table?, string? stopped = true, result = result.result, finish_reason = result.finish_reason, - memory_recall_error = memory_recall_error + memory_recall_error = memory_recall_error, + context_notes = context_notes, + prefix_fingerprint = fingerprint }, nil end @@ -681,7 +704,9 @@ function agent:step(prompt_builder: any, runtime_options: any): (table?, string? finish_reason = result.finish_reason, metadata = result.metadata, route_pin = route_pin(runtime_options.route, result.metadata), - memory_recall_error = memory_recall_error + memory_recall_error = memory_recall_error, + context_notes = context_notes, + prefix_fingerprint = fingerprint } if output.detect_truncation(result) then @@ -700,10 +725,6 @@ function agent:step(prompt_builder: any, runtime_options: any): (table?, string? } end - if memory_prompt then - response.memory_prompt = memory_prompt - end - if result.tool_calls then for _, tool_call in ipairs(result.tool_calls) do local tool_info = self.tools[tool_call.name] diff --git a/src/agent/src/canonical.lua b/src/agent/src/canonical.lua new file mode 100644 index 00000000..12074886 --- /dev/null +++ b/src/agent/src/canonical.lua @@ -0,0 +1,21 @@ +local json = require("json") + +-- Serializes a value with object keys in sorted order, so equal values always +-- produce equal text. Array indices are written as keys, which keeps sparse +-- tables unambiguous. +local canonical = {} + +function canonical.encode(value: any): string + if type(value) ~= "table" then return json.encode(value) or "null" end + local keys, out = {}, {} + for key in pairs(value) do keys[#keys+1] = tostring(key) end + table.sort(keys) + for _, key in ipairs(keys) do + local child = value[key] + if child == nil then child = value[tonumber(key)] end + out[#out+1] = (json.encode(key) or '""') .. ":" .. canonical.encode(child) + end + return "{" .. table.concat(out, ",") .. "}" +end + +return canonical diff --git a/src/agent/src/tools/_index.yaml b/src/agent/src/tools/_index.yaml index 9b5dc23b..bb3693ac 100644 --- a/src/agent/src/tools/_index.yaml +++ b/src/agent/src/tools/_index.yaml @@ -46,6 +46,8 @@ entries: source: file://attention_read.lua modules: - json + imports: + canonical: wippy.agent:canonical - name: attention_find_semantic kind: function.lua diff --git a/src/agent/src/tools/attention_read.lua b/src/agent/src/tools/attention_read.lua index c7db9487..80e07674 100644 --- a/src/agent/src/tools/attention_read.lua +++ b/src/agent/src/tools/attention_read.lua @@ -1,4 +1,5 @@ local json = require("json") +local canonical = require("canonical") local M = {} local PREFIX = "wippy.agent.tools:" @@ -105,18 +106,7 @@ function M.normalize(name, args) end -- Sorted serialization is used only for local deduplication, never as authority. -function M.canonical(value: any): string - if type(value) ~= "table" then return json.encode(value) or "null" end - local keys, out = {}, {} - for key in pairs(value) do keys[#keys+1] = tostring(key) end - table.sort(keys) - for _, key in ipairs(keys) do - local child = value[key] - if child == nil then child = value[tonumber(key)] end - out[#out+1] = (json.encode(key) or '""') .. ":" .. M.canonical(child) - end - return "{" .. table.concat(out, ",") .. "}" -end +M.canonical = canonical.encode M.validation_errors = { ["invalid-request"]=true, ["semantic-filter-required"]=true, ["css-root-required"]=true, diff --git a/src/agent/src/traits/_index.yaml b/src/agent/src/traits/_index.yaml index 58364d70..411fd177 100644 --- a/src/agent/src/traits/_index.yaml +++ b/src/agent/src/traits/_index.yaml @@ -64,13 +64,12 @@ entries: kind: function.lua meta: type: function - comment: Initializes time-aware trait with current rounded time context + comment: Declares the per-turn time context the host renders with each user turn source: file://init_time_aware.lua modules: - - time - ctx method: execute - + # wippy.agent.traits:time_aware - name: time_aware kind: registry.entry @@ -78,13 +77,11 @@ entries: name: Time Aware type: agent.trait title: Time Aware - comment: Adds current time context to agent for time-aware responses with cache stability + comment: Declares agent_options.turn_context.time so the host adds the current time to each user turn tags: - time - context - - rounded - cache_stable build_func_id: wippy.agent.traits:init_time_aware context: - time_interval: 15 timezone: UTC diff --git a/src/agent/src/traits/init_time_aware.lua b/src/agent/src/traits/init_time_aware.lua index 50bf4be3..8132c377 100644 --- a/src/agent/src/traits/init_time_aware.lua +++ b/src/agent/src/traits/init_time_aware.lua @@ -1,45 +1,26 @@ -local time = require("time") local ctx = require("ctx") type TimeAwareResult = { - prompt: string, + agent_options: { + turn_context: { + time: { timezone: string, format: string? }, + }, + }, } +-- Declares the per-turn time context; hosts render it with +-- wippy.agent:turn_context and persist it with each user turn. local function execute(base_prompt: string?): TimeAwareResult - local timezone = ctx.get("timezone") or "UTC" - local interval_minutes = ctx.get("time_interval") or 15 - - local current_time = time.now() - - if timezone ~= "UTC" then - local loc, err = time.load_location(tostring(timezone)) - if not err and loc then - current_time = current_time:in_location(loc) - end - end - - local year = current_time:year() - local month = current_time:month() - local day = current_time:day() - local hour = current_time:hour() - local minute = current_time:minute() - local weekday = current_time:weekday() - - local rounded_minute = math.floor(minute / interval_minutes) * interval_minutes - - local weekdays = { "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday" } - local weekday_name = weekdays[weekday + 1] - - local months = { "January", "February", "March", "April", "May", "June", - "July", "August", "September", "October", "November", "December" } - local month_name = months[month] - - local full_prompt = string.format( - "The current date and time is %s, %s %d, %d at %02d:%02d %s. Use this temporal context when relevant to your responses.", - weekday_name, month_name, day, year, hour, rounded_minute, timezone) - + local format = ctx.get("format") return { - prompt = full_prompt + agent_options = { + turn_context = { + time = { + timezone = tostring(ctx.get("timezone") or "UTC"), + format = type(format) == "string" and format or nil, + }, + }, + }, } end diff --git a/src/agent/src/turn_context.lua b/src/agent/src/turn_context.lua new file mode 100644 index 00000000..c042d20c --- /dev/null +++ b/src/agent/src/turn_context.lua @@ -0,0 +1,37 @@ +local time = require("time") + +-- Per-turn context is host state: the host renders the note once when a user +-- turn arrives, persists it as a positional note after that turn and replays +-- it unchanged on every later request, so the provider prefix stays stable. +local turn_context = {} + +local WEEKDAYS = { "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday" } +local MONTHS = { "January", "February", "March", "April", "May", "June", + "July", "August", "September", "October", "November", "December" } + +-- Returns the time note for agent_options.turn_context.time at the instant now, +-- or nil when the agent declares no time context. +function turn_context.render(agent_options: table?, now: any): (string?, string?) + local config = agent_options and agent_options.turn_context + local time_config = type(config) == "table" and config.time or nil + if type(time_config) ~= "table" then + return nil, nil + end + + local timezone = tostring(time_config.timezone or "UTC") + local location, location_err = time.load_location(timezone) + if location_err or not location then + return nil, "turn context timezone " .. timezone .. " cannot be loaded: " .. tostring(location_err) + end + local local_time = now:in_location(location) + + if type(time_config.format) == "string" and time_config.format ~= "" then + return "The current date and time is " .. local_time:format(time_config.format) .. ".", nil + end + + return string.format("The current date and time is %s, %s %d, %d at %02d:%02d %s.", + WEEKDAYS[local_time:weekday() + 1], MONTHS[local_time:month()], local_time:day(), local_time:year(), + local_time:hour(), local_time:minute(), timezone), nil +end + +return turn_context diff --git a/src/agent/test/_index.yaml b/src/agent/test/_index.yaml index 75f5d97f..21ebb601 100644 --- a/src/agent/test/_index.yaml +++ b/src/agent/test/_index.yaml @@ -362,7 +362,7 @@ entries: meta: name: Init Time Aware Trait Test type: test - comment: Tests the time-aware trait initialization and prompt generation + comment: Tests the time-aware trait turn context contribution group: Agent Framework tags: - traits @@ -370,13 +370,31 @@ entries: - tests source: file://traits/init_time_aware_test.lua modules: - - time - - ctx + - funcs imports: test: wippy.test:test init_time_aware: wippy.agent.traits:init_time_aware method: run_tests + - name: turn_context_test + kind: function.lua + meta: + name: Turn Context Test + type: test + comment: Tests the per-turn context note rendered by hosts + group: Agent Framework + tags: + - time + - context + - tests + source: file://turn_context_test.lua + modules: + - time + imports: + test: wippy.test:test + turn_context: wippy.agent:turn_context + method: run_tests + # app:delay_tool - name: delay_tool kind: function.lua diff --git a/src/agent/test/agent_memory_test.lua b/src/agent/test/agent_memory_test.lua index 15fb568d..3baf21cb 100644 --- a/src/agent/test/agent_memory_test.lua +++ b/src/agent/test/agent_memory_test.lua @@ -3,6 +3,9 @@ local prompt = require("prompt") local json = require("json") local time = require("time") +-- Recall runs only for hosts that persist the context notes a step returns. +local HOST = { persist_context_notes = true } + local function define_tests() describe("Agent Memory Processing - Detailed Logic", function() local mock_llm @@ -207,7 +210,7 @@ local function define_tests() local builder = mock_prompt.new() builder:add_user("first") builder:add_assistant("second") - local result, err = instance:step(builder) + local result, err = instance:step(builder, HOST) test.is_nil(err) test.not_nil(result) test.eq(#mock_contract._open_calls, 0) @@ -239,7 +242,7 @@ local function define_tests() prompt_builder:add_function_result("calculator", "Result: 4", "call_123") prompt_builder:add_user("What about 3 + 3?") - test_agent:step(prompt_builder) + test_agent:step(prompt_builder, HOST) -- Inspect what was passed to memory contract local call_log = memory_contract:get_call_log() @@ -279,7 +282,7 @@ local function define_tests() prompt_builder:add_assistant("Response 2") prompt_builder:add_user("Message 3") - test_agent:step(prompt_builder) + test_agent:step(prompt_builder, HOST) local call_log = memory_contract:get_call_log() local recent_actions = (call_log[1] :: any).recent_actions @@ -316,7 +319,7 @@ local function define_tests() prompt_builder:add_user("User message 2") prompt_builder:add_assistant("Assistant response 2") - test_agent:step(prompt_builder) + test_agent:step(prompt_builder, HOST) local call_log = memory_contract:get_call_log() local recent_actions = (call_log[1] :: any).recent_actions @@ -353,7 +356,7 @@ local function define_tests() table.insert(messages, { role = "assistant", content = {} }) -- Empty content table.insert(messages, { role = "user", content = {{ type = "text", text = "" }} }) -- Empty text - test_agent:step(prompt_builder) + test_agent:step(prompt_builder, HOST) local call_log = memory_contract:get_call_log() local recent_actions = (call_log[1] :: any).recent_actions @@ -397,7 +400,7 @@ local function define_tests() { type = "image", source = { type = "url", url = "https://example.com/a.png" } } })) - test_agent:step(prompt_builder) + test_agent:step(prompt_builder, HOST) local call_log = memory_contract:get_call_log() test.eq(#call_log, 1) @@ -441,7 +444,7 @@ local function define_tests() prompt_builder:add_user("Current message") - test_agent:step(prompt_builder) + test_agent:step(prompt_builder, HOST) local call_log = memory_contract:get_call_log() local previous_memories = (call_log[1] :: any).previous_memories @@ -481,7 +484,7 @@ local function define_tests() messages[#messages].metadata = { memory_ids = { "mem_" .. i } } end - test_agent:step(prompt_builder) + test_agent:step(prompt_builder, HOST) local call_log = memory_contract:get_call_log() @@ -523,6 +526,7 @@ local function define_tests() -- Pass runtime previous_memory_ids - FIXED: Use correct 2-parameter signature local runtime_options = { + persist_context_notes = true, previous_memory_ids = { "mem_from_runtime_1", "mem_from_runtime_2" } } @@ -570,7 +574,7 @@ local function define_tests() prompt_builder:add_user("Message " .. i) end - test_agent:step(prompt_builder) + test_agent:step(prompt_builder, HOST) local call_log = memory_contract:get_call_log() local constraints = (call_log[1] :: any).constraints @@ -622,7 +626,7 @@ local function define_tests() prompt_builder:add_user("Message " .. i) end - local result = test_agent:step(prompt_builder) + local result = test_agent:step(prompt_builder, HOST) test.not_nil(result) local call_log = memory_contract:get_call_log() @@ -662,7 +666,7 @@ local function define_tests() prompt_builder:add_assistant("Response 1") prompt_builder:add_user("Message 2") - local result = test_agent:step(prompt_builder) + local result = test_agent:step(prompt_builder, HOST) test.not_nil(result.memory_recall) test.not_nil(memory_contract:get_call_log()) @@ -691,7 +695,7 @@ local function define_tests() prompt_builder:add_user("Message 1") prompt_builder:add_assistant("Response 1") - local result = test_agent:step(prompt_builder) + local result = test_agent:step(prompt_builder, HOST) test.is_nil(result.memory_recall) test.eq(#memory_contract:get_call_log(), 0) @@ -731,7 +735,7 @@ local function define_tests() prompt_builder:add_user("Message 2") prompt_builder:add_assistant("Response 2") - local result = test_agent:step(prompt_builder) + local result = test_agent:step(prompt_builder, HOST) test.is_nil(result.memory_recall) -- Should not recall due to cooldown test.eq(#memory_contract:get_call_log(), 0) @@ -768,7 +772,7 @@ local function define_tests() prompt_builder:add_user("Message 2") prompt_builder:add_assistant("Response 2") - local result = test_agent:step(prompt_builder) + local result = test_agent:step(prompt_builder, HOST) test.not_nil(result.memory_recall) -- Should recall after cooldown test.eq(#memory_contract:get_call_log(), 1) @@ -797,7 +801,7 @@ local function define_tests() prompt_builder:add_user("Message 2") -- Disable memory recall via runtime options - FIXED: Use correct 2-parameter signature - local result = test_agent:step(prompt_builder, { disable_memory_recall = true }) + local result = test_agent:step(prompt_builder, { disable_memory_recall = true, persist_context_notes = true }) test.is_nil(result.memory_recall) test.eq(#memory_contract:get_call_log(), 0) @@ -817,7 +821,7 @@ local function define_tests() builder:add_user("Rebuild the Word handout and finish every edit.") builder:add_assistant("The first edit is complete.") builder:add_function_result("edit_document", "Second edit stored", "edit-2") - assert(instance:step(builder)) + assert(instance:step(builder, HOST)) local recall = test_messages[#test_messages] test.eq(recall.role, "developer") @@ -863,13 +867,14 @@ local function define_tests() prompt_builder:add_assistant("Response 1") prompt_builder:add_user("Message 2") - local result = test_agent:step(prompt_builder) + local result = test_agent:step(prompt_builder, HOST) test.not_nil(result.memory_recall) - test.not_nil(result.memory_prompt) + test.eq(#result.context_notes, 1) - local memory_prompt = result.memory_prompt + local memory_prompt = result.context_notes[1] test.eq(memory_prompt.role, "developer") + test.is_true(memory_prompt.metadata.preserve_position) local memory_text = memory_prompt.content[1].text test.contains(memory_text, "Background context from memory") test.contains(memory_text, "- First memory") @@ -903,10 +908,10 @@ local function define_tests() prompt_builder:add_assistant("Response 1") prompt_builder:add_user("Message 2") - local result = test_agent:step(prompt_builder) + local result = test_agent:step(prompt_builder, HOST) test.is_nil(result.memory_recall) -- No memories returned - test.is_nil(result.memory_prompt) + test.is_nil(result.context_notes) end) it("should inject memory into internal message flow", function() @@ -935,7 +940,7 @@ local function define_tests() prompt_builder:add_assistant("Response 1") prompt_builder:add_user("Trigger memory") - test_agent:step(prompt_builder) + test_agent:step(prompt_builder, HOST) -- Check that memory was injected into LLM messages local llm_messages = test_messages @@ -986,6 +991,7 @@ local function define_tests() -- Pass runtime context - FIXED: Use correct 2-parameter signature local runtime_options = { + persist_context_notes = true, context = { session_id = "test_session_123", shared_key = "from_runtime_context" -- Should override contract context @@ -1028,7 +1034,7 @@ local function define_tests() prompt_builder:add_assistant("Response 1") prompt_builder:add_user("Message 2") - test_agent:step(prompt_builder) + test_agent:step(prompt_builder, HOST) local call_log = memory_contract:get_call_log() test.eq(#call_log, 1) @@ -1055,7 +1061,7 @@ local function define_tests() }) do local builder = mock_prompt.new() builder:add_user("Recall this run's memories") - local result, err = instance:step(builder, { context = identity }) + local result, err = instance:step(builder, { context = identity, persist_context_notes = true }) test.is_nil(err) test.not_nil(result) local opened = mock_contract._open_calls[#mock_contract._open_calls] @@ -1095,7 +1101,7 @@ local function define_tests() prompt_builder:add_user("Message 2") -- Should not crash, just skip memory recall - local result = test_agent:step(prompt_builder) + local result = test_agent:step(prompt_builder, HOST) test.not_nil(result) test.is_nil(result.memory_recall) @@ -1127,7 +1133,7 @@ local function define_tests() prompt_builder:add_user("Message 2") -- Should not crash, just skip memory recall - local result = test_agent:step(prompt_builder) + local result = test_agent:step(prompt_builder, HOST) test.not_nil(result) test.is_nil(result.memory_recall) @@ -1162,7 +1168,7 @@ local function define_tests() prompt_builder:add_user("Message 2") -- Should handle gracefully - local result = test_agent:step(prompt_builder) + local result = test_agent:step(prompt_builder, HOST) test.not_nil(result) test.is_nil(result.memory_recall) @@ -1206,7 +1212,7 @@ local function define_tests() prompt_builder:add_assistant("Response 1") prompt_builder:add_user("Message 2") - local result = test_agent:step(prompt_builder) + local result = test_agent:step(prompt_builder, HOST) -- Should handle large dataset test.not_nil(result) @@ -1257,7 +1263,7 @@ local function define_tests() it("reports a provider recall error and still generates", function() mock_contract._current_instance = recalling(nil, "vector store offline") - local result, err = agent.new(recall_spec):step(conversation()) + local result, err = agent.new(recall_spec):step(conversation(), HOST) test.is_nil(err) test.eq(result.result, "Mock LLM response") @@ -1270,7 +1276,7 @@ local function define_tests() it("reports a provider that cannot be opened", function() mock_contract._current_error = "binding not found" - local result, err = agent.new(recall_spec):step(conversation()) + local result, err = agent.new(recall_spec):step(conversation(), HOST) test.is_nil(err) test.not_nil(result.memory_recall_error:find("binding not found", 1, true)) @@ -1280,7 +1286,7 @@ local function define_tests() mock_contract._current_instance = nil mock_contract._current_error = nil - local result, err = agent.new(recall_spec):step(conversation()) + local result, err = agent.new(recall_spec):step(conversation(), HOST) test.is_nil(err) test.not_nil(result.memory_recall_error) @@ -1289,7 +1295,7 @@ local function define_tests() it("reports a memory contract that cannot be resolved", function() agent._contract = { get = function() return nil, "contract registry unavailable" end } - local result, err = agent.new(recall_spec):step(conversation()) + local result, err = agent.new(recall_spec):step(conversation(), HOST) test.is_nil(err) test.not_nil(result.memory_recall_error:find("contract registry unavailable", 1, true)) @@ -1305,7 +1311,7 @@ local function define_tests() } for _, case in ipairs(malformed) do mock_contract._current_instance = recalling(case.value, nil) - local result, err = agent.new(recall_spec):step(conversation()) + local result, err = agent.new(recall_spec):step(conversation(), HOST) test.is_nil(err, case.label) test.not_nil(result.memory_recall_error, case.label) test.is_false(memory_prompt_sent(), case.label) @@ -1314,18 +1320,18 @@ local function define_tests() it("reports no error for an empty recall or a skipped recall", function() mock_contract._current_instance = recalling({ memories = {} }, nil) - local result = agent.new(recall_spec):step(conversation()) + local result = agent.new(recall_spec):step(conversation(), HOST) test.is_nil(result.memory_recall_error) - result = agent.new(recall_spec):step(conversation(), { disable_memory_recall = true }) + result = agent.new(recall_spec):step(conversation(), { disable_memory_recall = true, persist_context_notes = true }) test.is_nil(result.memory_recall_error) local short = mock_prompt.new() short:add_user("Only message") - result = agent.new(recall_spec):step(short) + result = agent.new(recall_spec):step(short, HOST) test.is_nil(result.memory_recall_error) - result = agent.new({ id = "plain", name = "Plain", prompt = "p", tools = {} }):step(conversation()) + result = agent.new({ id = "plain", name = "Plain", prompt = "p", tools = {} }):step(conversation(), HOST) test.is_nil(result.memory_recall_error) end) @@ -1333,7 +1339,7 @@ local function define_tests() mock_contract._current_instance = recalling(nil, "vector store offline") agent._llm = { generate = function() return nil, "provider overloaded" end } - local result, err = agent.new(recall_spec):step(conversation()) + local result, err = agent.new(recall_spec):step(conversation(), HOST) test.eq(err, "provider overloaded") test.not_nil(result) @@ -1348,7 +1354,7 @@ local function define_tests() message = "prompt exceeds context window", details = details }) agent._llm = { generate = function() return nil, cause, details end } - local result, err, admission = agent.new(recall_spec):step(conversation()) + local result, err, admission = agent.new(recall_spec):step(conversation(), HOST) test.eq(err, cause) test.eq(admission, details) @@ -1362,19 +1368,19 @@ local function define_tests() agent._llm = { generate = function() return { result = "", finish_reason = "length", tokens = {} } end } - local truncated = agent.new(recall_spec):step(conversation()) + local truncated = agent.new(recall_spec):step(conversation(), HOST) test.is_true(truncated.truncated) test.not_nil(truncated.memory_recall_error) agent._llm = { generate = function() return { result = "partial", finish_reason = "cancelled", tokens = {} } end } - local cancelled = agent.new(recall_spec):step(conversation()) + local cancelled = agent.new(recall_spec):step(conversation(), HOST) test.is_true(cancelled.stopped) test.not_nil(cancelled.memory_recall_error) agent._llm = mock_llm - local stopped = agent.new(recall_spec):step(conversation(), { stop_check = function() return true end }) + local stopped = agent.new(recall_spec):step(conversation(), { stop_check = function() return true end, persist_context_notes = true }) test.is_true(stopped.stopped) test.not_nil(stopped.memory_recall_error) end) @@ -1408,7 +1414,7 @@ local function define_tests() prompt_builder:add_assistant("First response") prompt_builder:add_user("Second message") - test_agent:step(prompt_builder) + test_agent:step(prompt_builder, HOST) -- Verify message order in LLM input local llm_messages = test_messages @@ -1462,7 +1468,7 @@ local function define_tests() local original_message_count = #prompt_builder:get_messages() - test_agent:step(prompt_builder) + test_agent:step(prompt_builder, HOST) -- Original prompt builder should be unchanged test.eq(#prompt_builder:get_messages(), original_message_count) diff --git a/src/agent/test/agent_test.lua b/src/agent/test/agent_test.lua index e7da33c0..928d9e53 100644 --- a/src/agent/test/agent_test.lua +++ b/src/agent/test/agent_test.lua @@ -974,13 +974,72 @@ local function define_tests() prompt_builder:add_assistant("Of course!") prompt_builder:add_user("What's 2 + 2?") - local result = test_agent:step(prompt_builder) + local result = test_agent:step(prompt_builder, { persist_context_notes = true }) test.not_nil(result) test.not_nil(result.memory_recall) test.not_nil(result.memory_recall.memory_ids) test.gt(result.memory_recall.count, 0) - test.not_nil(result.memory_prompt) + test.eq(#result.context_notes, 1) + local note = result.context_notes[1] + test.eq(note.role, prompt.ROLE.DEVELOPER) + test.is_true(note.metadata.preserve_position) + test.eq(note.metadata.memory_ids, result.memory_recall.memory_ids) + test.is_nil(result.memory_prompt) + end) + + it("does not consult memory when the host does not persist context notes", function() + local opened = 0 + agent._contract = { + get = function() + return { open = function() opened = opened + 1 end } + end + } + local sent: any = nil + agent._llm = { + generate = function(messages) + sent = messages + return { result = "ok", finish_reason = "stop" } + end + } + local test_agent = agent.new(compiled_spec_with_memory) + local prompt_builder = prompt.new() + prompt_builder:add_user("Hello") + prompt_builder:add_assistant("Hi there!") + prompt_builder:add_user("Can you help me calculate something?") + + local result = test_agent:step(prompt_builder) + + test.eq(opened, 0) + test.is_nil(result.memory_recall) + test.is_nil(result.context_notes) + for _, message in ipairs(sent) do + test.is_true(message.role ~= prompt.ROLE.DEVELOPER) + end + end) + + it("sends the recall note at the conversation tail where the host persists it", function() + local sent: any = nil + agent._llm = { + generate = function(messages) + sent = messages + return { result = "ok", finish_reason = "stop" } + end + } + local test_agent = agent.new(compiled_spec_with_memory) + local prompt_builder = prompt.new() + prompt_builder:add_user("Hello") + prompt_builder:add_assistant("Hi there!") + prompt_builder:add_user("Can you help me calculate something?") + + local result = test_agent:step(prompt_builder, { persist_context_notes = true }) + + test.eq(#result.context_notes, 1) + local n = #sent + test.eq(sent[n], result.context_notes[1]) + test.eq(sent[n - 1].role, prompt.ROLE.CACHE_MARKER) + test.eq(sent[n - 1].marker_id, "conversation_tail") + test.eq(sent[n - 2].role, prompt.ROLE.USER) end) it("should skip memory recall when conversation is too short", function() @@ -990,11 +1049,11 @@ local function define_tests() -- Only one message - below min_conversation_length prompt_builder:add_user("Hello") - local result = test_agent:step(prompt_builder) + local result = test_agent:step(prompt_builder, { persist_context_notes = true }) test.not_nil(result) test.is_nil(result.memory_recall) - test.is_nil(result.memory_prompt) + test.is_nil(result.context_notes) end) it("should skip memory recall when disabled", function() @@ -1009,11 +1068,11 @@ local function define_tests() prompt_builder:add_user("What's 2 + 2?") -- Disable memory recall via runtime options - local result = test_agent:step(prompt_builder, { disable_memory_recall = true }) + local result = test_agent:step(prompt_builder, { disable_memory_recall = true, persist_context_notes = true }) test.not_nil(result) test.is_nil(result.memory_recall) - test.is_nil(result.memory_prompt) + test.is_nil(result.context_notes) end) it("should handle memory contract errors gracefully", function() @@ -1037,7 +1096,7 @@ local function define_tests() prompt_builder:add_user("Test memory") -- Should not fail, just skip memory recall - local result = test_agent:step(prompt_builder) + local result = test_agent:step(prompt_builder, { persist_context_notes = true }) test.not_nil(result) test.is_nil(result.memory_recall) @@ -1059,7 +1118,7 @@ local function define_tests() -- Add only one more message (cooldown = 2, so should not trigger) prompt_builder:add_user("New question") - local result = test_agent:step(prompt_builder) + local result = test_agent:step(prompt_builder, { persist_context_notes = true }) test.not_nil(result) test.is_nil(result.memory_recall) -- Should not recall due to cooldown @@ -1077,7 +1136,7 @@ local function define_tests() test.not_nil(result) test.is_nil(result.memory_recall) - test.is_nil(result.memory_prompt) + test.is_nil(result.context_notes) end) it("should include session context in memory recall", function() @@ -1093,6 +1152,7 @@ local function define_tests() -- Pass session context via runtime options local result = test_agent:step(prompt_builder, { + persist_context_notes = true, context = { user_id = "test_user_123", session_id = "session_456" @@ -1233,6 +1293,21 @@ local function define_tests() test.eq(messages[n - 2].role, prompt.ROLE.USER) end) + it("places the conversation tail marker before trailing positional notes", function() + local builder = prompt.new() + builder:add_user("First message") + builder:add_system("The current date and time is Tuesday.", { preserve_position = true }) + builder:add_developer("Recalled memory", { preserve_position = true }) + + local messages = capture(builder) + local n = #messages + test.eq(messages[n].role, prompt.ROLE.DEVELOPER) + test.eq(messages[n - 1].role, prompt.ROLE.SYSTEM) + test.eq(messages[n - 2].role, prompt.ROLE.CACHE_MARKER) + test.eq(messages[n - 2].marker_id, "conversation_tail") + test.eq(messages[n - 3].role, prompt.ROLE.USER) + end) + -- Canonical encoding with sorted keys and cache_control removed, so -- prefixes compare by the bytes a provider hashes for its cache. local function canonical(value) @@ -1350,15 +1425,18 @@ local function define_tests() local step1 = prompt.new() step1:add_user("Plan the release") step1:add_developer("Lifecycle note for step 1") - local _, err1 = runner:step(step1, { context = {} }) + local result1, err1 = runner:step(step1, { context = {}, persist_context_notes = true }) test.is_nil(err1) local step2 = prompt.new() step2:add_user("Plan the release") + for _, note in ipairs(result1.context_notes) do + step2:add_message(note.role, note.content, nil, note.metadata) + end step2:add_function_call("calculator", '{"expression":"2+2"}', "call_1") step2:add_function_result("calculator", "4", "call_1") step2:add_developer("Lifecycle note for step 2") - local _, err2 = runner:step(step2, { context = {} }) + local _, err2 = runner:step(step2, { context = {}, persist_context_notes = true }) test.is_nil(err2) test.eq(#requests, 2) @@ -1374,6 +1452,161 @@ local function define_tests() test.eq(after, before, "prefix through message " .. tostring(prefix.messages)) end end) + + describe("Stable prefix", function() + local function without_markers(messages) + local result = {} + for _, message in ipairs(messages) do + if message.role ~= prompt.ROLE.CACHE_MARKER then + result[#result + 1] = message + end + end + return result + end + + local function recording_llm(requests, replies) + return { + generate = function(messages) + requests[#requests + 1] = messages + return replies and replies[#requests] or { result = "ok", finish_reason = "stop" } + end + } + end + + local function copy_spec(overrides) + local spec = {} + for k, v in pairs(basic_compiled_spec) do spec[k] = v end + local tools = {} + for name, tool in pairs(basic_compiled_spec.tools) do + local copied = {} + for k, v in pairs(tool) do copied[k] = v end + copied.schema = json.decode((json.encode(tool.schema))) + tools[name] = copied + end + spec.tools = tools + for k, v in pairs(overrides or {}) do spec[k] = v end + return spec + end + + it("sends the same system message on every step", function() + local requests = {} + agent._llm = recording_llm(requests) + agent._funcs = { + new = function() + return { + with_context = function(self) return self end, + call = function() return "Workspace: release planning" end + } + end + } + local spec = copy_spec({ + prompt_funcs = { { trait_id = "workspace", func_id = "test:workspace_prompt", context = {} } } + }) + local runner = agent.new(spec) + local builder = prompt.new() + builder:add_user("First") + runner:step(builder, { context = {} }) + builder:add_assistant("Answer") + builder:add_user("Second") + runner:step(builder, { context = {} }) + + test.eq(#requests, 2) + test.eq(requests[1][1].role, prompt.ROLE.SYSTEM) + test.eq(requests[2][1].content[1].text, requests[1][1].content[1].text) + end) + + it("keeps each request a prefix of the next once the host persists context notes", function() + agent._contract = { + get = function() + return { + open = function() + return { + recall = function(_, options) + return { memories = { { id = "m" .. #options.recent_actions, content = "Fact" } } } + end + } + end + } + end + } + local requests = {} + agent._llm = recording_llm(requests, { + { + result = "", + tool_calls = { { id = "call_1", name = "calculator", arguments = { expression = "2+2" } } }, + metadata = { thinking_blocks = { { type = "thinking", thinking = "plan", signature = "sig" } } }, + finish_reason = "tool_call" + }, + { result = "Done", finish_reason = "stop" }, + }) + local runner = agent.new(copy_spec({ + memory_contract = { implementation_id = "test:memory", options = { min_conversation_length = 1 } } + })) + + local history = prompt.new() + history:add_user("Plan the release") + local first = assert(runner:step(history, { context = {}, persist_context_notes = true })) + test.eq(#first.context_notes, 1) + for _, note in ipairs(first.context_notes) do + history:add_message(note.role, note.content, nil, note.metadata) + end + history:add_function_call("calculator", '{"expression":"2+2"}', "call_1") + history.messages[#history.messages].metadata = { thinking_blocks = first.metadata.thinking_blocks } + history:add_function_result("calculator", "4", "call_1") + local second = assert(runner:step(history, { context = {}, persist_context_notes = true })) + test.eq(#second.context_notes, 1) + + local before = without_markers(requests[1]) + local after = without_markers(requests[2]) + test.gt(#after, #before) + for index, message in ipairs(before) do + test.eq(canonical(after[index]), canonical(message), "message " .. tostring(index)) + end + end) + + it("returns a prefix fingerprint that is stable across steps", function() + local requests = {} + agent._llm = recording_llm(requests) + local runner = agent.new(copy_spec()) + local builder = prompt.new() + builder:add_user("First") + local first = assert(runner:step(builder)) + builder:add_assistant("Answer") + builder:add_user("Second") + local second = assert(runner:step(builder)) + local rebuilt = assert(agent.new(copy_spec()):step(builder)) + + test.eq(type(first.prefix_fingerprint), "string") + test.eq(#first.prefix_fingerprint, 64) + test.eq(second.prefix_fingerprint, first.prefix_fingerprint) + test.eq(rebuilt.prefix_fingerprint, first.prefix_fingerprint) + end) + + it("changes the prefix fingerprint when system, tools or model change", function() + agent._llm = recording_llm({}) + local function fingerprint(spec) + local builder = prompt.new() + builder:add_user("Hello") + return assert(agent.new(spec):step(builder)).prefix_fingerprint + end + local base = fingerprint(copy_spec()) + local fewer_tools = copy_spec() + fewer_tools.tools.weather = nil + local described = copy_spec() + described.tools.calculator.description = "Evaluate arithmetic" + local reshaped = copy_spec() + reshaped.tools.calculator.schema.required = { "expression" } + for label, spec in pairs({ + system = copy_spec({ prompt = "You are a different agent." }), + model = copy_spec({ model = "gpt-4o" }), + tool_removed = fewer_tools, + tool_description = described, + tool_schema = reshaped, + }) do + test.neq(fingerprint(spec), base, label) + end + end) + end) end) describe("Tool Schema Usage", function() diff --git a/src/agent/test/discovery/traits_test.lua b/src/agent/test/discovery/traits_test.lua index 641510f7..d1f97966 100644 --- a/src/agent/test/discovery/traits_test.lua +++ b/src/agent/test/discovery/traits_test.lua @@ -150,12 +150,11 @@ local function define_tests() meta = { type = "agent.trait", name = "Time Aware", - comment = "Adds current time context to agent for time-aware responses with cache stability." + comment = "Declares agent_options.turn_context.time so the host adds the current time to each user turn" }, data = { build_func_id = "wippy.agent.traits:init_time_aware", context = { - time_interval = 15, timezone = "UTC" } } @@ -482,7 +481,6 @@ local function define_tests() test.is_nil(trait.prompt_func_id) -- No runtime prompt function test.is_nil(trait.step_func_id) -- No step function test.eq(trait.prompt, "") -- No static prompt - test.eq(trait.context.time_interval, 15) test.eq(trait.context.timezone, "UTC") end) diff --git a/src/agent/test/traits/init_time_aware_test.lua b/src/agent/test/traits/init_time_aware_test.lua index f7b619ab..fb9a9deb 100644 --- a/src/agent/test/traits/init_time_aware_test.lua +++ b/src/agent/test/traits/init_time_aware_test.lua @@ -1,74 +1,26 @@ local init_time_aware = require("init_time_aware") +local funcs = require("funcs") local function define_tests() describe("Init Time Aware Trait", function() - it("should return prompt with time info", function() - local result = init_time_aware.execute("base prompt") + it("contributes turn context options and no prompt text", function() + local result: any = init_time_aware.execute("base prompt") test.not_nil(result) - test.not_nil(result.prompt) - test.eq(type(result.prompt), "string") - test.is_true(#result.prompt > 0) + test.is_nil(result.prompt) + test.eq(result.agent_options.turn_context.time.timezone, "UTC") + test.is_nil(result.agent_options.turn_context.time.format) end) - it("should contain date components in prompt", function() - local time = require("time") - local now = time.now() - local result = init_time_aware.execute("base prompt") + it("carries the configured timezone and format", function() + local result, err = funcs.new() + :with_context({ timezone = "Europe/Berlin", format = "2006-01-02 15:04" }) + :call("wippy.agent.traits:init_time_aware", "base prompt") - local year = tostring(now:year()) - test.not_nil(result.prompt:find(year)) - - local months = { - "January", "February", "March", "April", "May", "June", - "July", "August", "September", "October", "November", "December" - } - local month_name = months[now:month()] - test.not_nil(result.prompt:find(month_name)) - - local day = tostring(now:day()) - test.not_nil(result.prompt:find(day)) - - local hour_str = string.format("%02d", now:hour()) - test.not_nil(result.prompt:find(hour_str)) - end) - - it("should round minutes to interval", function() - local time = require("time") - local now = time.now() - local minute = now:minute() - local rounded = math.floor(minute / 15) * 15 - - local result = init_time_aware.execute("base prompt") - - local rounded_str = string.format("%02d", rounded) - test.not_nil(result.prompt:find(rounded_str)) - end) - - it("should contain weekday name", function() - local time = require("time") - local now = time.now() - local weekdays = { - "Sunday", "Monday", "Tuesday", "Wednesday", - "Thursday", "Friday", "Saturday" - } - local weekday_name = weekdays[now:weekday() + 1] - - local result = init_time_aware.execute("base prompt") - - test.not_nil(result.prompt:find(weekday_name)) - end) - - it("should contain timezone in prompt", function() - local result = init_time_aware.execute("base prompt") - - test.not_nil(result.prompt:find("UTC")) - end) - - it("should contain temporal context instruction", function() - local result = init_time_aware.execute("base prompt") - - test.not_nil(result.prompt:find("temporal context")) + test.is_nil(err) + test.is_nil(result.prompt) + test.eq(result.agent_options.turn_context.time.timezone, "Europe/Berlin") + test.eq(result.agent_options.turn_context.time.format, "2006-01-02 15:04") end) end) end diff --git a/src/agent/test/turn_context_test.lua b/src/agent/test/turn_context_test.lua new file mode 100644 index 00000000..9119fd45 --- /dev/null +++ b/src/agent/test/turn_context_test.lua @@ -0,0 +1,53 @@ +local turn_context = require("turn_context") +local time = require("time") + +local function define_tests() + describe("Turn context", function() + local now = time.date(2026, 11, 3, 9, 44, 7, 0, time.utc) + + local function options(time_config) + return { turn_context = { time = time_config } } + end + + it("renders the exact current minute in the configured timezone", function() + local text, err = turn_context.render(options({ timezone = "UTC" }), now) + test.is_nil(err) + test.eq(text, "The current date and time is Tuesday, November 3, 2026 at 09:44 UTC.") + end) + + it("converts the time into a named timezone", function() + local text, err = turn_context.render(options({ timezone = "America/New_York" }), now) + test.is_nil(err) + test.eq(text, "The current date and time is Tuesday, November 3, 2026 at 04:44 America/New_York.") + end) + + it("formats the time with a configured layout", function() + local text, err = turn_context.render( + options({ timezone = "America/New_York", format = "2006-01-02 15:04 MST" }), now) + test.is_nil(err) + test.eq(text, "The current date and time is 2026-11-03 04:44 EST.") + end) + + it("renders identical text for the same instant", function() + local first = turn_context.render(options({ timezone = "UTC" }), now) + local second = turn_context.render(options({ timezone = "UTC" }), time.unix(now:unix(), 0)) + test.eq(second, first) + end) + + it("renders nothing when the agent has no turn context", function() + for _, agent_options in ipairs({ {}, { turn_context = {} } }) do + local text, err = turn_context.render(agent_options, now) + test.is_nil(text) + test.is_nil(err) + end + end) + + it("rejects an unknown timezone", function() + local text, err = turn_context.render(options({ timezone = "Mars/Olympus" }), now) + test.is_nil(text) + test.contains(tostring(err), "Mars/Olympus") + end) + end) +end + +return require("test").run_cases(define_tests) From 7b2138b6ad8ac9b9c5dfcb410bbb6b6b81f46d1a Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 16:36:21 -0500 Subject: [PATCH 07/37] fix(llm): render Claude developer notes independently of cache markers Developer messages render as their own text blocks after the user turn they follow (or open a user turn after an assistant turn) and never edit an existing block, so the same history renders identically wherever the rolling cache marker sits. The Claude golden body changes only for its developer messages. bind_thinking becomes drop_stale_thinking: empty block lists carry nothing, every block up to the last stale message is removed, and an assistant turn (consecutive assistant and function call messages, including function_call content parts) with an unanswered call refuses removal and leaves the history unchanged. --- src/llm/src/README.md | 16 ++-- src/llm/src/claude/mapper.lua | 117 +++++++++----------------- src/llm/src/prompt.lua | 87 ++++++++++++++----- src/llm/test/claude/generate_test.lua | 6 +- src/llm/test/claude/mapper_test.lua | 92 +++++++++++++++++--- src/llm/test/prompt_test.lua | 105 ++++++++++++++++++----- src/llm/test/request_golden_test.lua | 9 +- 7 files changed, 283 insertions(+), 149 deletions(-) diff --git a/src/llm/src/README.md b/src/llm/src/README.md index 1e6ef39d..0479ad07 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -218,7 +218,9 @@ A system or developer message with `metadata.preserve_position = true` is a posi - Google sends it as a user turn; `systemInstruction` holds only ordinary system messages. - OpenAI Responses keeps it in `input`; OpenAI-compatible keeps it as a `system` message in place. -Per-turn context (current time, recalled memory, lifecycle notes) belongs in positional notes written once and kept in the history. Changing the top-level system prompt, the tools or any earlier message changes the provider prefix, which restarts the prompt cache and invalidates every later thinking block. +A positional note travels with the user turn it follows. Slicing a history (compaction, windowing) must keep or drop the note together with that turn: a slice that keeps the note but drops its user turn is `invalid_request` on Claude and Bedrock. + +Per-turn and per-step context (current time, recalled memory, lifecycle notes) belongs in positional notes written once and persisted in the history. A note that is sent once and then removed, rebuilt or moved, and any change to the top-level system prompt, the tools or an earlier message, changes the provider prefix: the prompt cache restarts from that point and every later thinking block is invalidated (the provider rejects or drops it), not only cached. ```lua builder:add_user("What is on my calendar?") @@ -226,19 +228,21 @@ builder:add_system("The current date and time is Tuesday, November 3, 2026 at 09 { preserve_position = true }) ``` -Bedrock renders ordinary developer messages as user text at their position, like Claude. +Ordinary developer messages render the same way on Claude and Bedrock: as their own text blocks after the content of the user turn they follow, or as a new user turn after an assistant turn. Existing blocks are never edited, so the rendering does not depend on where cache markers sit. + +Bedrock sends developer messages as user text instead of `system` blocks. A Bedrock conversation stored before this change therefore renders a different prefix once after upgrading: its cache restarts and its earlier thinking blocks no longer match. The provider rejects such blocks unless the request lets it drop them; hosts that keep thinking blocks across the upgrade strip them with `drop_stale_thinking`. ### Thinking Blocks -Assistant and function call messages carry provider reasoning in `metadata.thinking_blocks`. Each block is bound to the prefix before it and chained to the previous block, so blocks can be dropped only from the front of the history. `bind_thinking` removes the blocks of every message the predicate reports as not current and keeps the rest: +Assistant and function call messages carry provider reasoning in `metadata.thinking_blocks`. Each block is bound to the prefix before it and chained to the previous block, so blocks can be dropped only from the front of the history. `drop_stale_thinking` takes a predicate that reports whether a message's blocks are still valid and removes the blocks of every message up to and including the last one the predicate rejects: ```lua -local bound, err = builder:bind_thinking(function(msg) - return msg.metadata.turn_id == current_turn_id +local builder, err = builder:drop_stale_thinking(function(msg) + return msg.metadata.prefix_fingerprint == current_fingerprint end) ``` -It fails, leaving the history unchanged, when a non-current message follows a current one, or when a non-current function call has no result in the history yet. Messages without thinking blocks are not consulted. +An empty `thinking_blocks` list counts as no blocks; messages without blocks are not consulted. An assistant turn is the run of consecutive assistant and function call messages. If a turn whose blocks would be removed still has a function call without a result (a function call message or a `function_call` content part), the call fails and leaves the history unchanged: the provider needs those blocks to continue the tool round. ### Builder Operations diff --git a/src/llm/src/claude/mapper.lua b/src/llm/src/claude/mapper.lua index 31ab69ee..dcb0ff10 100644 --- a/src/llm/src/claude/mapper.lua +++ b/src/llm/src/claude/mapper.lua @@ -280,6 +280,26 @@ local function process_content_array(content) return content end +local function note_blocks(content): {any} + local blocks: {any} = {} + if type(content) == "string" then + if content ~= "" then + table.insert(blocks, { type = "text", text = content }) + end + elseif type(content) == "table" then + for _, part in ipairs(content) do + if part.type == "text" then + if part.text and part.text ~= "" then + table.insert(blocks, { type = "text", text = part.text }) + end + else + table.insert(blocks, convert_image_content(part)) + end + end + end + return blocks +end + local function normalize_tool_arguments(raw_arguments) local arguments = raw_arguments @@ -350,21 +370,27 @@ function mapper.map_messages(contract_messages) local in_system_phase = true for i, msg in ipairs(contract_messages) do - if prompt.is_positional_note(msg) then - -- A positional note joins the user turn it follows, as blocks after - -- that turn's content, so the system prompt and every earlier - -- block stay byte-identical when the note changes. + if prompt.is_positional_note(msg) or msg.role == prompt.ROLE.DEVELOPER then + -- Notes join the user turn they follow as blocks after that turn's + -- content. Existing blocks are never edited, so the rendering does + -- not depend on where cache markers sit and every earlier block + -- stays byte-identical across requests. in_system_phase = false + local blocks = note_blocks(msg.content) local last_msg = claude_messages[#claude_messages] if not last_msg or last_msg.role ~= "user" then - return nil, string.format("positional %s note at message %d must follow a user message", msg.role, i) - end - local note_content: any = process_content_array(msg.content) - if type(note_content) == "string" then - table.insert(last_msg.content, { type = "text", text = note_content }) - elseif type(note_content) == "table" then - for _, part in ipairs(note_content) do - table.insert(last_msg.content, part) + if prompt.is_positional_note(msg) then + return nil, string.format("positional %s note at message %d must follow a user message", msg.role, i) + end + -- Developer guidance after an assistant turn opens a user turn: + -- the request never ends on an assistant turn (a prefill that + -- current models reject) and model output is never amended. + if #blocks > 0 then + table.insert(claude_messages, { role = "user", content = blocks }) + end + else + for _, block in ipairs(blocks) do + table.insert(last_msg.content, block) end end elseif msg.role == prompt.ROLE.SYSTEM then @@ -393,73 +419,6 @@ function mapper.map_messages(contract_messages) table.insert(message_cache_positions, position) end end - elseif msg.role == prompt.ROLE.DEVELOPER then - in_system_phase = false - local dev_text = type(msg.content) == "string" and msg.content or - (type(msg.content) == "table" and msg.content[1] and msg.content[1].text) or "" - - if dev_text ~= "" then - -- Developer guidance merges only into a preceding plain user - -- message. Anywhere else it opens a new user message: gluing - -- it into an assistant message would attribute text the model - -- never produced and leave the request ending on an assistant - -- turn, which the API treats as a prefill and current models - -- reject; after a tool_result it stays a separate message. - local should_create_new_message = false - - if #claude_messages == 0 then - should_create_new_message = true - else - local last_msg = claude_messages[#claude_messages] - if last_msg.role ~= "user" then - should_create_new_message = true - elseif last_msg.content and last_msg.content[1] and - last_msg.content[1].type == "tool_result" then - should_create_new_message = true - else - -- A later developer note must not alter a prefix already - -- selected for caching by a preceding marker. - for _, position in ipairs(message_cache_positions) do - if position.message == #claude_messages then - should_create_new_message = true - break - end - end - end - end - - if should_create_new_message then - -- Create new user message for developer content - table.insert(claude_messages, { - role = "user", - content = { - { - type = "text", - text = dev_text - } - } - }) - else - -- Try to merge into previous message (existing logic) - local last_msg = claude_messages[#claude_messages] - local merged = false - for j = #last_msg.content, 1, -1 do - local part = last_msg.content[j] :: any - if part.type == "text" then - part.text = part.text .. - "\n" .. dev_text .. "" - merged = true - break - end - end - if not merged then - table.insert(last_msg.content, { - type = "text", - text = "" .. dev_text .. "" - }) - end - end - end elseif msg.role == prompt.ROLE.FUNCTION_RESULT then in_system_phase = false local result_text = type(msg.content) == "string" and msg.content or diff --git a/src/llm/src/prompt.lua b/src/llm/src/prompt.lua index 33308fb6..053b6325 100644 --- a/src/llm/src/prompt.lua +++ b/src/llm/src/prompt.lua @@ -52,7 +52,7 @@ type PromptBuilder = { add_function_call: (self: any, function_name: string, arguments: string, function_call_id: string?, options: FunctionCallOptions?) -> PromptBuilder, add_function_result: (self: any, name: string, content: any, function_call_id: string?, options: FunctionResultOptions?) -> PromptBuilder, add_cache_marker: (self: any, marker_id: string?) -> PromptBuilder, - bind_thinking: (self: any, is_current: (msg: Message) -> boolean) -> (PromptBuilder?, string?), + drop_stale_thinking: (self: any, is_current: (msg: Message) -> boolean) -> (PromptBuilder?, string?), get_messages: (self: any) -> {Message}, build: (self: any) -> {messages: {Message}}, clone: (self: any) -> PromptBuilder, @@ -374,9 +374,11 @@ function prompt.new(messages: {Message}?) -- Remove thinking blocks the current request can no longer replay. -- Providers bind each block to the exact prefix before it and chain it to - -- the previous block, so only a leading run of blocks may be dropped, and - -- an assistant turn whose tool calls still await results keeps its blocks. - builder.bind_thinking = function(self: any, is_current: (msg: Message) -> boolean): (PromptBuilder?, string?) + -- the previous block, so blocks may be dropped only from the front: every + -- block up to the last stale one goes. An assistant turn (a run of + -- assistant and function call messages) whose calls still await results + -- keeps its blocks, because the provider needs them to continue the round. + builder.drop_stale_thinking = function(self: any, is_current: (msg: Message) -> boolean): (PromptBuilder?, string?) local messages: {Message} = self.messages :: {Message} local resolved: {[string]: boolean} = {} for _, msg in ipairs(messages) do @@ -385,27 +387,70 @@ function prompt.new(messages: {Message}?) end end - local stale: {Message} = {} - local kept_index: integer? = nil + local function in_turn(msg: Message): boolean + return msg.role == prompt.ROLE.ASSISTANT or msg.role == prompt.ROLE.FUNCTION_CALL + end + + local function carries(msg: Message): boolean + if not in_turn(msg) or type(msg.metadata) ~= "table" then return false end + local blocks = msg.metadata.thinking_blocks + return type(blocks) == "table" and #blocks > 0 + end + + -- First unanswered call of each assistant turn, keyed by message index. + local open_call: {[integer]: string} = {} + local turn_start: integer? = nil for index, msg in ipairs(messages) do - local carries = (msg.role == prompt.ROLE.ASSISTANT or msg.role == prompt.ROLE.FUNCTION_CALL) - and msg.metadata ~= nil and msg.metadata.thinking_blocks ~= nil - if carries then - if is_current(msg) then - kept_index = kept_index or index - elseif kept_index then - return nil, string.format( - "thinking blocks can be removed only as a leading run: message %d is not current but follows current message %d", - index, kept_index) + if in_turn(msg) then + turn_start = turn_start or index + else + turn_start = nil + end + if turn_start then + local ids: {any} = {} + if msg.role == prompt.ROLE.FUNCTION_CALL then + table.insert(ids, msg.function_call and msg.function_call.id or false) else - local call_id = msg.function_call and msg.function_call.id - if msg.role == prompt.ROLE.FUNCTION_CALL and not (call_id and resolved[call_id]) then - return nil, string.format( - "thinking blocks of open tool call %s must stay until its result is present", - tostring(call_id)) + for _, part in ipairs(msg.content or {}) do + if (part :: any).type == "function_call" then + table.insert(ids, (part :: any).id or false) + end end - table.insert(stale, msg) end + for _, id in ipairs(ids) do + if not open_call[turn_start] and not (id and resolved[id]) then + open_call[turn_start] = id and tostring(id) or "without id" + end + end + end + end + + local last_stale: integer? = nil + for index, msg in ipairs(messages) do + if carries(msg) and not is_current(msg) then + last_stale = index + end + end + if not last_stale then + return self, nil + end + + local stale: {Message} = {} + turn_start = nil + for index = 1, last_stale do + local msg = messages[index] + if in_turn(msg) then + turn_start = turn_start or index + else + turn_start = nil + end + if carries(msg) then + local open = turn_start and open_call[turn_start] + if open then + return nil, string.format( + "thinking blocks of message %d must stay: its turn has open tool call %s", index, open) + end + table.insert(stale, msg) end end diff --git a/src/llm/test/claude/generate_test.lua b/src/llm/test/claude/generate_test.lua index 0b6bd079..79661e40 100644 --- a/src/llm/test/claude/generate_test.lua +++ b/src/llm/test/claude/generate_test.lua @@ -408,9 +408,9 @@ local function define_tests() test.eq(#payload.messages, 1) local user_msg = payload.messages[1] test.eq(user_msg.role, "user") - test.contains(user_msg.content[1].text, "Hello") - test.contains(user_msg.content[1].text, - "Be concise") + test.eq(user_msg.content[1].text, "Hello") + test.eq(user_msg.content[2].type, "text") + test.eq(user_msg.content[2].text, "Be concise") return { content = { { type = "text", text = "Hi!" } }, diff --git a/src/llm/test/claude/mapper_test.lua b/src/llm/test/claude/mapper_test.lua index 091b3a02..89900b4a 100644 --- a/src/llm/test/claude/mapper_test.lua +++ b/src/llm/test/claude/mapper_test.lua @@ -88,6 +88,25 @@ local function define_tests() test.is_nil(content[2].cache_control) end) + it("keeps a persisted positional note block unchanged when a developer note follows", function() + local function build(with_developer) + local messages = { + { role = "user", content = text("Question") }, + note("system", "The current date and time is Tuesday."), + } + if with_developer then + table.insert(messages, { role = "developer", content = text("Attached: report.pdf") }) + end + return assert(mapper.map_messages(messages)) + end + local before, after = build(false), build(true) + test.eq(#after.messages, 1) + test.eq(after.messages[1].content[2].text, before.messages[1].content[2].text) + test.eq(after.messages[1].content[2].text, "The current date and time is Tuesday.") + test.eq(#after.messages[1].content, 3) + test.eq(after.messages[1].content[3].text, "Attached: report.pdf") + end) + it("rejects a positional note that has no preceding user message", function() for _, history in ipairs({ { { role = "system", content = text("Base") }, note("system", "Orphan") }, @@ -374,13 +393,14 @@ local function define_tests() { role = prompt.ROLE.DEVELOPER, content = "Be concise" } } - local result = mapper.map_messages(contract_messages) + local result = assert(mapper.map_messages(contract_messages)) test.eq(#result.messages, 1) local user_msg = result.messages[1] test.eq(user_msg.role, "user") - local first_content = user_msg.content[1] :: any - test.contains(tostring(first_content.text), "Hello") - test.contains(tostring(first_content.text), "Be concise") + test.eq(#user_msg.content, 2) + test.eq(tostring((user_msg.content[1] :: any).text), "Hello") + test.eq((user_msg.content[2] :: any).type, "text") + test.eq(tostring((user_msg.content[2] :: any).text), "Be concise") end) it("should append developer messages to an image-only previous message", function() @@ -398,14 +418,9 @@ local function define_tests() local result = mapper.map_messages(contract_messages) local last_msg = result.messages[#result.messages] :: any - local found = false - for _, part in ipairs(last_msg.content) do - if (part :: any).type == "text" and - tostring((part :: any).text):find("Be concise", 1, true) then - found = true - end - end - test.is_true(found, "developer instruction must survive an image-only previous message") + test.eq(#last_msg.content, 2, "developer instruction must survive an image-only previous message") + test.eq((last_msg.content[2] :: any).type, "text") + test.eq(tostring((last_msg.content[2] :: any).text), "Be concise") -- The image itself must still be there. test.eq((last_msg.content[1] :: any).type, "image") @@ -615,8 +630,57 @@ local function define_tests() test.eq(second.messages[1].content[1].text, first.messages[1].content[1].text) test.not_nil((second.messages[1].content[1] :: any).cache_control) - test.eq(second.messages[2].role, "user") - test.eq(second.messages[2].content[1].text, "Changing memory recall") + test.eq(#second.messages, 1) + test.eq(second.messages[1].content[2].text, "Changing memory recall") + test.is_nil((second.messages[1].content[2] :: any).cache_control) + end) + + it("renders a developer note identically wherever the rolling cache marker sits", function() + local function history(marker_on_question) + local messages = { + { role = "system", content = {{ type = "text", text = "Base" }} }, + { role = "user", content = {{ type = "text", text = "Summarize the file" }} }, + } + if marker_on_question then + table.insert(messages, { role = "cache_marker", marker_id = "conversation_tail" }) + end + table.insert(messages, { role = "developer", content = {{ type = "text", text = "Attached: report.pdf" }} }) + table.insert(messages, { role = "function_call", content = {}, + function_call = { id = "call_1", name = "read", arguments = "{}" } }) + table.insert(messages, { role = "function_result", name = "read", function_call_id = "call_1", + content = {{ type = "text", text = "contents" }} }) + if not marker_on_question then + table.insert(messages, { role = "cache_marker", marker_id = "conversation_tail" }) + end + return assert(mapper.map_messages(messages)) + end + local function strip(value) + if type(value) ~= "table" then return value end + local copy = {} + for k, v in pairs(value) do + if k ~= "cache_control" then copy[k] = strip(v) end + end + return copy + end + local earlier, later = history(true), history(false) + test.not_nil((earlier.messages[1].content[1] :: any).cache_control) + test.not_nil((later.messages[3].content[1] :: any).cache_control) + test.eq(#later.messages, #earlier.messages) + for m, message in ipairs(earlier.messages) do + local other = later.messages[m] + test.eq(other.role, message.role) + test.eq(#other.content, #message.content) + for b, block in ipairs(message.content) do + local left, right = strip(block), strip(other.content[b]) + local keys = {} + for key in pairs(left) do keys[key] = true end + for key in pairs(right) do keys[key] = true end + for key in pairs(keys) do + test.eq((json.encode(right[key])), (json.encode(left[key])), "message " .. m .. " block " .. b .. " " .. key) + end + end + end + test.eq(earlier.messages[1].content[2].text, "Attached: report.pdf") end) it("should handle empty messages gracefully", function() diff --git a/src/llm/test/prompt_test.lua b/src/llm/test/prompt_test.lua index 420c554d..6b36f02f 100644 --- a/src/llm/test/prompt_test.lua +++ b/src/llm/test/prompt_test.lua @@ -37,7 +37,7 @@ local function define_tests() test.eq(messages[4].metadata.thinking_blocks[1].signature, "sig") end) - describe("bind_thinking", function() + describe("drop_stale_thinking", function() local function block(name) return { { type = "thinking", thinking = name, signature = "sig-" .. name } } end @@ -56,56 +56,117 @@ local function define_tests() return builder end + local function thinking_of(messages, index) + local blocks = messages[index].metadata.thinking_blocks + return blocks and blocks[1] and blocks[1].thinking or nil + end + it("strips thinking blocks from a leading run of older messages", function() local builder = history() - local bound, err = builder:bind_thinking(function(msg) return msg.metadata.turn >= 2 end) + local dropped, err = builder:drop_stale_thinking(function(msg) return msg.metadata.turn >= 2 end) test.is_nil(err) - test.eq(bound, builder) + test.eq(dropped, builder) local messages = builder:get_messages() test.is_nil(messages[2].metadata.thinking_blocks) test.eq(messages[2].metadata.turn, 1) test.eq(messages[2].function_call.id, "call_1") test.is_nil(messages[4].metadata.thinking_blocks) test.eq(messages[4].content[1].text, "answer one") - test.eq(messages[6].metadata.thinking_blocks[1].thinking, "c") + test.eq(thinking_of(messages, 6), "c") end) it("keeps every block when all messages are current", function() local builder = history() - local _, err = builder:bind_thinking(function() return true end) + local _, err = builder:drop_stale_thinking(function() return true end) test.is_nil(err) local messages = builder:get_messages() - test.eq(messages[2].metadata.thinking_blocks[1].thinking, "a") - test.eq(messages[4].metadata.thinking_blocks[1].thinking, "b") - test.eq(messages[6].metadata.thinking_blocks[1].thinking, "c") + test.eq(thinking_of(messages, 2), "a") + test.eq(thinking_of(messages, 4), "b") + test.eq(thinking_of(messages, 6), "c") end) - it("rejects removal from the middle and leaves the history unchanged", function() + it("strips every block up to the last stale message", function() local builder = history() - local bound, err = builder:bind_thinking(function(msg) + local _, err = builder:drop_stale_thinking(function(msg) return msg.metadata.thinking_blocks[1].thinking ~= "b" end) - test.is_nil(bound) - test.contains(tostring(err), "leading run") + test.is_nil(err) local messages = builder:get_messages() - test.eq(messages[2].metadata.thinking_blocks[1].thinking, "a") - test.eq(messages[4].metadata.thinking_blocks[1].thinking, "b") - test.eq(messages[6].metadata.thinking_blocks[1].thinking, "c") + test.is_nil(messages[2].metadata.thinking_blocks) + test.is_nil(messages[4].metadata.thinking_blocks) + test.eq(thinking_of(messages, 6), "c") end) - it("rejects a boundary that splits an open tool round", function() + it("treats an empty thinking block list as carrying nothing", function() + local builder = prompt.new() + builder:add_user("first") + builder:add_message(prompt.ROLE.ASSISTANT, { prompt.text("old") }, nil, { thinking_blocks = {} }) + builder:add_user("second") + builder:add_function_call("lookup", "{}", "call_open") + builder.messages[#builder.messages].metadata = { thinking_blocks = {} } + local consulted = 0 + local dropped, err = builder:drop_stale_thinking(function() + consulted = consulted + 1 + return false + end) + test.is_nil(err) + test.eq(dropped, builder) + test.eq(consulted, 0) + end) + + it("refuses to strip a function call whose result is not present", function() local builder = prompt.new() builder:add_user("first") builder:add_message(prompt.ROLE.ASSISTANT, { prompt.text("old") }, nil, { thinking_blocks = block("a") }) builder:add_user("second") builder:add_function_call("lookup", "{}", "call_open") builder.messages[#builder.messages].metadata = { thinking_blocks = block("b") } - local bound, err = builder:bind_thinking(function() return false end) - test.is_nil(bound) + local dropped, err = builder:drop_stale_thinking(function() return false end) + test.is_nil(dropped) test.contains(tostring(err), "call_open") local messages = builder:get_messages() - test.eq(messages[2].metadata.thinking_blocks[1].thinking, "a") - test.eq(messages[4].metadata.thinking_blocks[1].thinking, "b") + test.eq(thinking_of(messages, 2), "a") + test.eq(thinking_of(messages, 4), "b") + end) + + it("refuses to strip an assistant message whose turn still has an open call", function() + local builder = prompt.new() + builder:add_user("first") + builder:add_message(prompt.ROLE.ASSISTANT, { prompt.text("Looking it up") }, nil, + { thinking_blocks = block("a") }) + builder:add_function_call("lookup", "{}", "call_done") + builder:add_function_call("fetch", "{}", "call_open") + builder:add_function_result("lookup", "found", "call_done") + local dropped, err = builder:drop_stale_thinking(function() return false end) + test.is_nil(dropped) + test.contains(tostring(err), "call_open") + test.eq(thinking_of(builder:get_messages(), 2), "a") + end) + + it("refuses to strip an assistant message with an unanswered function call part", function() + local builder = prompt.new() + builder:add_user("first") + builder:add_message(prompt.ROLE.ASSISTANT, { + prompt.text("Looking it up"), + { type = "function_call", id = "call_part", name = "lookup", arguments = "{}" }, + }, nil, { thinking_blocks = block("a") }) + local dropped, err = builder:drop_stale_thinking(function() return false end) + test.is_nil(dropped) + test.contains(tostring(err), "call_part") + end) + + it("strips an assistant turn once every call in it has a result", function() + local builder = prompt.new() + builder:add_user("first") + builder:add_message(prompt.ROLE.ASSISTANT, { prompt.text("Looking it up") }, nil, + { thinking_blocks = block("a") }) + builder:add_function_call("lookup", "{}", "call_1") + builder:add_function_call("fetch", "{}", "call_2") + builder:add_function_result("lookup", "found", "call_1") + builder:add_function_result("fetch", "fetched", "call_2") + local _, err = builder:drop_stale_thinking(function() return false end) + test.is_nil(err) + test.is_nil(builder:get_messages()[2].metadata.thinking_blocks) end) it("is a no-op when no message carries thinking blocks", function() @@ -114,12 +175,12 @@ local function define_tests() builder:add_assistant("hi", { turn = 1 }) builder:add_function_call("lookup", "{}", "call_1") local consulted = false - local bound, err = builder:bind_thinking(function() + local dropped, err = builder:drop_stale_thinking(function() consulted = true return false end) test.is_nil(err) - test.eq(bound, builder) + test.eq(dropped, builder) test.is_false(consulted) local messages = builder:get_messages() test.eq(#messages, 3) diff --git a/src/llm/test/request_golden_test.lua b/src/llm/test/request_golden_test.lua index 693160d4..040523d8 100644 --- a/src/llm/test/request_golden_test.lua +++ b/src/llm/test/request_golden_test.lua @@ -5,11 +5,12 @@ local openai_mapper = require("openai_mapper") local compat_mapper = require("compat_mapper") local codec = require("codec") --- Message-derived request body sections, recorded from the drivers before --- positional notes existed. Prompts without positional notes must keep these --- bytes exactly; any difference changes the provider prefix and its cache. +-- Message-derived request body sections for prompts without positional notes. +-- They are recorded from the drivers before positional notes existed, except +-- Claude developer messages, which render as separate text blocks in the user +-- turn they follow. Any difference changes the provider prefix and its cache. local GOLDEN = { - claude = "{\"messages\":[{\"content\":[{\"text\":\"Hello\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"signature\":\"sig-greet\",\"thinking\":\"greet\",\"type\":\"thinking\"},{\"text\":\"Hi\",\"type\":\"text\"}],\"role\":\"assistant\"},{\"content\":[{\"text\":\"Look it up\\n\\u003cdeveloper-instruction\\u003ePrefer cached data\\u003c/developer-instruction\\u003e\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"signature\":\"sig-lookup\",\"thinking\":\"lookup\",\"type\":\"thinking\"},{\"id\":\"call_1\",\"input\":{\"q\":\"x\"},\"name\":\"lookup\",\"type\":\"tool_use\"}],\"role\":\"assistant\"},{\"content\":[{\"cache_control\":{\"type\":\"ephemeral\"},\"content\":\"found\",\"tool_use_id\":\"call_1\",\"type\":\"tool_result\"}],\"role\":\"user\"},{\"content\":[{\"text\":\"Answer briefly\",\"type\":\"text\"}],\"role\":\"user\"}],\"system\":[{\"cache_control\":{\"type\":\"ephemeral\"},\"text\":\"You are concise.\",\"type\":\"text\"},{\"text\":\"Mode: review\",\"type\":\"text\"}]}", + claude = "{\"messages\":[{\"content\":[{\"text\":\"Hello\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"signature\":\"sig-greet\",\"thinking\":\"greet\",\"type\":\"thinking\"},{\"text\":\"Hi\",\"type\":\"text\"}],\"role\":\"assistant\"},{\"content\":[{\"text\":\"Look it up\",\"type\":\"text\"},{\"text\":\"Prefer cached data\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"signature\":\"sig-lookup\",\"thinking\":\"lookup\",\"type\":\"thinking\"},{\"id\":\"call_1\",\"input\":{\"q\":\"x\"},\"name\":\"lookup\",\"type\":\"tool_use\"}],\"role\":\"assistant\"},{\"content\":[{\"cache_control\":{\"type\":\"ephemeral\"},\"content\":\"found\",\"tool_use_id\":\"call_1\",\"type\":\"tool_result\"},{\"text\":\"Answer briefly\",\"type\":\"text\"}],\"role\":\"user\"}],\"system\":[{\"cache_control\":{\"type\":\"ephemeral\"},\"text\":\"You are concise.\",\"type\":\"text\"},{\"text\":\"Mode: review\",\"type\":\"text\"}]}", bedrock = "{\"messages\":[{\"content\":[{\"text\":\"Hello\"}],\"role\":\"user\"},{\"content\":[{\"reasoningContent\":{\"reasoningText\":{\"signature\":\"sig-greet\",\"text\":\"greet\"}}},{\"text\":\"Hi\"}],\"role\":\"assistant\"},{\"content\":[{\"text\":\"Look it up\"}],\"role\":\"user\"},{\"content\":[{\"reasoningContent\":{\"reasoningText\":{\"signature\":\"sig-lookup\",\"text\":\"lookup\"}}},{\"toolUse\":{\"input\":{\"q\":\"x\"},\"name\":\"lookup\",\"toolUseId\":\"call_1\"}}],\"role\":\"assistant\"},{\"content\":[{\"toolResult\":{\"content\":[{\"text\":\"found\"}],\"status\":\"success\",\"toolUseId\":\"call_1\"}},{\"cachePoint\":{\"type\":\"default\"}}],\"role\":\"user\"}],\"system\":[{\"text\":\"You are concise.\"},{\"cachePoint\":{\"type\":\"default\"}},{\"text\":\"Mode: review\"}]}", google = "{\"contents\":[{\"parts\":[{\"text\":\"Hello\"}],\"role\":\"user\"},{\"parts\":[{\"text\":\"Hi\"}],\"role\":\"model\"},{\"parts\":[{\"text\":\"Look it up\"}],\"role\":\"user\"},{\"parts\":[{\"text\":\"Prefer cached data\"}],\"role\":\"user\"},{\"parts\":[{\"functionCall\":{\"args\":{\"q\":\"x\"},\"name\":\"lookup\"}}],\"role\":\"model\"},{\"parts\":[{\"functionResponse\":{\"name\":\"lookup\",\"response\":{\"content\":\"found\"}}}],\"role\":\"user\"},{\"parts\":[{\"text\":\"Answer briefly\"}],\"role\":\"user\"}],\"systemInstruction\":[{\"text\":\"You are concise.\"},{\"text\":\"Mode: review\"}]}", openai = "{\"input\":[{\"content\":[{\"text\":\"Hello\",\"type\":\"input_text\"}],\"role\":\"user\",\"type\":\"message\"},{\"content\":[{\"text\":\"Hi\",\"type\":\"output_text\"}],\"role\":\"assistant\",\"type\":\"message\"},{\"content\":[{\"text\":\"Look it up\",\"type\":\"input_text\"}],\"role\":\"user\",\"type\":\"message\"},{\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\",\"call_id\":\"call_1\",\"name\":\"lookup\",\"type\":\"function_call\"},{\"call_id\":\"call_1\",\"output\":\"found\",\"type\":\"function_call_output\"}],\"instructions\":\"You are concise.\\n\\nPrefer cached data\\n\\nMode: review\\n\\nAnswer briefly\"}", From 89db370c2597f2640c1cdd7ce67e21b70feaacfa Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 16:36:21 -0500 Subject: [PATCH 08/37] fix(agent): fingerprint the sent prefix and fail on prompt and trait errors prefix_fingerprint hashes the system text as drivers send it (compiled prompt plus hoisted conversation system messages) and each tool's name, description, schema and native definition; registry ids, registry metadata and the model are excluded. A failing prompt function fails the step with its kind and retryability. A failing trait build function fails compilation; time_aware rejects time_interval naming turn_context. turn_context.render validates timezone and format and keeps the guidance sentence. A test guards the llm >= 0.5.13 dependency on prompt.is_positional_note. --- src/agent/src/README.md | 50 +++++++++--- src/agent/src/agent.lua | 63 +++++++++++---- src/agent/src/compiler/compiler.lua | 39 +++++++-- src/agent/src/traits/init_time_aware.lua | 8 +- src/agent/src/turn_context.lua | 19 +++-- src/agent/test/agent_test.lua | 81 ++++++++++++++++--- src/agent/test/compiler/compiler_test.lua | 9 +-- .../test/traits/init_time_aware_test.lua | 10 +++ src/agent/test/turn_context_test.lua | 23 +++++- 9 files changed, 243 insertions(+), 59 deletions(-) diff --git a/src/agent/src/README.md b/src/agent/src/README.md index c67cd6cd..4bba34d7 100644 --- a/src/agent/src/README.md +++ b/src/agent/src/README.md @@ -237,7 +237,11 @@ conversation independently of the agent's prompt and tool execution `context`. `stop_check` runs after prompt functions and memory recall, immediately before provider generation. When it returns true, `step` returns `{ stopped = true }` -without a provider request. +without a provider request; that result carries no `prefix_fingerprint` and no +`context_notes`, since nothing was sent. + +A prompt function that fails fails the step with an error naming the function, +keeping its error kind and retryability; the request is not sent without its text. `cancel` is a channel the host closes to cancel the in-flight provider call (see `llm.generate`). The step then returns `{ stopped = true, result = = 0.5.13 is required: prompt.is_positional_note is missing") + end + end) + local function without_markers(messages) local result = {} for _, message in ipairs(messages) do @@ -1488,6 +1494,34 @@ local function define_tests() return spec end + it("fails the step when a prompt function fails", function() + local requests = {} + agent._llm = recording_llm(requests) + agent._funcs = { + new = function() + return { + with_context = function(self) return self end, + call = function() + return nil, errors.new({ message = "workspace store unavailable", + kind = errors.UNAVAILABLE, retryable = true }) + end + } + end + } + local runner = agent.new(copy_spec({ + prompt_funcs = { { trait_id = "workspace", func_id = "test:workspace_prompt", context = {} } } + })) + local builder = prompt.new() + builder:add_user("First") + local result, err = runner:step(builder, { context = {} }) + test.is_nil(result) + test.contains(tostring(err), "test:workspace_prompt") + test.contains(tostring(err), "workspace store unavailable") + test.eq(err:kind(), errors.UNAVAILABLE) + test.is_true(err:retryable()) + test.eq(#requests, 0) + end) + it("sends the same system message on every step", function() local requests = {} agent._llm = recording_llm(requests) @@ -1582,30 +1616,59 @@ local function define_tests() test.eq(rebuilt.prefix_fingerprint, first.prefix_fingerprint) end) - it("changes the prefix fingerprint when system, tools or model change", function() + local function fingerprint_of(spec, extend) agent._llm = recording_llm({}) - local function fingerprint(spec) - local builder = prompt.new() - builder:add_user("Hello") - return assert(agent.new(spec):step(builder)).prefix_fingerprint - end - local base = fingerprint(copy_spec()) + local builder = prompt.new() + builder:add_user("Hello") + if extend then extend(builder) end + return assert(agent.new(spec):step(builder)).prefix_fingerprint + end + + it("changes the prefix fingerprint when the sent system text or tools change", function() + local base = fingerprint_of(copy_spec()) local fewer_tools = copy_spec() fewer_tools.tools.weather = nil local described = copy_spec() described.tools.calculator.description = "Evaluate arithmetic" local reshaped = copy_spec() reshaped.tools.calculator.schema.required = { "expression" } + local renamed = copy_spec() + renamed.tools.calculate = renamed.tools.calculator + renamed.tools.calculator = nil + local native = copy_spec() + native.tools.calculator.meta = { claude = { type = "code_execution_20250522" } } for label, spec in pairs({ system = copy_spec({ prompt = "You are a different agent." }), - model = copy_spec({ model = "gpt-4o" }), tool_removed = fewer_tools, + tool_renamed = renamed, tool_description = described, tool_schema = reshaped, + tool_native_type = native, }) do - test.neq(fingerprint(spec), base, label) + test.neq(fingerprint_of(spec), base, label) end end) + + it("includes conversation system messages that drivers hoist into the system prompt", function() + local base = fingerprint_of(copy_spec()) + local hoisted = fingerprint_of(copy_spec(), function(builder) + builder:add_system("Mode: review") + end) + local positional = fingerprint_of(copy_spec(), function(builder) + builder:add_system("The current date and time is Tuesday.", { preserve_position = true }) + end) + test.neq(hoisted, base) + test.eq(positional, base) + end) + + it("keeps the prefix fingerprint across model and registry metadata changes", function() + local base = fingerprint_of(copy_spec()) + local commented = copy_spec() + commented.tools.calculator.meta = { comment = "Renamed in the registry", tags = { "math" }, title = "Calc" } + commented.tools.calculator.registry_id = "other:calculator" + test.eq(fingerprint_of(commented), base) + test.eq(fingerprint_of(copy_spec({ model = "gpt-4o" })), base) + end) end) end) diff --git a/src/agent/test/compiler/compiler_test.lua b/src/agent/test/compiler/compiler_test.lua index 3664f161..dbb0f8f4 100644 --- a/src/agent/test/compiler/compiler_test.lua +++ b/src/agent/test/compiler/compiler_test.lua @@ -2139,7 +2139,7 @@ local function define_tests() test.is_false(has_delegate_tools) end) - it("should handle failing trait build methods gracefully", function() + it("should fail compilation when a trait build method fails", function() local failing_trait = { id = "failing_trait", name = "Failing Trait", @@ -2164,10 +2164,9 @@ local function define_tests() local compiled_spec, err = compiler.compile(spec_with_failing_trait) - test.is_nil(err) - test.not_nil(compiled_spec) - test.contains(compiled_spec.prompt, "Test prompt") - test.contains(compiled_spec.prompt, "Base prompt") + test.is_nil(compiled_spec) + test.contains(tostring(err), "failing_trait") + test.contains(tostring(err), "Init method failed") end) it("should use custom context merger in unified structure", function() diff --git a/src/agent/test/traits/init_time_aware_test.lua b/src/agent/test/traits/init_time_aware_test.lua index fb9a9deb..ac2b8f37 100644 --- a/src/agent/test/traits/init_time_aware_test.lua +++ b/src/agent/test/traits/init_time_aware_test.lua @@ -22,6 +22,16 @@ local function define_tests() test.eq(result.agent_options.turn_context.time.timezone, "Europe/Berlin") test.eq(result.agent_options.turn_context.time.format, "2006-01-02 15:04") end) + + it("rejects the removed time_interval setting and names its replacement", function() + local result, err = funcs.new() + :with_context({ timezone = "UTC", time_interval = 15 }) + :call("wippy.agent.traits:init_time_aware", "base prompt") + + test.is_nil(result) + test.contains(tostring(err), "time_interval") + test.contains(tostring(err), "turn_context") + end) end) end diff --git a/src/agent/test/turn_context_test.lua b/src/agent/test/turn_context_test.lua index 9119fd45..3a6ca687 100644 --- a/src/agent/test/turn_context_test.lua +++ b/src/agent/test/turn_context_test.lua @@ -12,20 +12,23 @@ local function define_tests() it("renders the exact current minute in the configured timezone", function() local text, err = turn_context.render(options({ timezone = "UTC" }), now) test.is_nil(err) - test.eq(text, "The current date and time is Tuesday, November 3, 2026 at 09:44 UTC.") + test.eq(text, "The current date and time is Tuesday, November 3, 2026 at 09:44 UTC. " + .. "Use this temporal context when relevant to your responses.") end) it("converts the time into a named timezone", function() local text, err = turn_context.render(options({ timezone = "America/New_York" }), now) test.is_nil(err) - test.eq(text, "The current date and time is Tuesday, November 3, 2026 at 04:44 America/New_York.") + test.eq(text, "The current date and time is Tuesday, November 3, 2026 at 04:44 America/New_York. " + .. "Use this temporal context when relevant to your responses.") end) it("formats the time with a configured layout", function() local text, err = turn_context.render( options({ timezone = "America/New_York", format = "2006-01-02 15:04 MST" }), now) test.is_nil(err) - test.eq(text, "The current date and time is 2026-11-03 04:44 EST.") + test.eq(text, "The current date and time is 2026-11-03 04:44 EST. " + .. "Use this temporal context when relevant to your responses.") end) it("renders identical text for the same instant", function() @@ -42,6 +45,20 @@ local function define_tests() end end) + it("rejects an empty timezone", function() + local text, err = turn_context.render(options({ timezone = "" }), now) + test.is_nil(text) + test.contains(tostring(err), "timezone") + end) + + it("rejects a format that is not a non-empty string", function() + for _, format in ipairs({ "", 42, true }) do + local text, err = turn_context.render(options({ timezone = "UTC", format = format }), now) + test.is_nil(text) + test.contains(tostring(err), "format") + end + end) + it("rejects an unknown timezone", function() local text, err = turn_context.render(options({ timezone = "Mars/Olympus" }), now) test.is_nil(text) From 8e9efe9187d9fe8887d2a1846d94055303ab8c53 Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 16:53:26 -0500 Subject: [PATCH 09/37] fix(llm): reject non-list Bedrock binding fields from options.request A bound Bedrock request whose additionalModelRequestFields.anthropic_beta or additionalModelResponseFieldPaths is not a list fails with invalid_request before HTTP. README notes the uncaptured fallback array, the between_tools effort limits and further input_transformations values. --- src/llm/src/README.md | 4 +- src/llm/src/bedrock/generate.lua | 5 ++- src/llm/src/bedrock/mapper.lua | 16 ++++++-- src/llm/src/bedrock/structured_output.lua | 5 ++- src/llm/test/bedrock/generate_test.lua | 21 +++++++++++ .../test/bedrock/structured_output_test.lua | 37 ++++++++++++++++++- src/llm/test/claude/generate_test.lua | 2 - 7 files changed, 79 insertions(+), 11 deletions(-) diff --git a/src/llm/src/README.md b/src/llm/src/README.md index bb100baf..8006525d 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -137,9 +137,9 @@ When a request replays at least one thinking block and sends a thinking object o A request without a thinking object (`adaptive` or `budget` at zero or absent effort, or `thinking = "none"`) carries no binding: `block_binding` exists only inside an adaptive or enabled thinking object, and the driver never turns thinking on to carry it, since on several models an omitted thinking object means no thinking. Such a request replays its blocks unbound, so keep that conversation append-only. `always` is the form that carries the binding at zero effort. -Claude Sonnet 5.5 `thinking.type = "between_tools"` rejects `block_binding`, so a between_tools request has no drop_block recovery. Routes that must survive agent handoffs or history edits use `thinking: always` with a low default effort (`options.thinking_effort: 10`, the `low` level; 20 through 50 map to `medium`); keep between_tools for append-only routes. between_tools also rejects the `xhigh` level that thinking_effort 80 and above maps to, and per-message effort changes. +Claude Sonnet 5.5 `thinking.type = "between_tools"` rejects `block_binding`, so a between_tools request has no drop_block recovery. Routes that must survive agent handoffs or history edits use `thinking: always` with a low default effort (`options.thinking_effort: 10`, the `low` level; 20 through 50 map to `medium`); keep between_tools for append-only routes. between_tools also rejects the `xhigh` and `max` levels (thinking_effort 80 and above) and per-message effort changes. -Responses to a bound request report each dropped block in `metadata.input_transformations`, for example `{ type = "thinking_dropped", path = "messages.1.content.0", reason = "prefix_binding_mismatch" }`. `prefix_binding_mismatch` means the conversation before the block changed; `model_binding_mismatch` means the conversation moved to a model that cannot read the block. An empty array means nothing was dropped. Streaming Claude responses report the array from `message_start`; Bedrock reports it from `messageStop`. Ignore entry types and reasons you do not recognize. +Responses to a bound request report each dropped block in `metadata.input_transformations`, for example `{ type = "thinking_dropped", path = "messages.1.content.0", reason = "prefix_binding_mismatch" }`. `prefix_binding_mismatch` means the conversation before the block changed; `model_binding_mismatch` means the conversation moved to a model that cannot read the block. Other values exist and later checks add more, for example the reason `organization_binding_mismatch` (a block from another organization) and the entry type `thinking_mismatch_allowed` (a block that failed the check but still reached the model); ignore entry types and reasons you do not recognize. An empty array means nothing was dropped. Streaming Claude responses report the array from `message_start`. After a mid-stream server-side fallback the API sends the final array again in the closing `message_delta`, in a location that is not documented; the driver does not capture that final array yet, so such a stream reports the array from `message_start`. Bedrock reports it from `messageStop`. Model switches and fallbacks need no client-side handling: the API drops blocks the target model cannot read, so the drivers pass every block back unchanged. Prefix changes are different: keeping the system prompt, tools and earlier messages append-only is what keeps earlier reasoning in use. diff --git a/src/llm/src/bedrock/generate.lua b/src/llm/src/bedrock/generate.lua index 816f952e..317d166d 100644 --- a/src/llm/src/bedrock/generate.lua +++ b/src/llm/src/bedrock/generate.lua @@ -159,7 +159,10 @@ function generate_handler.handler(contract_args) } -- The binding depends on the thinking object as sent, so it is the last change to the payload. - generate_handler._mapper.bind_thinking(converse_payload) + local bind_err = generate_handler._mapper.bind_thinking(converse_payload) + if bind_err then + return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(bind_err):build() + end if contract_args.stream and contract_args.stream.reply_to then local stream_response, stream_err = generate_handler._client.converse_stream( diff --git a/src/llm/src/bedrock/mapper.lua b/src/llm/src/bedrock/mapper.lua index a3d5f89d..8c5cab16 100644 --- a/src/llm/src/bedrock/mapper.lua +++ b/src/llm/src/bedrock/mapper.lua @@ -553,23 +553,31 @@ end -- Binds the reasoning a Converse payload replays to drop_block. Bedrock takes the beta in -- the anthropic_beta model field and returns input_transformations only when the payload --- names it in additionalModelResponseFieldPaths. -function mapper.bind_thinking(payload) +-- names it in additionalModelResponseFieldPaths. Returns an error when either list a +-- bound payload extends is not a list. +function mapper.bind_thinking(payload): string? local fields = payload.additionalModelRequestFields if not fields then - return + return nil end if replays_reasoning(payload.messages) then thinking.bind_replayed_blocks(fields.thinking) end if not thinking.is_bound(fields.thinking) then - return + return nil + end + if fields.anthropic_beta ~= nil and type(fields.anthropic_beta) ~= "table" then + return "additionalModelRequestFields.anthropic_beta must be a list of beta names" + end + if payload.additionalModelResponseFieldPaths ~= nil and type(payload.additionalModelResponseFieldPaths) ~= "table" then + return "additionalModelResponseFieldPaths must be a list of field paths" end fields.anthropic_beta = transport.merge_unique({ fields.anthropic_beta or {}, { thinking.BINDING_BETA } }) payload.additionalModelResponseFieldPaths = transport.merge_unique({ payload.additionalModelResponseFieldPaths or {}, { "/input_transformations" } }) + return nil end -- Try to parse a text block as an embedded tool call. diff --git a/src/llm/src/bedrock/structured_output.lua b/src/llm/src/bedrock/structured_output.lua index b26f487e..b30bf3d5 100644 --- a/src/llm/src/bedrock/structured_output.lua +++ b/src/llm/src/bedrock/structured_output.lua @@ -144,7 +144,10 @@ function structured_output_handler.handler(contract_args) end -- The binding depends on the thinking object as sent, so it is the last change to the payload. - structured_output_handler._mapper.bind_thinking(converse_payload) + local bind_err = structured_output_handler._mapper.bind_thinking(converse_payload) + if bind_err then + return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(bind_err):build() + end local response, request_err = structured_output_handler._client.converse( contract_args.model, diff --git a/src/llm/test/bedrock/generate_test.lua b/src/llm/test/bedrock/generate_test.lua index 7f9e54ef..65efee01 100644 --- a/src/llm/test/bedrock/generate_test.lua +++ b/src/llm/test/bedrock/generate_test.lua @@ -402,6 +402,27 @@ local function define_tests() test.is_nil(captured.payload.additionalModelResponseFieldPaths) end) + it("rejects a request list field the binding extends when it is not a list", function() + for _, request in ipairs({ + { additionalModelRequestFields = { anthropic_beta = "context-1m-2025-08-07" } }, + { additionalModelResponseFieldPaths = "/stop_sequence" } + }) do + local captured = capture() + local response, err = generate_handler.handler({ + model = CLAUDE_MODEL, + messages = history_with_thinking({ SIGNED_THINKING }), + accepts = { thinking = "adaptive" }, + options = { thinking_effort = 50, request = request } + }) + + test.is_nil(response) + test.eq(err:kind(), "Invalid") + test.contains(err:message(), next(request) == "additionalModelRequestFields" + and "additionalModelRequestFields.anthropic_beta" or "additionalModelResponseFieldPaths") + test.is_nil(captured.payload) + end + end) + it("sends no binding when the history replays no reasoning", function() local captured = capture() diff --git a/src/llm/test/bedrock/structured_output_test.lua b/src/llm/test/bedrock/structured_output_test.lua index 1aa9091d..e5a09433 100644 --- a/src/llm/test/bedrock/structured_output_test.lua +++ b/src/llm/test/bedrock/structured_output_test.lua @@ -242,8 +242,43 @@ local function define_tests() test.eq(captured_payload.additionalModelResponseFieldPaths[1], "/input_transformations") end) - it("should send forced tool_choice in payload", function() + it("rejects a request anthropic_beta the binding extends when it is not a list", function() + local called = false + structured_output_handler._client = { + converse = function() + called = true + return nil, { status_code = 500, message = "unreachable" } + end + } + local response, err = structured_output_handler.handler({ + model = "us.anthropic.claude-opus-5-5-v1:0", + messages = { + { role = "user", content = { { type = "text", text = "Plan" } } }, + { + role = "assistant", + content = { { type = "text", text = "Planned." } }, + metadata = { thinking_blocks = { { type = "thinking", thinking = "t", signature = "sig" } } } + }, + { role = "user", content = { { type = "text", text = "Answer" } } } + }, + schema = { + type = "object", + properties = { x = { type = "number" } }, + required = { "x" }, + additionalProperties = false + }, + options = { thinking_effort = 50, request = { additionalModelRequestFields = { anthropic_beta = "context-1m-2025-08-07" } } }, + accepts = { thinking = "adaptive" } + }) + + test.is_nil(response) + test.eq(err:kind(), "Invalid") + test.contains(err:message(), "additionalModelRequestFields.anthropic_beta") + test.is_false(called) + end) + + it("should send forced tool_choice in payload", function() local captured_payload = nil structured_output_handler._client = { converse = function(model_id, payload, options) diff --git a/src/llm/test/claude/generate_test.lua b/src/llm/test/claude/generate_test.lua index 36662339..93a8a1c5 100644 --- a/src/llm/test/claude/generate_test.lua +++ b/src/llm/test/claude/generate_test.lua @@ -1622,7 +1622,6 @@ local function define_tests() end) it("sends no binding when the history replays no thinking blocks", function() - local captured = {} generate_handler._client = capturing_client(captured) @@ -1748,7 +1747,6 @@ local function define_tests() end) it("replays stored blocks in the API shape of their type", function() - local captured = {} generate_handler._client = capturing_client(captured) From 471e748efb0545013c87dbf77fc5a21165acd3d3 Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 17:02:17 -0500 Subject: [PATCH 10/37] fix(llm): open a user turn for positional notes that follow no user turn Claude and Bedrock render a positional note at the start of the conversation or after an assistant or tool call turn as its own user turn instead of rejecting it, matching persisted checkpoint-resume and stop-boundary notes. drop_stale_thinking replaces stripped messages with copies so caller-held message tables keep their blocks. A Bedrock developer golden case pins the branch rendering. --- src/llm/src/README.md | 6 +- src/llm/src/bedrock/mapper.lua | 21 ++--- src/llm/src/claude/mapper.lua | 9 +-- src/llm/src/prompt.lua | 17 ++-- src/llm/test/bedrock/generate_test.lua | 21 ----- src/llm/test/bedrock/mapper_test.lua | 104 +++++++++++++++++++++++-- src/llm/test/claude/generate_test.lua | 21 ----- src/llm/test/claude/mapper_test.lua | 104 +++++++++++++++++++++++-- src/llm/test/prompt_test.lua | 17 ++++ src/llm/test/request_golden_test.lua | 8 +- 10 files changed, 239 insertions(+), 89 deletions(-) diff --git a/src/llm/src/README.md b/src/llm/src/README.md index 0479ad07..da12c9dd 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -214,11 +214,11 @@ builder:add_user("Question") A system or developer message with `metadata.preserve_position = true` is a positional note. It stays at its conversation position on every driver and is never merged with neighbouring messages: -- Claude and Bedrock append it as text blocks after the content of the user turn it follows (including a turn of tool results). The top-level system prompt never contains it. A positional note that does not follow a user turn fails with `invalid_request` before dispatch. +- Claude and Bedrock append it as text blocks after the content of the user turn it follows (including a turn of tool results). At the start of the conversation or after an assistant turn it opens a new user turn; it is never merged into earlier blocks. The top-level system prompt never contains it. - Google sends it as a user turn; `systemInstruction` holds only ordinary system messages. - OpenAI Responses keeps it in `input`; OpenAI-compatible keeps it as a `system` message in place. -A positional note travels with the user turn it follows. Slicing a history (compaction, windowing) must keep or drop the note together with that turn: a slice that keeps the note but drops its user turn is `invalid_request` on Claude and Bedrock. +A positional note travels with the message it follows. Slicing a history (compaction, windowing) keeps or drops the note together with that message; a slice that keeps the note but drops its user turn renders the note as a user turn of its own, which changes the prefix from that point. Per-turn and per-step context (current time, recalled memory, lifecycle notes) belongs in positional notes written once and persisted in the history. A note that is sent once and then removed, rebuilt or moved, and any change to the top-level system prompt, the tools or an earlier message, changes the provider prefix: the prompt cache restarts from that point and every later thinking block is invalidated (the provider rejects or drops it), not only cached. @@ -228,7 +228,7 @@ builder:add_system("The current date and time is Tuesday, November 3, 2026 at 09 { preserve_position = true }) ``` -Ordinary developer messages render the same way on Claude and Bedrock: as their own text blocks after the content of the user turn they follow, or as a new user turn after an assistant turn. Existing blocks are never edited, so the rendering does not depend on where cache markers sit. +Ordinary developer messages render the same way on Claude and Bedrock: as their own text blocks after the content of the user turn they follow, or as a new user turn at the start or after an assistant turn. Existing blocks are never edited, so the rendering does not depend on where cache markers sit. The text is sent as written: Claude no longer wraps it in `` tags. A host that needs the model to tell its guidance apart from the user's prose marks the note text itself. Bedrock sends developer messages as user text instead of `system` blocks. A Bedrock conversation stored before this change therefore renders a different prefix once after upgrading: its cache restarts and its earlier thinking blocks no longer match. The provider rejects such blocks unless the request lets it drop them; hosts that keep thinking blocks across the upgrade strip them with `drop_stale_thinking`. diff --git a/src/llm/src/bedrock/mapper.lua b/src/llm/src/bedrock/mapper.lua index dcf7330d..9584786b 100644 --- a/src/llm/src/bedrock/mapper.lua +++ b/src/llm/src/bedrock/mapper.lua @@ -237,19 +237,8 @@ function mapper.map_messages(contract_messages) local system_cache_positions = {} local message_cache_positions = {} - for i, msg in ipairs(contract_messages) do - if prompt.is_positional_note(msg) then - -- A positional note joins the user turn it follows, as blocks after - -- that turn's content, so the system blocks and every earlier - -- block stay byte-identical when the note changes. - local last_msg = converse_messages[#converse_messages] - if not last_msg or last_msg.role ~= "user" then - return nil, string.format("positional %s note at message %d must follow a user message", msg.role, i) - end - for _, block in ipairs(user_content_blocks(msg.content)) do - table.insert(last_msg.content, block) - end - elseif msg.role == prompt.ROLE.SYSTEM then + for _, msg in ipairs(contract_messages) do + if msg.role == prompt.ROLE.SYSTEM and not prompt.is_positional_note(msg) then if type(msg.content) == "string" then table.insert(system_blocks, { text = msg.content }) elseif type(msg.content) == "table" then @@ -377,8 +366,10 @@ function mapper.map_messages(contract_messages) }) end else - -- User messages and developer guidance, which renders as user text - -- so per-step guidance never rewrites the system blocks. + -- User messages, developer guidance and positional notes. Notes + -- render as user text, so they never rewrite the system blocks; + -- consolidation appends them as blocks after the user turn they + -- follow, or they open a user turn after an assistant turn. local content_blocks = user_content_blocks(msg.content) if #content_blocks > 0 then table.insert(converse_messages, { diff --git a/src/llm/src/claude/mapper.lua b/src/llm/src/claude/mapper.lua index dcb0ff10..3c1e9b85 100644 --- a/src/llm/src/claude/mapper.lua +++ b/src/llm/src/claude/mapper.lua @@ -379,12 +379,9 @@ function mapper.map_messages(contract_messages) local blocks = note_blocks(msg.content) local last_msg = claude_messages[#claude_messages] if not last_msg or last_msg.role ~= "user" then - if prompt.is_positional_note(msg) then - return nil, string.format("positional %s note at message %d must follow a user message", msg.role, i) - end - -- Developer guidance after an assistant turn opens a user turn: - -- the request never ends on an assistant turn (a prefill that - -- current models reject) and model output is never amended. + -- A note at the start or after an assistant turn opens a user + -- turn: the request never ends on an assistant turn (a prefill + -- that current models reject) and model output is never amended. if #blocks > 0 then table.insert(claude_messages, { role = "user", content = blocks }) end diff --git a/src/llm/src/prompt.lua b/src/llm/src/prompt.lua index 053b6325..7542b26e 100644 --- a/src/llm/src/prompt.lua +++ b/src/llm/src/prompt.lua @@ -435,7 +435,7 @@ function prompt.new(messages: {Message}?) return self, nil end - local stale: {Message} = {} + local stale: {integer} = {} turn_start = nil for index = 1, last_stale do local msg = messages[index] @@ -450,16 +450,23 @@ function prompt.new(messages: {Message}?) return nil, string.format( "thinking blocks of message %d must stay: its turn has open tool call %s", index, open) end - table.insert(stale, msg) + table.insert(stale, index) end end - for _, msg in ipairs(stale) do + -- Stripped messages are replaced by copies, so message tables the + -- caller shares with the builder keep their blocks. + for _, index in ipairs(stale) do + local copy: Message = {} :: Message + for key, value in pairs(messages[index] :: table) do + (copy :: any)[key] = value + end local metadata = {} - for key, value in pairs(msg.metadata :: table) do + for key, value in pairs(messages[index].metadata :: table) do if key ~= "thinking_blocks" then metadata[key] = value end end - msg.metadata = metadata + copy.metadata = metadata + messages[index] = copy end return self, nil end diff --git a/src/llm/test/bedrock/generate_test.lua b/src/llm/test/bedrock/generate_test.lua index 2b5bd3fe..a6857e97 100644 --- a/src/llm/test/bedrock/generate_test.lua +++ b/src/llm/test/bedrock/generate_test.lua @@ -9,27 +9,6 @@ local function define_tests() end) describe("Contract Validation", function() - it("should reject a positional note without a preceding user message before dispatch", function() - generate_handler._client = { - ENDPOINTS = { MESSAGES = "/v1/messages" }, - request = function() error("request must not be dispatched") end, - converse = function() error("request must not be dispatched") end, - } - local response, err = generate_handler.handler({ - model = "us.anthropic.claude-haiku-4-5-20251001-v1:0", - messages = { - { role = "user", content = { { type = "text", text = "Question" } } }, - { role = "assistant", content = { { type = "text", text = "Answer" } } }, - { role = "system", content = { { type = "text", text = "Note" } }, - metadata = { preserve_position = true } }, - } - }) - test.is_nil(response) - test.not_nil(err) - test.eq(err:kind(), "Invalid") - test.contains(err:message(), "positional") - end) - it("should require model parameter", function() local response, err = generate_handler.handler({ messages = { { role = "user", content = { { type = "text", text = "Test" } } } } diff --git a/src/llm/test/bedrock/mapper_test.lua b/src/llm/test/bedrock/mapper_test.lua index 02896f43..e382284a 100644 --- a/src/llm/test/bedrock/mapper_test.lua +++ b/src/llm/test/bedrock/mapper_test.lua @@ -82,18 +82,106 @@ local function define_tests() test.eq(content[3].text, "Recalled memory") end) - it("rejects a positional note that has no preceding user message", function() - for _, history in ipairs({ - { { role = "system", content = text("Base") }, note("system", "Orphan") }, - { + local function same(a, b) + if type(a) ~= "table" or type(b) ~= "table" then return a == b end + for k, v in pairs(a) do + if k ~= "cache_control" and k ~= "cachePoint" and not same(v, b[k]) then return false end + end + for k in pairs(b) do + if k ~= "cache_control" and k ~= "cachePoint" and a[k] == nil then return false end + end + return true + end + + -- Every block of the earlier request appears unchanged, in order, in the later one. + local function assert_prefix(earlier, later, label) + test.is_true(same(earlier.system, later.system), label .. ": system") + for m, message in ipairs(earlier.messages) do + local other = later.messages[m] + test.not_nil(other, label .. ": message " .. m) + test.eq(other.role, message.role, label .. ": role " .. m) + local b2 = 0 + for _, block in ipairs(message.content) do + if not block.cachePoint then + b2 = b2 + 1 + while other.content[b2] and other.content[b2].cachePoint do b2 = b2 + 1 end + test.is_true(same(block, other.content[b2]), label .. ": message " .. m .. " block " .. b2) + end + end + end + end + + local function shapes() + return { + first = { + { role = "system", content = text("Base") }, + note("developer", "Resumed from checkpoint"), + { role = "user", content = text("Continue") }, + }, + after_assistant = { { role = "user", content = text("Question") }, { role = "assistant", content = text("Answer") }, - note("developer", "After the answer"), + note("developer", "Stopped by the user"), }, - }) do + after_call = { + { role = "user", content = text("Question") }, + { role = "function_call", content = {}, function_call = { id = "call_1", name = "lookup", arguments = "{}" } }, + note("system", "Interrupted"), + }, + after_result = { + { role = "user", content = text("Question") }, + { role = "function_call", content = {}, function_call = { id = "call_1", name = "lookup", arguments = "{}" } }, + { role = "function_result", name = "lookup", function_call_id = "call_1", content = text("found") }, + note("system", "Reminder"), + }, + } + end + + it("opens a user turn for a positional note that follows no user turn", function() + for label, history in pairs(shapes()) do local mapped, err = mapper.map_messages(history) - test.is_nil(mapped) - test.contains(tostring(err), "positional") + test.is_nil(err, label) + assert(mapped) + test.is_true(mapped.system == nil or #mapped.system <= 1, label) + local note_text = nil + for _, message in ipairs(history) do + if message.metadata and message.metadata.preserve_position then + note_text = message.content[1].text + end + end + local found = nil + for m, message in ipairs(mapped.messages) do + for b, block in ipairs(message.content) do + if block.text == note_text then + found = { m = m, b = b, role = message.role } + end + end + end + test.not_nil(found, label) + test.eq(found.role, "user", label) + if label ~= "after_result" then + test.eq(found.b, 1, label .. ": the note opens its own turn") + end + end + end) + + it("keeps earlier assistant output unchanged when a positional note follows it", function() + local mapped = assert(mapper.map_messages(shapes().after_assistant)) + test.eq(mapped.messages[2].role, "assistant") + test.eq(#mapped.messages[2].content, 1) + test.eq(mapped.messages[3].role, "user") + test.eq(mapped.messages[3].content[1].text, "Stopped by the user") + end) + + it("renders each positional note shape as a stable prefix of the next request", function() + for label, history in pairs(shapes()) do + local later = {} + for i, message in ipairs(history) do later[i] = message end + table.insert(later, { role = "assistant", content = text("Next answer") }) + table.insert(later, { role = "user", content = text("Next question") }) + local earlier_mapped = assert(mapper.map_messages(history)) + local later_mapped = assert(mapper.map_messages(later)) + assert_prefix(earlier_mapped, later_mapped, label) end end) diff --git a/src/llm/test/claude/generate_test.lua b/src/llm/test/claude/generate_test.lua index 79661e40..4d30980e 100644 --- a/src/llm/test/claude/generate_test.lua +++ b/src/llm/test/claude/generate_test.lua @@ -10,27 +10,6 @@ local function define_tests() end) describe("Contract Validation", function() - it("should reject a positional note without a preceding user message before dispatch", function() - generate_handler._client = { - ENDPOINTS = { MESSAGES = "/v1/messages" }, - request = function() error("request must not be dispatched") end, - converse = function() error("request must not be dispatched") end, - } - local response, err = generate_handler.handler({ - model = "claude-3-5-sonnet-20241022", - messages = { - { role = "user", content = { { type = "text", text = "Question" } } }, - { role = "assistant", content = { { type = "text", text = "Answer" } } }, - { role = "system", content = { { type = "text", text = "Note" } }, - metadata = { preserve_position = true } }, - } - }) - test.is_nil(response) - test.not_nil(err) - test.eq(err:kind(), "Invalid") - test.contains(err:message(), "positional") - end) - it("should require model parameter", function() local response, err = generate_handler.handler({ messages = { { role = "user", content = { { type = "text", text = "Test" } } } } diff --git a/src/llm/test/claude/mapper_test.lua b/src/llm/test/claude/mapper_test.lua index 89900b4a..eddd35ce 100644 --- a/src/llm/test/claude/mapper_test.lua +++ b/src/llm/test/claude/mapper_test.lua @@ -107,18 +107,106 @@ local function define_tests() test.eq(after.messages[1].content[3].text, "Attached: report.pdf") end) - it("rejects a positional note that has no preceding user message", function() - for _, history in ipairs({ - { { role = "system", content = text("Base") }, note("system", "Orphan") }, - { + local function same(a, b) + if type(a) ~= "table" or type(b) ~= "table" then return a == b end + for k, v in pairs(a) do + if k ~= "cache_control" and k ~= "cachePoint" and not same(v, b[k]) then return false end + end + for k in pairs(b) do + if k ~= "cache_control" and k ~= "cachePoint" and a[k] == nil then return false end + end + return true + end + + -- Every block of the earlier request appears unchanged, in order, in the later one. + local function assert_prefix(earlier, later, label) + test.is_true(same(earlier.system, later.system), label .. ": system") + for m, message in ipairs(earlier.messages) do + local other = later.messages[m] + test.not_nil(other, label .. ": message " .. m) + test.eq(other.role, message.role, label .. ": role " .. m) + local b2 = 0 + for _, block in ipairs(message.content) do + if not block.cachePoint then + b2 = b2 + 1 + while other.content[b2] and other.content[b2].cachePoint do b2 = b2 + 1 end + test.is_true(same(block, other.content[b2]), label .. ": message " .. m .. " block " .. b2) + end + end + end + end + + local function shapes() + return { + first = { + { role = "system", content = text("Base") }, + note("developer", "Resumed from checkpoint"), + { role = "user", content = text("Continue") }, + }, + after_assistant = { { role = "user", content = text("Question") }, { role = "assistant", content = text("Answer") }, - note("developer", "After the answer"), + note("developer", "Stopped by the user"), }, - }) do + after_call = { + { role = "user", content = text("Question") }, + { role = "function_call", content = {}, function_call = { id = "call_1", name = "lookup", arguments = "{}" } }, + note("system", "Interrupted"), + }, + after_result = { + { role = "user", content = text("Question") }, + { role = "function_call", content = {}, function_call = { id = "call_1", name = "lookup", arguments = "{}" } }, + { role = "function_result", name = "lookup", function_call_id = "call_1", content = text("found") }, + note("system", "Reminder"), + }, + } + end + + it("opens a user turn for a positional note that follows no user turn", function() + for label, history in pairs(shapes()) do local mapped, err = mapper.map_messages(history) - test.is_nil(mapped) - test.contains(tostring(err), "positional") + test.is_nil(err, label) + assert(mapped) + test.is_true(mapped.system == nil or #mapped.system <= 1, label) + local note_text = nil + for _, message in ipairs(history) do + if message.metadata and message.metadata.preserve_position then + note_text = message.content[1].text + end + end + local found = nil + for m, message in ipairs(mapped.messages) do + for b, block in ipairs(message.content) do + if block.text == note_text then + found = { m = m, b = b, role = message.role } + end + end + end + test.not_nil(found, label) + test.eq(found.role, "user", label) + if label ~= "after_result" then + test.eq(found.b, 1, label .. ": the note opens its own turn") + end + end + end) + + it("keeps earlier assistant output unchanged when a positional note follows it", function() + local mapped = assert(mapper.map_messages(shapes().after_assistant)) + test.eq(mapped.messages[2].role, "assistant") + test.eq(#mapped.messages[2].content, 1) + test.eq(mapped.messages[3].role, "user") + test.eq(mapped.messages[3].content[1].text, "Stopped by the user") + end) + + it("renders each positional note shape as a stable prefix of the next request", function() + for label, history in pairs(shapes()) do + local later = {} + for i, message in ipairs(history) do later[i] = message end + table.insert(later, { role = "assistant", content = text("Next answer") }) + table.insert(later, { role = "user", content = text("Next question") }) + local earlier_mapped = assert(mapper.map_messages(history)) + local later_mapped = assert(mapper.map_messages(later)) + assert_prefix(earlier_mapped, later_mapped, label) end end) end) diff --git a/src/llm/test/prompt_test.lua b/src/llm/test/prompt_test.lua index 6b36f02f..97475976 100644 --- a/src/llm/test/prompt_test.lua +++ b/src/llm/test/prompt_test.lua @@ -169,6 +169,23 @@ local function define_tests() test.is_nil(builder:get_messages()[2].metadata.thinking_blocks) end) + it("leaves the caller's message tables unchanged when it strips", function() + local persisted = { role = prompt.ROLE.ASSISTANT, content = { prompt.text("old") }, + metadata = { thinking_blocks = block("a"), turn = 1 } } + local builder = prompt.new({ + { role = prompt.ROLE.USER, content = { prompt.text("question") } }, + persisted, + { role = prompt.ROLE.USER, content = { prompt.text("again") } }, + }) + local _, err = builder:drop_stale_thinking(function() return false end) + test.is_nil(err) + test.eq(#persisted.metadata.thinking_blocks, 1) + local stripped = builder:get_messages()[2] + test.is_nil(stripped.metadata.thinking_blocks) + test.eq(stripped.metadata.turn, 1) + test.eq(stripped.content[1].text, "old") + end) + it("is a no-op when no message carries thinking blocks", function() local builder = prompt.new() builder:add_user("hello") diff --git a/src/llm/test/request_golden_test.lua b/src/llm/test/request_golden_test.lua index 040523d8..5ed60ccb 100644 --- a/src/llm/test/request_golden_test.lua +++ b/src/llm/test/request_golden_test.lua @@ -8,12 +8,15 @@ local codec = require("codec") -- Message-derived request body sections for prompts without positional notes. -- They are recorded from the drivers before positional notes existed, except -- Claude developer messages, which render as separate text blocks in the user --- turn they follow. Any difference changes the provider prefix and its cache. +-- turn they follow, and bedrock_developer, recorded after Bedrock developer +-- messages moved from system blocks to user text. Any difference changes the +-- provider prefix and its cache. local GOLDEN = { claude = "{\"messages\":[{\"content\":[{\"text\":\"Hello\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"signature\":\"sig-greet\",\"thinking\":\"greet\",\"type\":\"thinking\"},{\"text\":\"Hi\",\"type\":\"text\"}],\"role\":\"assistant\"},{\"content\":[{\"text\":\"Look it up\",\"type\":\"text\"},{\"text\":\"Prefer cached data\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"signature\":\"sig-lookup\",\"thinking\":\"lookup\",\"type\":\"thinking\"},{\"id\":\"call_1\",\"input\":{\"q\":\"x\"},\"name\":\"lookup\",\"type\":\"tool_use\"}],\"role\":\"assistant\"},{\"content\":[{\"cache_control\":{\"type\":\"ephemeral\"},\"content\":\"found\",\"tool_use_id\":\"call_1\",\"type\":\"tool_result\"},{\"text\":\"Answer briefly\",\"type\":\"text\"}],\"role\":\"user\"}],\"system\":[{\"cache_control\":{\"type\":\"ephemeral\"},\"text\":\"You are concise.\",\"type\":\"text\"},{\"text\":\"Mode: review\",\"type\":\"text\"}]}", bedrock = "{\"messages\":[{\"content\":[{\"text\":\"Hello\"}],\"role\":\"user\"},{\"content\":[{\"reasoningContent\":{\"reasoningText\":{\"signature\":\"sig-greet\",\"text\":\"greet\"}}},{\"text\":\"Hi\"}],\"role\":\"assistant\"},{\"content\":[{\"text\":\"Look it up\"}],\"role\":\"user\"},{\"content\":[{\"reasoningContent\":{\"reasoningText\":{\"signature\":\"sig-lookup\",\"text\":\"lookup\"}}},{\"toolUse\":{\"input\":{\"q\":\"x\"},\"name\":\"lookup\",\"toolUseId\":\"call_1\"}}],\"role\":\"assistant\"},{\"content\":[{\"toolResult\":{\"content\":[{\"text\":\"found\"}],\"status\":\"success\",\"toolUseId\":\"call_1\"}},{\"cachePoint\":{\"type\":\"default\"}}],\"role\":\"user\"}],\"system\":[{\"text\":\"You are concise.\"},{\"cachePoint\":{\"type\":\"default\"}},{\"text\":\"Mode: review\"}]}", google = "{\"contents\":[{\"parts\":[{\"text\":\"Hello\"}],\"role\":\"user\"},{\"parts\":[{\"text\":\"Hi\"}],\"role\":\"model\"},{\"parts\":[{\"text\":\"Look it up\"}],\"role\":\"user\"},{\"parts\":[{\"text\":\"Prefer cached data\"}],\"role\":\"user\"},{\"parts\":[{\"functionCall\":{\"args\":{\"q\":\"x\"},\"name\":\"lookup\"}}],\"role\":\"model\"},{\"parts\":[{\"functionResponse\":{\"name\":\"lookup\",\"response\":{\"content\":\"found\"}}}],\"role\":\"user\"},{\"parts\":[{\"text\":\"Answer briefly\"}],\"role\":\"user\"}],\"systemInstruction\":[{\"text\":\"You are concise.\"},{\"text\":\"Mode: review\"}]}", openai = "{\"input\":[{\"content\":[{\"text\":\"Hello\",\"type\":\"input_text\"}],\"role\":\"user\",\"type\":\"message\"},{\"content\":[{\"text\":\"Hi\",\"type\":\"output_text\"}],\"role\":\"assistant\",\"type\":\"message\"},{\"content\":[{\"text\":\"Look it up\",\"type\":\"input_text\"}],\"role\":\"user\",\"type\":\"message\"},{\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\",\"call_id\":\"call_1\",\"name\":\"lookup\",\"type\":\"function_call\"},{\"call_id\":\"call_1\",\"output\":\"found\",\"type\":\"function_call_output\"}],\"instructions\":\"You are concise.\\n\\nPrefer cached data\\n\\nMode: review\\n\\nAnswer briefly\"}", + bedrock_developer = "{\"messages\":[{\"content\":[{\"text\":\"Hello\"}],\"role\":\"user\"},{\"content\":[{\"reasoningContent\":{\"reasoningText\":{\"signature\":\"sig-greet\",\"text\":\"greet\"}}},{\"text\":\"Hi\"}],\"role\":\"assistant\"},{\"content\":[{\"text\":\"Look it up\"},{\"text\":\"Prefer cached data\"}],\"role\":\"user\"},{\"content\":[{\"reasoningContent\":{\"reasoningText\":{\"signature\":\"sig-lookup\",\"text\":\"lookup\"}}},{\"toolUse\":{\"input\":{\"q\":\"x\"},\"name\":\"lookup\",\"toolUseId\":\"call_1\"}}],\"role\":\"assistant\"},{\"content\":[{\"toolResult\":{\"content\":[{\"text\":\"found\"}],\"status\":\"success\",\"toolUseId\":\"call_1\"}},{\"cachePoint\":{\"type\":\"default\"}},{\"text\":\"Answer briefly\"}],\"role\":\"user\"}],\"system\":[{\"text\":\"You are concise.\"},{\"cachePoint\":{\"type\":\"default\"}},{\"text\":\"Mode: review\"}]}", openai_compat = "[{\"content\":[{\"text\":\"You are concise.\",\"type\":\"text\"}],\"role\":\"system\"},{\"content\":[{\"text\":\"Hello\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":\"Hi\",\"role\":\"assistant\"},{\"content\":[{\"text\":\"Look it up\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"text\":\"Prefer cached data\",\"type\":\"text\"}],\"role\":\"system\"},{\"content\":\"\",\"role\":\"assistant\",\"tool_calls\":[{\"function\":{\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\",\"name\":\"lookup\"},\"id\":\"call_1\",\"type\":\"function\"}]},{\"content\":\"found\",\"name\":\"lookup\",\"role\":\"tool\",\"tool_call_id\":\"call_1\"},{\"content\":[{\"text\":\"Mode: review\",\"type\":\"text\"}],\"role\":\"system\"},{\"content\":[{\"text\":\"Answer briefly\",\"type\":\"text\"}],\"role\":\"system\"}]", } @@ -56,6 +59,7 @@ local function bodies() return { claude = codec.canonical_json(claude_mapper.map_messages(conversation(true))), bedrock = codec.canonical_json(bedrock_mapper.map_messages(conversation(false))), + bedrock_developer = codec.canonical_json(bedrock_mapper.map_messages(conversation(true))), google = codec.canonical_json({ contents = google_contents, systemInstruction = google_system }), openai = codec.canonical_json({ instructions = openai_mapper.extract_instructions(conversation(true)), @@ -67,7 +71,7 @@ end local function define_tests() describe("Request body golden sections", function() - for _, driver in ipairs({ "claude", "bedrock", "google", "openai", "openai_compat" }) do + for _, driver in ipairs({ "claude", "bedrock", "bedrock_developer", "google", "openai", "openai_compat" }) do it("keeps the " .. driver .. " body byte-identical for prompts without positional notes", function() local body = bodies()[driver] test.not_nil(body) From 9e3c4cbf021f83e77b89f3a76fe951d1365ddb0e Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 17:02:17 -0500 Subject: [PATCH 11/37] fix(agent): strip stale thinking before the request The prefix fingerprint is computed before generation; with drop_stale_thinking the agent strips blocks whose stored metadata.prefix_fingerprint differs (or that thinking_is_current rejects) from a copy of the history, including checkpoint replacements, and fails the step on an open tool round. Prompt function errors keep their details. time_aware reads its layout from time_format so a generic agent context format key is not taken as a time layout. --- src/agent/src/README.md | 44 ++++++- src/agent/src/agent.lua | 48 +++++-- src/agent/src/traits/init_time_aware.lua | 4 +- src/agent/test/agent_test.lua | 119 +++++++++++++++++- .../test/traits/init_time_aware_test.lua | 11 +- 5 files changed, 211 insertions(+), 15 deletions(-) diff --git a/src/agent/src/README.md b/src/agent/src/README.md index 4bba34d7..1c0a40f3 100644 --- a/src/agent/src/README.md +++ b/src/agent/src/README.md @@ -225,6 +225,8 @@ local response, err = runner:step(builder, { tool_call = "auto", disable_memory_recall = false, persist_context_notes = true, -- the host persists response.context_notes in place + drop_stale_thinking = true, -- strip blocks bound to another prefix before the request + thinking_is_current = nil, -- optional function(msg, fingerprint) -> boolean previous_memory_ids = { "mem_1", "mem_2" }, stop_check = function() return stop_requested end, cancel = turn_cancel, @@ -318,12 +320,41 @@ system, and each tool's name, description, schema and native definition not reach the provider and are not part of it. It is stable across steps while the sent system text and tools stay the same. +The projection is the Claude and Bedrock prefix. Other drivers bind a different +one: OpenAI moves developer messages into `instructions`, and Google filters tool +schemas, so equal fingerprints there are necessary but not sufficient for an +identical prefix. + The model is not part of the fingerprint. Switching models needs no stripping: the provider drops thinking blocks the target model cannot read, and some models read each other's blocks. The fingerprint covers only the system and tools -prefix; edits to earlier messages are not detected by it. Hosts store it on -persisted rows and strip the thinking blocks of rows whose fingerprint differs -from the current step's with `builder:drop_stale_thinking`. +prefix; edits to earlier messages are not detected by it. + +### Stale thinking + +The fingerprint is computed before the request, so the agent strips stale +thinking blocks itself: + +1. The host stores `response.prefix_fingerprint` as `metadata.prefix_fingerprint` + on the assistant and function call rows it persists from that step. +2. On later steps the host passes `drop_stale_thinking = true`. Before the + request, the agent removes the blocks of every message up to and including the + last one whose `metadata.prefix_fingerprint` differs from the current + fingerprint; a message with blocks and no stored fingerprint counts as stale. +3. A host with its own epoch (a checkpoint id, a compaction boundary) passes + `thinking_is_current = function(msg, fingerprint) ... end`, which replaces the + default comparison. + +Stripping works on copies: the host's message tables keep their blocks, and the +stored fingerprint never reaches the provider. When a stale turn still has a +function call without a result, the step fails with that error and sends no +request. The same stripping applies to a checkpoint replacement history. + +The Claude API can drop mismatched blocks itself +(`thinking.block_binding.prefix_mismatch_behavior = "drop_block"`). That covers +what the host cannot know, such as an edit to an earlier message; stripping by +fingerprint keeps known-stale blocks out of the request so they are not dropped, +re-billed and reported on every request. A configured recall that fails (the provider errors, cannot be opened, or returns no valid `memories` list) does not stop the step: generation runs @@ -350,7 +381,10 @@ persists instead (see Turn context). `agent_options.turn_context` declares per-turn context the host adds to each user turn. The `time_aware` trait sets -`turn_context.time = { timezone = , format = }`. +`turn_context.time = { timezone = , format = }`. +The layout key is `time_format` because the trait context is merged with the +agent context, where a generic `format` key belongs to other features; +`timezone` keeps its existing meaning. When a user message arrives, the host renders the note once and persists it as a positional note right after that message: @@ -575,7 +609,7 @@ return { execute = execute } ### Built-in Traits -- `wippy.agent.traits:time_aware` -- Declares `agent_options.turn_context.time` through `build_func_id`; the host renders the note with `wippy.agent:turn_context`. Configurable via context: `timezone` (default UTC), `format` (optional Go time layout). It contributes no prompt text; a `time_interval` setting is rejected. +- `wippy.agent.traits:time_aware` -- Declares `agent_options.turn_context.time` through `build_func_id`; the host renders the note with `wippy.agent:turn_context`. Configurable via context: `timezone` (default UTC), `time_format` (optional Go time layout). It contributes no prompt text; a `time_interval` setting is rejected. A trait build function that returns an error fails `compiler.compile` with an error naming the trait and the function. diff --git a/src/agent/src/agent.lua b/src/agent/src/agent.lua index daa85509..cac6ed18 100644 --- a/src/agent/src/agent.lua +++ b/src/agent/src/agent.lua @@ -148,6 +148,7 @@ local function execute_prompt_functions(self: any, runtime_context: any): (strin message = "prompt function " .. tostring(prompt_func.func_id) .. " failed: " .. tostring(err), kind = err:kind(), retryable = err:retryable(), + details = err:details(), }) end @@ -651,14 +652,52 @@ function agent:step(prompt_builder: any, runtime_options: any): (table?, string? return final_messages end - local final_messages = build_messages(prompt_builder:get_messages()) + local conversation = prompt_builder:get_messages() + local projected = { system_message } + for _, message in ipairs(conversation) do table.insert(projected, message) end + local fingerprint, fingerprint_err = hash.sha256(canonical.encode(prefix_projection(projected, options.tools))) + if fingerprint_err then + return nil, output.errors.generate():kind(output.ERROR_TYPE.INVALID_REQUEST) + :message("Prefix fingerprint failed: " .. tostring(fingerprint_err)):build() + end + + -- Thinking blocks are bound to the prefix they were produced under. With + -- drop_stale_thinking the request carries only blocks the host's rows mark + -- with the current fingerprint (or that the host predicate accepts); the + -- host's message tables keep theirs. + local function current_thinking(messages: {any}): ({any}?, any) + if not runtime_options.drop_stale_thinking then + return messages, nil + end + local is_current = runtime_options.thinking_is_current + local working = prompt.new(messages) + local _, strip_err = working:drop_stale_thinking(function(msg) + if is_current then + return is_current(msg, fingerprint) == true + end + return msg.metadata.prefix_fingerprint == fingerprint + end) + if strip_err then + return nil, output.errors.generate():kind(output.ERROR_TYPE.INVALID_REQUEST) + :message(tostring(strip_err)):build() + end + return working:get_messages(), nil + end + + local current, thinking_err = current_thinking(conversation) + if not current then + return nil, thinking_err + end + local final_messages = build_messages(current) if runtime_options.checkpoint then options.checkpoint = { token_threshold = runtime_options.checkpoint.token_threshold, compact = function(info) local replacement, compact_err = runtime_options.checkpoint.compact(info) if not replacement then return nil, compact_err end - return build_messages(replacement:get_messages()) + local retained, retained_err = current_thinking(replacement:get_messages()) + if not retained then return nil, retained_err end + return build_messages(retained) end, } end @@ -698,11 +737,6 @@ function agent:step(prompt_builder: any, runtime_options: any): (table?, string? --end - local fingerprint, fingerprint_err = hash.sha256(canonical.encode(prefix_projection(final_messages, options.tools))) - if fingerprint_err then - return nil, output.errors.generate():kind(output.ERROR_TYPE.INVALID_REQUEST) - :message("Prefix fingerprint failed: " .. tostring(fingerprint_err)):build() - end local context_notes = memory_prompt and { memory_prompt } or nil if runtime_options.stop_check and runtime_options.stop_check() then diff --git a/src/agent/src/traits/init_time_aware.lua b/src/agent/src/traits/init_time_aware.lua index a4491873..bcd11a1c 100644 --- a/src/agent/src/traits/init_time_aware.lua +++ b/src/agent/src/traits/init_time_aware.lua @@ -15,7 +15,9 @@ local function execute(base_prompt: string?): (TimeAwareResult?, string?) return nil, "time_aware no longer rounds the time: remove time_interval; hosts render the exact minute " .. "from agent_options.turn_context with wippy.agent:turn_context" end - local format = ctx.get("format") + -- The trait context merges with the agent context, so the layout key is + -- namespaced: a generic "format" there belongs to someone else. + local format = ctx.get("time_format") return { agent_options = { turn_context = { diff --git a/src/agent/test/agent_test.lua b/src/agent/test/agent_test.lua index 651c849d..9ed985dc 100644 --- a/src/agent/test/agent_test.lua +++ b/src/agent/test/agent_test.lua @@ -1503,7 +1503,8 @@ local function define_tests() with_context = function(self) return self end, call = function() return nil, errors.new({ message = "workspace store unavailable", - kind = errors.UNAVAILABLE, retryable = true }) + kind = errors.UNAVAILABLE, retryable = true, + details = { store = "workspace", attempt = 2 } }) end } end @@ -1519,9 +1520,125 @@ local function define_tests() test.contains(tostring(err), "workspace store unavailable") test.eq(err:kind(), errors.UNAVAILABLE) test.is_true(err:retryable()) + test.eq(err:details().store, "workspace") + test.eq(err:details().attempt, 2) test.eq(#requests, 0) end) + describe("Stale thinking", function() + local function blocks(name) + return { { type = "thinking", thinking = name, signature = "sig-" .. name } } + end + + local function current_fingerprint() + agent._llm = recording_llm({}) + local builder = prompt.new() + builder:add_user("Hello") + return assert(agent.new(copy_spec()):step(builder)).prefix_fingerprint + end + + local function history(answer_metadata) + local answer = { role = prompt.ROLE.ASSISTANT, content = { prompt.text("Earlier answer") }, + metadata = answer_metadata } + local builder = prompt.new({ + { role = prompt.ROLE.USER, content = { prompt.text("Earlier question") } }, + answer, + { role = prompt.ROLE.USER, content = { prompt.text("New question") } }, + }) + return builder, answer + end + + local function sent_answer(requests) + for _, message in ipairs(requests[#requests]) do + if message.role == prompt.ROLE.ASSISTANT then return message end + end + return nil + end + + it("strips blocks bound to another prefix before the request", function() + local builder, answer = history({ thinking_blocks = blocks("old"), prefix_fingerprint = "previous" }) + local requests = {} + agent._llm = recording_llm(requests) + local result, err = agent.new(copy_spec()):step(builder, { drop_stale_thinking = true }) + test.is_nil(err) + test.not_nil(result) + local sent = assert(sent_answer(requests)) + test.is_nil(sent.metadata.thinking_blocks) + test.eq(sent.metadata.prefix_fingerprint, "previous") + test.eq(#answer.metadata.thinking_blocks, 1) + end) + + it("treats blocks without a stored fingerprint as stale", function() + local builder = history({ thinking_blocks = blocks("old") }) + local requests = {} + agent._llm = recording_llm(requests) + assert(agent.new(copy_spec()):step(builder, { drop_stale_thinking = true })) + test.is_nil(assert(sent_answer(requests)).metadata.thinking_blocks) + end) + + it("keeps blocks bound to the current prefix", function() + local fingerprint = current_fingerprint() + local builder = history({ thinking_blocks = blocks("kept"), prefix_fingerprint = fingerprint }) + local requests = {} + agent._llm = recording_llm(requests) + assert(agent.new(copy_spec()):step(builder, { drop_stale_thinking = true })) + test.eq(assert(sent_answer(requests)).metadata.thinking_blocks[1].thinking, "kept") + end) + + it("leaves blocks alone when the host does not ask for stripping", function() + local builder = history({ thinking_blocks = blocks("old"), prefix_fingerprint = "previous" }) + local requests = {} + agent._llm = recording_llm(requests) + assert(agent.new(copy_spec()):step(builder)) + test.eq(assert(sent_answer(requests)).metadata.thinking_blocks[1].thinking, "old") + end) + + it("honours a host predicate", function() + local fingerprint = current_fingerprint() + local builder = history({ thinking_blocks = blocks("epoch"), prefix_fingerprint = fingerprint, + checkpoint = "before" }) + local requests = {} + local seen = nil + agent._llm = recording_llm(requests) + assert(agent.new(copy_spec()):step(builder, { + drop_stale_thinking = true, + thinking_is_current = function(msg, current) + seen = current + return msg.metadata.prefix_fingerprint == current and msg.metadata.checkpoint == "after" + end, + })) + test.eq(seen, fingerprint) + test.is_nil(assert(sent_answer(requests)).metadata.thinking_blocks) + end) + + it("fails the step without a request when a stale turn has an open call", function() + local builder = prompt.new({ + { role = prompt.ROLE.USER, content = { prompt.text("Question") } }, + { role = prompt.ROLE.ASSISTANT, content = { prompt.text("Looking") }, + metadata = { thinking_blocks = blocks("open"), prefix_fingerprint = "previous" } }, + { role = prompt.ROLE.FUNCTION_CALL, content = {}, + function_call = { id = "call_open", name = "calculator", arguments = "{}" } }, + }) + local requests = {} + agent._llm = recording_llm(requests) + local result, err = agent.new(copy_spec()):step(builder, { drop_stale_thinking = true }) + test.is_nil(result) + test.contains(tostring(err), "call_open") + test.eq(#requests, 0) + end) + + it("never sends the stored fingerprint to the provider", function() + local fingerprint = current_fingerprint() + local builder = history({ thinking_blocks = blocks("kept"), prefix_fingerprint = fingerprint }) + local requests = {} + agent._llm = recording_llm(requests) + assert(agent.new(copy_spec()):step(builder, { drop_stale_thinking = true })) + local mapped = assert(claude_mapper.map_messages(requests[1])) + test.is_nil(canonical(mapped):find(fingerprint, 1, true)) + test.is_nil(canonical(mapped):find("prefix_fingerprint", 1, true)) + end) + end) + it("sends the same system message on every step", function() local requests = {} agent._llm = recording_llm(requests) diff --git a/src/agent/test/traits/init_time_aware_test.lua b/src/agent/test/traits/init_time_aware_test.lua index ac2b8f37..2f35db32 100644 --- a/src/agent/test/traits/init_time_aware_test.lua +++ b/src/agent/test/traits/init_time_aware_test.lua @@ -14,7 +14,7 @@ local function define_tests() it("carries the configured timezone and format", function() local result, err = funcs.new() - :with_context({ timezone = "Europe/Berlin", format = "2006-01-02 15:04" }) + :with_context({ timezone = "Europe/Berlin", time_format = "2006-01-02 15:04" }) :call("wippy.agent.traits:init_time_aware", "base prompt") test.is_nil(err) @@ -23,6 +23,15 @@ local function define_tests() test.eq(result.agent_options.turn_context.time.format, "2006-01-02 15:04") end) + it("ignores a format key that the agent context uses for another purpose", function() + local result, err = funcs.new() + :with_context({ timezone = "UTC", format = "json" }) + :call("wippy.agent.traits:init_time_aware", "base prompt") + + test.is_nil(err) + test.is_nil(result.agent_options.turn_context.time.format) + end) + it("rejects the removed time_interval setting and names its replacement", function() local result, err = funcs.new() :with_context({ timezone = "UTC", time_interval = 15 }) From f8d7317594ca7853f45d409ad02f355a5608d2bc Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 17:32:47 -0500 Subject: [PATCH 12/37] fix(llm)!: wrap developer notes in their own instruction block Developer messages, positional or not, render on Claude and Bedrock as their own "..." text block after the user turn they follow, restoring the model-visible wrapper without editing earlier blocks. Positional system notes stay plain. The unreachable map_messages nil guards in the Claude and Bedrock handlers are removed. drop_stale_thinking names the unanswered call id and tool name when it refuses. BREAKING CHANGE: Claude and Bedrock render developer messages and positional notes differently: each developer message is its own wrapped text block after the user turn it follows (previously merged into the preceding text, or a separate user message after tool results or cache markers), Bedrock developer messages are no longer sent as system blocks, positional notes render in place instead of the top-level system prompt, and builder:drop_stale_thinking replaces the earlier bind_thinking draft. Stored conversations render a different prefix once after upgrading. --- src/llm/src/README.md | 2 +- src/llm/src/bedrock/generate.lua | 5 +-- src/llm/src/bedrock/mapper.lua | 11 ++++-- src/llm/src/bedrock/structured_output.lua | 5 +-- src/llm/src/claude/generate.lua | 5 +-- src/llm/src/claude/mapper.lua | 11 ++++-- src/llm/src/claude/structured_output.lua | 5 +-- src/llm/src/prompt.lua | 20 +++++++--- src/llm/test/_index.yaml | 1 + src/llm/test/bedrock/mapper_test.lua | 42 +++++++++++++++------ src/llm/test/claude/generate_test.lua | 2 +- src/llm/test/claude/mapper_test.lua | 45 ++++++++++++++++------- src/llm/test/prompt_test.lua | 2 + src/llm/test/request_golden_test.lua | 12 +++--- 14 files changed, 105 insertions(+), 63 deletions(-) diff --git a/src/llm/src/README.md b/src/llm/src/README.md index da12c9dd..8fa3f03d 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -228,7 +228,7 @@ builder:add_system("The current date and time is Tuesday, November 3, 2026 at 09 { preserve_position = true }) ``` -Ordinary developer messages render the same way on Claude and Bedrock: as their own text blocks after the content of the user turn they follow, or as a new user turn at the start or after an assistant turn. Existing blocks are never edited, so the rendering does not depend on where cache markers sit. The text is sent as written: Claude no longer wraps it in `` tags. A host that needs the model to tell its guidance apart from the user's prose marks the note text itself. +Developer messages, positional or not, render the same way on Claude and Bedrock: each text part becomes its own text block, `` + text + ``, after the content of the user turn it follows, or a new user turn at the start or after an assistant turn. The wrapper tells the model host guidance apart from the user's prose. Positional system notes render as plain text. Existing blocks are never edited, so the rendering does not depend on where cache markers sit. Bedrock sends developer messages as user text instead of `system` blocks. A Bedrock conversation stored before this change therefore renders a different prefix once after upgrading: its cache restarts and its earlier thinking blocks no longer match. The provider rejects such blocks unless the request lets it drop them; hosts that keep thinking blocks across the upgrade strip them with `drop_stale_thinking`. diff --git a/src/llm/src/bedrock/generate.lua b/src/llm/src/bedrock/generate.lua index db41b818..a706f30d 100644 --- a/src/llm/src/bedrock/generate.lua +++ b/src/llm/src/bedrock/generate.lua @@ -98,10 +98,7 @@ function generate_handler.handler(contract_args) name_to_id_map = {} } - local mapped, map_err = generate_handler._mapper.map_messages(contract_args.messages) - if not mapped then - return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(tostring(map_err)):build() - end + local mapped = generate_handler._mapper.map_messages(contract_args.messages) local inference_config, additional_fields, adjusted, options_err = generate_handler._mapper.map_options(contract_args.options or {}, contract_args.accepts) if options_err then return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(options_err):build() diff --git a/src/llm/src/bedrock/mapper.lua b/src/llm/src/bedrock/mapper.lua index 9584786b..70ece0ff 100644 --- a/src/llm/src/bedrock/mapper.lua +++ b/src/llm/src/bedrock/mapper.lua @@ -140,14 +140,17 @@ local function convert_document_to_converse(content_part) return nil end -local function user_content_blocks(content) +local function user_content_blocks(content, developer: boolean) local content_blocks = {} + local function text_of(value: string): string + return developer and prompt.developer_instruction(value) or value + end if type(content) == "string" then - table.insert(content_blocks, { text = content }) + table.insert(content_blocks, { text = text_of(content) }) elseif type(content) == "table" then for _, part in ipairs(content) do if part.type == "text" and part.text then - table.insert(content_blocks, { text = part.text }) + table.insert(content_blocks, { text = text_of(part.text) }) elseif part.type == "image" then local img = convert_image_to_converse(part) if img then @@ -370,7 +373,7 @@ function mapper.map_messages(contract_messages) -- render as user text, so they never rewrite the system blocks; -- consolidation appends them as blocks after the user turn they -- follow, or they open a user turn after an assistant turn. - local content_blocks = user_content_blocks(msg.content) + local content_blocks = user_content_blocks(msg.content, msg.role == prompt.ROLE.DEVELOPER) if #content_blocks > 0 then table.insert(converse_messages, { role = "user", diff --git a/src/llm/src/bedrock/structured_output.lua b/src/llm/src/bedrock/structured_output.lua index b00b9819..7a5d3d0e 100644 --- a/src/llm/src/bedrock/structured_output.lua +++ b/src/llm/src/bedrock/structured_output.lua @@ -91,10 +91,7 @@ function structured_output_handler.handler(contract_args) return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(forced_err):build() end - local mapped, map_err = structured_output_handler._mapper.map_messages(contract_args.messages) - if not mapped then - return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(tostring(map_err)):build() - end + local mapped = structured_output_handler._mapper.map_messages(contract_args.messages) local inference_config, additional_fields, adjusted, options_err = structured_output_handler._mapper.map_options(contract_args.options or {}, contract_args.accepts) if options_err then return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(options_err):build() diff --git a/src/llm/src/claude/generate.lua b/src/llm/src/claude/generate.lua index 3c1aeaa6..840d5377 100644 --- a/src/llm/src/claude/generate.lua +++ b/src/llm/src/claude/generate.lua @@ -100,10 +100,7 @@ function generate_handler.handler(contract_args) name_to_id_map = {} } - local mapped_messages, map_err = generate_handler._mapper.map_messages(contract_args.messages) - if not mapped_messages then - return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(tostring(map_err)):build() - end + local mapped_messages = generate_handler._mapper.map_messages(contract_args.messages) local mapped_options, adjusted, options_err = generate_handler._mapper.map_options(contract_args.options or {}, contract_args.accepts) if options_err then return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(options_err):build() diff --git a/src/llm/src/claude/mapper.lua b/src/llm/src/claude/mapper.lua index 3c1e9b85..fe3d9583 100644 --- a/src/llm/src/claude/mapper.lua +++ b/src/llm/src/claude/mapper.lua @@ -280,17 +280,20 @@ local function process_content_array(content) return content end -local function note_blocks(content): {any} +local function note_blocks(content, developer: boolean): {any} local blocks: {any} = {} + local function text_of(value: string): string + return developer and prompt.developer_instruction(value) or value + end if type(content) == "string" then if content ~= "" then - table.insert(blocks, { type = "text", text = content }) + table.insert(blocks, { type = "text", text = text_of(content) }) end elseif type(content) == "table" then for _, part in ipairs(content) do if part.type == "text" then if part.text and part.text ~= "" then - table.insert(blocks, { type = "text", text = part.text }) + table.insert(blocks, { type = "text", text = text_of(part.text) }) end else table.insert(blocks, convert_image_content(part)) @@ -376,7 +379,7 @@ function mapper.map_messages(contract_messages) -- not depend on where cache markers sit and every earlier block -- stays byte-identical across requests. in_system_phase = false - local blocks = note_blocks(msg.content) + local blocks = note_blocks(msg.content, msg.role == prompt.ROLE.DEVELOPER) local last_msg = claude_messages[#claude_messages] if not last_msg or last_msg.role ~= "user" then -- A note at the start or after an assistant turn opens a user diff --git a/src/llm/src/claude/structured_output.lua b/src/llm/src/claude/structured_output.lua index 80d02108..353ab5c2 100644 --- a/src/llm/src/claude/structured_output.lua +++ b/src/llm/src/claude/structured_output.lua @@ -173,10 +173,7 @@ function structured_output_handler.handler(contract_args) end end - local mapped_messages, map_err = structured_output_handler._mapper.map_messages(contract_args.messages) - if not mapped_messages then - return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(tostring(map_err)):build() - end + local mapped_messages = structured_output_handler._mapper.map_messages(contract_args.messages) local mapped_options, adjusted, options_err = structured_output_handler._mapper.map_options(contract_args.options or {}, contract_args.accepts) if options_err then return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(options_err):build() diff --git a/src/llm/src/prompt.lua b/src/llm/src/prompt.lua index 7542b26e..47ad31d0 100644 --- a/src/llm/src/prompt.lua +++ b/src/llm/src/prompt.lua @@ -141,6 +141,12 @@ function prompt.is_positional_note(msg: Message): boolean and type(msg.metadata) == "table" and msg.metadata.preserve_position == true end +-- Text of a developer message as drivers without a developer role render it +-- inside a user turn, so the model can tell host guidance from user prose. +function prompt.developer_instruction(text: string): string + return "" .. text .. "" +end + --------------------------- -- Helper Functions --------------------------- @@ -409,17 +415,21 @@ function prompt.new(messages: {Message}?) if turn_start then local ids: {any} = {} if msg.role == prompt.ROLE.FUNCTION_CALL then - table.insert(ids, msg.function_call and msg.function_call.id or false) + table.insert(ids, { + id = msg.function_call and msg.function_call.id or false, + name = msg.function_call and msg.function_call.name, + }) else for _, part in ipairs(msg.content or {}) do if (part :: any).type == "function_call" then - table.insert(ids, (part :: any).id or false) + table.insert(ids, { id = (part :: any).id or false, name = (part :: any).name }) end end end - for _, id in ipairs(ids) do - if not open_call[turn_start] and not (id and resolved[id]) then - open_call[turn_start] = id and tostring(id) or "without id" + for _, call in ipairs(ids) do + if not open_call[turn_start] and not (call.id and resolved[call.id]) then + open_call[turn_start] = string.format("%s (%s)", + call.id and tostring(call.id) or "without id", tostring(call.name or "unnamed")) end end end diff --git a/src/llm/test/_index.yaml b/src/llm/test/_index.yaml index 54d9b3d7..245b299c 100644 --- a/src/llm/test/_index.yaml +++ b/src/llm/test/_index.yaml @@ -402,6 +402,7 @@ entries: - json imports: mapper: wippy.llm.bedrock:mapper + prompt: wippy.llm:prompt output: wippy.llm:output test: wippy.test:test method: run_tests diff --git a/src/llm/test/bedrock/mapper_test.lua b/src/llm/test/bedrock/mapper_test.lua index e382284a..9eed0fa7 100644 --- a/src/llm/test/bedrock/mapper_test.lua +++ b/src/llm/test/bedrock/mapper_test.lua @@ -1,4 +1,5 @@ local mapper = require("mapper") +local prompt = require("prompt") local output = require("output") local json = require("json") @@ -29,7 +30,8 @@ local function define_tests() local content = mapped.messages[1].content test.eq(#content, 2) test.eq(content[1].text, "Question") - test.eq(content[2].text, "The current date and time is Tuesday.") + test.eq(content[2].text, role == "developer" and "The current date and time is Tuesday." + or "The current date and time is Tuesday.") end end) @@ -79,7 +81,7 @@ local function define_tests() test.eq(#content, 3) test.eq(content[1].text, "Question") test.eq(content[2].cachePoint.type, "default") - test.eq(content[3].text, "Recalled memory") + test.eq(content[3].text, "Recalled memory") end) local function same(a, b) @@ -123,11 +125,6 @@ local function define_tests() { role = "assistant", content = text("Answer") }, note("developer", "Stopped by the user"), }, - after_call = { - { role = "user", content = text("Question") }, - { role = "function_call", content = {}, function_call = { id = "call_1", name = "lookup", arguments = "{}" } }, - note("system", "Interrupted"), - }, after_result = { { role = "user", content = text("Question") }, { role = "function_call", content = {}, function_call = { id = "call_1", name = "lookup", arguments = "{}" } }, @@ -137,6 +134,27 @@ local function define_tests() } end + it("wraps developer text in its own developer-instruction block", function() + for _, metadata in ipairs({ { preserve_position = true }, {} }) do + local mapped = assert(mapper.map_messages({ + { role = "user", content = text("Question") }, + { role = "developer", content = text("Be concise"), metadata = metadata }, + })) + local content = mapped.messages[1].content + test.eq(#content, 2) + test.eq(content[1].text, "Question") + test.eq(content[2].text, "Be concise") + end + end) + + it("renders positional system notes as plain text", function() + local mapped = assert(mapper.map_messages({ + { role = "user", content = text("Question") }, + note("system", "The current date and time is Tuesday."), + })) + test.eq(mapped.messages[1].content[2].text, "The current date and time is Tuesday.") + end) + it("opens a user turn for a positional note that follows no user turn", function() for label, history in pairs(shapes()) do local mapped, err = mapper.map_messages(history) @@ -152,7 +170,7 @@ local function define_tests() local found = nil for m, message in ipairs(mapped.messages) do for b, block in ipairs(message.content) do - if block.text == note_text then + if block.text == note_text or block.text == prompt.developer_instruction(note_text) then found = { m = m, b = b, role = message.role } end end @@ -170,7 +188,7 @@ local function define_tests() test.eq(mapped.messages[2].role, "assistant") test.eq(#mapped.messages[2].content, 1) test.eq(mapped.messages[3].role, "user") - test.eq(mapped.messages[3].content[1].text, "Stopped by the user") + test.eq(mapped.messages[3].content[1].text, "Stopped by the user") end) it("renders each positional note shape as a stable prefix of the next request", function() @@ -201,8 +219,8 @@ local function define_tests() test.eq(mapped.messages[1].role, "user") test.eq(mapped.messages[1].content[1].text, "Question") end - test.eq(first.messages[1].content[2].text, "Step one guidance") - test.eq(second.messages[1].content[2].text, "Step two guidance") + test.eq(first.messages[1].content[2].text, "Step one guidance") + test.eq(second.messages[1].content[2].text, "Step two guidance") end) end) @@ -509,7 +527,7 @@ local function define_tests() test.eq((result.system[1] :: any).text, "System prompt") test.eq(#result.messages, 1) test.eq(result.messages[1].role, "user") - test.eq(result.messages[1].content[1].text, "Developer instruction") + test.eq(result.messages[1].content[1].text, "Developer instruction") test.eq(result.messages[1].content[2].text, "Hello") end) diff --git a/src/llm/test/claude/generate_test.lua b/src/llm/test/claude/generate_test.lua index 4d30980e..341ab471 100644 --- a/src/llm/test/claude/generate_test.lua +++ b/src/llm/test/claude/generate_test.lua @@ -389,7 +389,7 @@ local function define_tests() test.eq(user_msg.role, "user") test.eq(user_msg.content[1].text, "Hello") test.eq(user_msg.content[2].type, "text") - test.eq(user_msg.content[2].text, "Be concise") + test.eq(user_msg.content[2].text, "Be concise") return { content = { { type = "text", text = "Hi!" } }, diff --git a/src/llm/test/claude/mapper_test.lua b/src/llm/test/claude/mapper_test.lua index eddd35ce..10f9c902 100644 --- a/src/llm/test/claude/mapper_test.lua +++ b/src/llm/test/claude/mapper_test.lua @@ -33,7 +33,8 @@ local function define_tests() test.eq(#content, 2) test.eq(content[1].text, "Question") test.eq(content[2].type, "text") - test.eq(content[2].text, "The current date and time is Tuesday.") + test.eq(content[2].text, role == "developer" and "The current date and time is Tuesday." + or "The current date and time is Tuesday.") end end) @@ -84,7 +85,7 @@ local function define_tests() })) local content = mapped.messages[1].content test.eq(content[1].cache_control.type, "ephemeral") - test.eq(content[2].text, "Recalled memory") + test.eq(content[2].text, "Recalled memory") test.is_nil(content[2].cache_control) end) @@ -104,7 +105,7 @@ local function define_tests() test.eq(after.messages[1].content[2].text, before.messages[1].content[2].text) test.eq(after.messages[1].content[2].text, "The current date and time is Tuesday.") test.eq(#after.messages[1].content, 3) - test.eq(after.messages[1].content[3].text, "Attached: report.pdf") + test.eq(after.messages[1].content[3].text, "Attached: report.pdf") end) local function same(a, b) @@ -148,11 +149,6 @@ local function define_tests() { role = "assistant", content = text("Answer") }, note("developer", "Stopped by the user"), }, - after_call = { - { role = "user", content = text("Question") }, - { role = "function_call", content = {}, function_call = { id = "call_1", name = "lookup", arguments = "{}" } }, - note("system", "Interrupted"), - }, after_result = { { role = "user", content = text("Question") }, { role = "function_call", content = {}, function_call = { id = "call_1", name = "lookup", arguments = "{}" } }, @@ -162,6 +158,27 @@ local function define_tests() } end + it("wraps developer text in its own developer-instruction block", function() + for _, metadata in ipairs({ { preserve_position = true }, {} }) do + local mapped = assert(mapper.map_messages({ + { role = "user", content = text("Question") }, + { role = "developer", content = text("Be concise"), metadata = metadata }, + })) + local content = mapped.messages[1].content + test.eq(#content, 2) + test.eq(content[1].text, "Question") + test.eq(content[2].text, "Be concise") + end + end) + + it("renders positional system notes as plain text", function() + local mapped = assert(mapper.map_messages({ + { role = "user", content = text("Question") }, + note("system", "The current date and time is Tuesday."), + })) + test.eq(mapped.messages[1].content[2].text, "The current date and time is Tuesday.") + end) + it("opens a user turn for a positional note that follows no user turn", function() for label, history in pairs(shapes()) do local mapped, err = mapper.map_messages(history) @@ -177,7 +194,7 @@ local function define_tests() local found = nil for m, message in ipairs(mapped.messages) do for b, block in ipairs(message.content) do - if block.text == note_text then + if block.text == note_text or block.text == prompt.developer_instruction(note_text) then found = { m = m, b = b, role = message.role } end end @@ -195,7 +212,7 @@ local function define_tests() test.eq(mapped.messages[2].role, "assistant") test.eq(#mapped.messages[2].content, 1) test.eq(mapped.messages[3].role, "user") - test.eq(mapped.messages[3].content[1].text, "Stopped by the user") + test.eq(mapped.messages[3].content[1].text, "Stopped by the user") end) it("renders each positional note shape as a stable prefix of the next request", function() @@ -488,7 +505,7 @@ local function define_tests() test.eq(#user_msg.content, 2) test.eq(tostring((user_msg.content[1] :: any).text), "Hello") test.eq((user_msg.content[2] :: any).type, "text") - test.eq(tostring((user_msg.content[2] :: any).text), "Be concise") + test.eq(tostring((user_msg.content[2] :: any).text), "Be concise") end) it("should append developer messages to an image-only previous message", function() @@ -508,7 +525,7 @@ local function define_tests() test.eq(#last_msg.content, 2, "developer instruction must survive an image-only previous message") test.eq((last_msg.content[2] :: any).type, "text") - test.eq(tostring((last_msg.content[2] :: any).text), "Be concise") + test.eq(tostring((last_msg.content[2] :: any).text), "Be concise") -- The image itself must still be there. test.eq((last_msg.content[1] :: any).type, "image") @@ -719,7 +736,7 @@ local function define_tests() test.eq(second.messages[1].content[1].text, first.messages[1].content[1].text) test.not_nil((second.messages[1].content[1] :: any).cache_control) test.eq(#second.messages, 1) - test.eq(second.messages[1].content[2].text, "Changing memory recall") + test.eq(second.messages[1].content[2].text, "Changing memory recall") test.is_nil((second.messages[1].content[2] :: any).cache_control) end) @@ -768,7 +785,7 @@ local function define_tests() end end end - test.eq(earlier.messages[1].content[2].text, "Attached: report.pdf") + test.eq(earlier.messages[1].content[2].text, "Attached: report.pdf") end) it("should handle empty messages gracefully", function() diff --git a/src/llm/test/prompt_test.lua b/src/llm/test/prompt_test.lua index 97475976..8ccbf678 100644 --- a/src/llm/test/prompt_test.lua +++ b/src/llm/test/prompt_test.lua @@ -140,6 +140,7 @@ local function define_tests() local dropped, err = builder:drop_stale_thinking(function() return false end) test.is_nil(dropped) test.contains(tostring(err), "call_open") + test.contains(tostring(err), "fetch") test.eq(thinking_of(builder:get_messages(), 2), "a") end) @@ -153,6 +154,7 @@ local function define_tests() local dropped, err = builder:drop_stale_thinking(function() return false end) test.is_nil(dropped) test.contains(tostring(err), "call_part") + test.contains(tostring(err), "lookup") end) it("strips an assistant turn once every call in it has a result", function() diff --git a/src/llm/test/request_golden_test.lua b/src/llm/test/request_golden_test.lua index 5ed60ccb..d4a6700b 100644 --- a/src/llm/test/request_golden_test.lua +++ b/src/llm/test/request_golden_test.lua @@ -7,16 +7,16 @@ local codec = require("codec") -- Message-derived request body sections for prompts without positional notes. -- They are recorded from the drivers before positional notes existed, except --- Claude developer messages, which render as separate text blocks in the user --- turn they follow, and bedrock_developer, recorded after Bedrock developer --- messages moved from system blocks to user text. Any difference changes the --- provider prefix and its cache. +-- developer messages on Claude and Bedrock (claude, bedrock_developer): each +-- renders as its own "..." text +-- block in the user turn it follows, and Bedrock no longer sends them as system +-- blocks. Any difference changes the provider prefix and its cache. local GOLDEN = { - claude = "{\"messages\":[{\"content\":[{\"text\":\"Hello\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"signature\":\"sig-greet\",\"thinking\":\"greet\",\"type\":\"thinking\"},{\"text\":\"Hi\",\"type\":\"text\"}],\"role\":\"assistant\"},{\"content\":[{\"text\":\"Look it up\",\"type\":\"text\"},{\"text\":\"Prefer cached data\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"signature\":\"sig-lookup\",\"thinking\":\"lookup\",\"type\":\"thinking\"},{\"id\":\"call_1\",\"input\":{\"q\":\"x\"},\"name\":\"lookup\",\"type\":\"tool_use\"}],\"role\":\"assistant\"},{\"content\":[{\"cache_control\":{\"type\":\"ephemeral\"},\"content\":\"found\",\"tool_use_id\":\"call_1\",\"type\":\"tool_result\"},{\"text\":\"Answer briefly\",\"type\":\"text\"}],\"role\":\"user\"}],\"system\":[{\"cache_control\":{\"type\":\"ephemeral\"},\"text\":\"You are concise.\",\"type\":\"text\"},{\"text\":\"Mode: review\",\"type\":\"text\"}]}", + claude = "{\"messages\":[{\"content\":[{\"text\":\"Hello\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"signature\":\"sig-greet\",\"thinking\":\"greet\",\"type\":\"thinking\"},{\"text\":\"Hi\",\"type\":\"text\"}],\"role\":\"assistant\"},{\"content\":[{\"text\":\"Look it up\",\"type\":\"text\"},{\"text\":\"\\u003cdeveloper-instruction\\u003ePrefer cached data\\u003c/developer-instruction\\u003e\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"signature\":\"sig-lookup\",\"thinking\":\"lookup\",\"type\":\"thinking\"},{\"id\":\"call_1\",\"input\":{\"q\":\"x\"},\"name\":\"lookup\",\"type\":\"tool_use\"}],\"role\":\"assistant\"},{\"content\":[{\"cache_control\":{\"type\":\"ephemeral\"},\"content\":\"found\",\"tool_use_id\":\"call_1\",\"type\":\"tool_result\"},{\"text\":\"\\u003cdeveloper-instruction\\u003eAnswer briefly\\u003c/developer-instruction\\u003e\",\"type\":\"text\"}],\"role\":\"user\"}],\"system\":[{\"cache_control\":{\"type\":\"ephemeral\"},\"text\":\"You are concise.\",\"type\":\"text\"},{\"text\":\"Mode: review\",\"type\":\"text\"}]}", bedrock = "{\"messages\":[{\"content\":[{\"text\":\"Hello\"}],\"role\":\"user\"},{\"content\":[{\"reasoningContent\":{\"reasoningText\":{\"signature\":\"sig-greet\",\"text\":\"greet\"}}},{\"text\":\"Hi\"}],\"role\":\"assistant\"},{\"content\":[{\"text\":\"Look it up\"}],\"role\":\"user\"},{\"content\":[{\"reasoningContent\":{\"reasoningText\":{\"signature\":\"sig-lookup\",\"text\":\"lookup\"}}},{\"toolUse\":{\"input\":{\"q\":\"x\"},\"name\":\"lookup\",\"toolUseId\":\"call_1\"}}],\"role\":\"assistant\"},{\"content\":[{\"toolResult\":{\"content\":[{\"text\":\"found\"}],\"status\":\"success\",\"toolUseId\":\"call_1\"}},{\"cachePoint\":{\"type\":\"default\"}}],\"role\":\"user\"}],\"system\":[{\"text\":\"You are concise.\"},{\"cachePoint\":{\"type\":\"default\"}},{\"text\":\"Mode: review\"}]}", google = "{\"contents\":[{\"parts\":[{\"text\":\"Hello\"}],\"role\":\"user\"},{\"parts\":[{\"text\":\"Hi\"}],\"role\":\"model\"},{\"parts\":[{\"text\":\"Look it up\"}],\"role\":\"user\"},{\"parts\":[{\"text\":\"Prefer cached data\"}],\"role\":\"user\"},{\"parts\":[{\"functionCall\":{\"args\":{\"q\":\"x\"},\"name\":\"lookup\"}}],\"role\":\"model\"},{\"parts\":[{\"functionResponse\":{\"name\":\"lookup\",\"response\":{\"content\":\"found\"}}}],\"role\":\"user\"},{\"parts\":[{\"text\":\"Answer briefly\"}],\"role\":\"user\"}],\"systemInstruction\":[{\"text\":\"You are concise.\"},{\"text\":\"Mode: review\"}]}", openai = "{\"input\":[{\"content\":[{\"text\":\"Hello\",\"type\":\"input_text\"}],\"role\":\"user\",\"type\":\"message\"},{\"content\":[{\"text\":\"Hi\",\"type\":\"output_text\"}],\"role\":\"assistant\",\"type\":\"message\"},{\"content\":[{\"text\":\"Look it up\",\"type\":\"input_text\"}],\"role\":\"user\",\"type\":\"message\"},{\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\",\"call_id\":\"call_1\",\"name\":\"lookup\",\"type\":\"function_call\"},{\"call_id\":\"call_1\",\"output\":\"found\",\"type\":\"function_call_output\"}],\"instructions\":\"You are concise.\\n\\nPrefer cached data\\n\\nMode: review\\n\\nAnswer briefly\"}", - bedrock_developer = "{\"messages\":[{\"content\":[{\"text\":\"Hello\"}],\"role\":\"user\"},{\"content\":[{\"reasoningContent\":{\"reasoningText\":{\"signature\":\"sig-greet\",\"text\":\"greet\"}}},{\"text\":\"Hi\"}],\"role\":\"assistant\"},{\"content\":[{\"text\":\"Look it up\"},{\"text\":\"Prefer cached data\"}],\"role\":\"user\"},{\"content\":[{\"reasoningContent\":{\"reasoningText\":{\"signature\":\"sig-lookup\",\"text\":\"lookup\"}}},{\"toolUse\":{\"input\":{\"q\":\"x\"},\"name\":\"lookup\",\"toolUseId\":\"call_1\"}}],\"role\":\"assistant\"},{\"content\":[{\"toolResult\":{\"content\":[{\"text\":\"found\"}],\"status\":\"success\",\"toolUseId\":\"call_1\"}},{\"cachePoint\":{\"type\":\"default\"}},{\"text\":\"Answer briefly\"}],\"role\":\"user\"}],\"system\":[{\"text\":\"You are concise.\"},{\"cachePoint\":{\"type\":\"default\"}},{\"text\":\"Mode: review\"}]}", + bedrock_developer = "{\"messages\":[{\"content\":[{\"text\":\"Hello\"}],\"role\":\"user\"},{\"content\":[{\"reasoningContent\":{\"reasoningText\":{\"signature\":\"sig-greet\",\"text\":\"greet\"}}},{\"text\":\"Hi\"}],\"role\":\"assistant\"},{\"content\":[{\"text\":\"Look it up\"},{\"text\":\"\\u003cdeveloper-instruction\\u003ePrefer cached data\\u003c/developer-instruction\\u003e\"}],\"role\":\"user\"},{\"content\":[{\"reasoningContent\":{\"reasoningText\":{\"signature\":\"sig-lookup\",\"text\":\"lookup\"}}},{\"toolUse\":{\"input\":{\"q\":\"x\"},\"name\":\"lookup\",\"toolUseId\":\"call_1\"}}],\"role\":\"assistant\"},{\"content\":[{\"toolResult\":{\"content\":[{\"text\":\"found\"}],\"status\":\"success\",\"toolUseId\":\"call_1\"}},{\"cachePoint\":{\"type\":\"default\"}},{\"text\":\"\\u003cdeveloper-instruction\\u003eAnswer briefly\\u003c/developer-instruction\\u003e\"}],\"role\":\"user\"}],\"system\":[{\"text\":\"You are concise.\"},{\"cachePoint\":{\"type\":\"default\"}},{\"text\":\"Mode: review\"}]}", openai_compat = "[{\"content\":[{\"text\":\"You are concise.\",\"type\":\"text\"}],\"role\":\"system\"},{\"content\":[{\"text\":\"Hello\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":\"Hi\",\"role\":\"assistant\"},{\"content\":[{\"text\":\"Look it up\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"text\":\"Prefer cached data\",\"type\":\"text\"}],\"role\":\"system\"},{\"content\":\"\",\"role\":\"assistant\",\"tool_calls\":[{\"function\":{\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\",\"name\":\"lookup\"},\"id\":\"call_1\",\"type\":\"function\"}]},{\"content\":\"found\",\"name\":\"lookup\",\"role\":\"tool\",\"tool_call_id\":\"call_1\"},{\"content\":[{\"text\":\"Mode: review\",\"type\":\"text\"}],\"role\":\"system\"},{\"content\":[{\"text\":\"Answer briefly\",\"type\":\"text\"}],\"role\":\"system\"}]", } From b7e647a598168c4ebb5d76d3e4874639dc06cac6 Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 17:32:47 -0500 Subject: [PATCH 13/37] fix(agent)!: require llm 0.5.13 and validate time_format The agent calls prompt.is_positional_note, so it requires wippy/llm >= v0.5.13; the test app lock records 0.5.13, resolved from the local llm source through the workspace replacement. time_aware rejects a time_format that is not a non-empty string. Tests cover stale-thinking stripping of checkpoint replacement histories. The README states that the fingerprint belongs on the rows that carry thinking blocks. BREAKING CHANGE: step responses no longer include memory_prompt; memory recall runs only when the host sets persist_context_notes and is returned as positional context_notes the host persists before the step output. The time_aware trait adds no prompt text, declares agent_options.turn_context for hosts to render with wippy.agent:turn_context, and rejects time_interval. A trait build function that returns an error fails compilation, and a failing prompt function fails the step. --- src/agent/src/README.md | 4 +- src/agent/src/_index.yaml | 2 +- src/agent/src/traits/init_time_aware.lua | 5 +- src/agent/test/agent_test.lua | 62 +++++++++++++++++++ .../test/traits/init_time_aware_test.lua | 11 ++++ src/agent/test/wippy.lock | 2 +- 6 files changed, 82 insertions(+), 4 deletions(-) diff --git a/src/agent/src/README.md b/src/agent/src/README.md index 1c0a40f3..10b7e3d4 100644 --- a/src/agent/src/README.md +++ b/src/agent/src/README.md @@ -336,7 +336,9 @@ The fingerprint is computed before the request, so the agent strips stale thinking blocks itself: 1. The host stores `response.prefix_fingerprint` as `metadata.prefix_fingerprint` - on the assistant and function call rows it persists from that step. + on every row it persists from that step that carries `thinking_blocks` (the + assistant row; function call messages built through `add_function_call` carry + no metadata, so blocks for a tool round live on its assistant row). 2. On later steps the host passes `drop_stale_thinking = true`. Before the request, the agent removes the blocks of every message up to and including the last one whose `metadata.prefix_fingerprint` differs from the current diff --git a/src/agent/src/_index.yaml b/src/agent/src/_index.yaml index 5fe8badf..9c562df1 100644 --- a/src/agent/src/_index.yaml +++ b/src/agent/src/_index.yaml @@ -37,7 +37,7 @@ entries: meta: description: "LLM component" component: "wippy/llm" - version: ">=v0.5.8" + version: ">=v0.5.13" # wippy.agent:agent - name: agent diff --git a/src/agent/src/traits/init_time_aware.lua b/src/agent/src/traits/init_time_aware.lua index bcd11a1c..f979f0db 100644 --- a/src/agent/src/traits/init_time_aware.lua +++ b/src/agent/src/traits/init_time_aware.lua @@ -18,12 +18,15 @@ local function execute(base_prompt: string?): (TimeAwareResult?, string?) -- The trait context merges with the agent context, so the layout key is -- namespaced: a generic "format" there belongs to someone else. local format = ctx.get("time_format") + if format ~= nil and (type(format) ~= "string" or format == "") then + return nil, "time_aware time_format must be a non-empty Go time layout string" + end return { agent_options = { turn_context = { time = { timezone = tostring(ctx.get("timezone") or "UTC"), - format = type(format) == "string" and format or nil, + format = format, }, }, }, diff --git a/src/agent/test/agent_test.lua b/src/agent/test/agent_test.lua index 9ed985dc..474df698 100644 --- a/src/agent/test/agent_test.lua +++ b/src/agent/test/agent_test.lua @@ -1627,6 +1627,68 @@ local function define_tests() test.eq(#requests, 0) end) + local function compacting_llm(captured) + return { + ERROR_TYPE = { CONTEXT_WINDOW = "context_window" }, + generate = function(_, options) + local replacement, err = options.checkpoint.compact({ estimated_prompt_tokens = 10 }) + captured.replacement, captured.err = replacement, err + return { result = "done", finish_reason = "stop" } + end + } + end + + local function compaction() + local replacement = prompt.new() + replacement:add_user("Retained task") + replacement:add_message(prompt.ROLE.ASSISTANT, { prompt.text("Retained answer") }, nil, + { thinking_blocks = blocks("retained"), prefix_fingerprint = "previous" }) + replacement:add_user("Next question") + return replacement + end + + local function retained_answer(messages) + for _, message in ipairs(messages) do + if message.role == prompt.ROLE.ASSISTANT then return message end + end + return nil + end + + it("strips stale blocks from a checkpoint replacement history", function() + local captured = {} + agent._llm = compacting_llm(captured) + local builder = prompt.new() + builder:add_user("Large conversation") + local result, err = agent.new(copy_spec()):step(builder, { + drop_stale_thinking = true, + checkpoint = { token_threshold = 1, compact = function() return compaction() end }, + }) + test.is_nil(err) + test.not_nil(result) + test.is_nil(captured.err) + local answer = assert(retained_answer(captured.replacement)) + test.is_nil(answer.metadata.thinking_blocks) + end) + + it("consults the host predicate for checkpoint replacement messages", function() + local captured = {} + local consulted = {} + agent._llm = compacting_llm(captured) + local builder = prompt.new() + builder:add_user("Large conversation") + assert(agent.new(copy_spec()):step(builder, { + drop_stale_thinking = true, + thinking_is_current = function(msg) + table.insert(consulted, msg.content[1].text) + return true + end, + checkpoint = { token_threshold = 1, compact = function() return compaction() end }, + })) + test.eq(#consulted, 1) + test.eq(consulted[1], "Retained answer") + test.eq(assert(retained_answer(captured.replacement)).metadata.thinking_blocks[1].thinking, "retained") + end) + it("never sends the stored fingerprint to the provider", function() local fingerprint = current_fingerprint() local builder = history({ thinking_blocks = blocks("kept"), prefix_fingerprint = fingerprint }) diff --git a/src/agent/test/traits/init_time_aware_test.lua b/src/agent/test/traits/init_time_aware_test.lua index 2f35db32..9ce33e32 100644 --- a/src/agent/test/traits/init_time_aware_test.lua +++ b/src/agent/test/traits/init_time_aware_test.lua @@ -32,6 +32,17 @@ local function define_tests() test.is_nil(result.agent_options.turn_context.time.format) end) + it("rejects a time_format that is not a non-empty string", function() + for _, layout in ipairs({ 42, true, "" }) do + local result, err = funcs.new() + :with_context({ timezone = "UTC", time_format = layout }) + :call("wippy.agent.traits:init_time_aware", "base prompt") + + test.is_nil(result) + test.contains(tostring(err), "time_format") + end + end) + it("rejects the removed time_interval setting and names its replacement", function() local result, err = funcs.new() :with_context({ timezone = "UTC", time_interval = 15 }) diff --git a/src/agent/test/wippy.lock b/src/agent/test/wippy.lock index 30e5664e..7bcb7825 100644 --- a/src/agent/test/wippy.lock +++ b/src/agent/test/wippy.lock @@ -3,7 +3,7 @@ directories: src: .. modules: - name: wippy/llm - version: 0.5.8 + version: 0.5.13 - name: wippy/security version: 0.4.2 hash: 9020016afc9de612f5297d38ce99e8500c63a2986077e8b0e560d8636c506627 From 170fa97ebf90a6bfabe410601d9fc91f1ea9a27e Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 17:33:50 -0500 Subject: [PATCH 14/37] fix(agent): require the llm release that carries positional notes The breaking llm changes in this branch release as wippy/llm 0.6.0, so the agent requires >=v0.6.0. --- src/agent/src/_index.yaml | 2 +- src/agent/test/agent_test.lua | 2 +- src/agent/test/wippy.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/agent/src/_index.yaml b/src/agent/src/_index.yaml index 9c562df1..89332e43 100644 --- a/src/agent/src/_index.yaml +++ b/src/agent/src/_index.yaml @@ -37,7 +37,7 @@ entries: meta: description: "LLM component" component: "wippy/llm" - version: ">=v0.5.13" + version: ">=v0.6.0" # wippy.agent:agent - name: agent diff --git a/src/agent/test/agent_test.lua b/src/agent/test/agent_test.lua index 474df698..36f0b0bd 100644 --- a/src/agent/test/agent_test.lua +++ b/src/agent/test/agent_test.lua @@ -1456,7 +1456,7 @@ local function define_tests() describe("Stable prefix", function() it("requires an llm package that provides positional notes", function() if type(prompt.is_positional_note) ~= "function" then - test.fail("wippy/llm >= 0.5.13 is required: prompt.is_positional_note is missing") + test.fail("wippy/llm >= 0.6.0 is required: prompt.is_positional_note is missing") end end) diff --git a/src/agent/test/wippy.lock b/src/agent/test/wippy.lock index 7bcb7825..c4fa25a2 100644 --- a/src/agent/test/wippy.lock +++ b/src/agent/test/wippy.lock @@ -3,7 +3,7 @@ directories: src: .. modules: - name: wippy/llm - version: 0.5.13 + version: 0.6.0 - name: wippy/security version: 0.4.2 hash: 9020016afc9de612f5297d38ce99e8500c63a2986077e8b0e560d8636c506627 From e1aee6ced40bf5328b56144ec939750775dadee6 Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 17:41:11 -0500 Subject: [PATCH 15/37] fix(agent): require llm 0.5.13 The llm changes in this branch release as a patch, wippy/llm 0.5.13. --- src/agent/src/_index.yaml | 2 +- src/agent/test/agent_test.lua | 2 +- src/agent/test/wippy.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/agent/src/_index.yaml b/src/agent/src/_index.yaml index 89332e43..9c562df1 100644 --- a/src/agent/src/_index.yaml +++ b/src/agent/src/_index.yaml @@ -37,7 +37,7 @@ entries: meta: description: "LLM component" component: "wippy/llm" - version: ">=v0.6.0" + version: ">=v0.5.13" # wippy.agent:agent - name: agent diff --git a/src/agent/test/agent_test.lua b/src/agent/test/agent_test.lua index 36f0b0bd..474df698 100644 --- a/src/agent/test/agent_test.lua +++ b/src/agent/test/agent_test.lua @@ -1456,7 +1456,7 @@ local function define_tests() describe("Stable prefix", function() it("requires an llm package that provides positional notes", function() if type(prompt.is_positional_note) ~= "function" then - test.fail("wippy/llm >= 0.6.0 is required: prompt.is_positional_note is missing") + test.fail("wippy/llm >= 0.5.13 is required: prompt.is_positional_note is missing") end end) diff --git a/src/agent/test/wippy.lock b/src/agent/test/wippy.lock index c4fa25a2..7bcb7825 100644 --- a/src/agent/test/wippy.lock +++ b/src/agent/test/wippy.lock @@ -3,7 +3,7 @@ directories: src: .. modules: - name: wippy/llm - version: 0.6.0 + version: 0.5.13 - name: wippy/security version: 0.4.2 hash: 9020016afc9de612f5297d38ce99e8500c63a2986077e8b0e560d8636c506627 From 9a0e42762556f9acbba2dbad3af8f8d9191a16a9 Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 17:56:22 -0500 Subject: [PATCH 16/37] fix(llm): keep the current tool round's thinking in drop_stale_thinking The last assistant turn with no user message after it is the round the request continues; the provider requires its thinking, so it is never stripped or consulted and stale blocks there are left to the provider's drop-on-mismatch handling. Earlier turns are stripped through the last stale one before it. --- src/llm/src/README.md | 2 +- src/llm/src/prompt.lua | 21 +++++++++++++++- src/llm/test/prompt_test.lua | 48 ++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/src/llm/src/README.md b/src/llm/src/README.md index 8fa3f03d..86571bec 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -242,7 +242,7 @@ local builder, err = builder:drop_stale_thinking(function(msg) end) ``` -An empty `thinking_blocks` list counts as no blocks; messages without blocks are not consulted. An assistant turn is the run of consecutive assistant and function call messages. If a turn whose blocks would be removed still has a function call without a result (a function call message or a `function_call` content part), the call fails and leaves the history unchanged: the provider needs those blocks to continue the tool round. +An empty `thinking_blocks` list counts as no blocks; messages without blocks are not consulted. The current round, the last assistant turn when no user message follows it (the history ends with its function results, or its calls are still pending), is never stripped and never consulted: the provider requires the final assistant turn of a tool round to keep its thinking. Stale blocks there are left to the provider's own drop-on-mismatch handling; once a later user turn exists, the turn is strippable like any other. An assistant turn is the run of consecutive assistant and function call messages. If a turn whose blocks would be removed still has a function call without a result (a function call message or a `function_call` content part), the call fails and leaves the history unchanged: the provider needs those blocks to continue the tool round. ### Builder Operations diff --git a/src/llm/src/prompt.lua b/src/llm/src/prompt.lua index 47ad31d0..bac83767 100644 --- a/src/llm/src/prompt.lua +++ b/src/llm/src/prompt.lua @@ -435,8 +435,27 @@ function prompt.new(messages: {Message}?) end end + -- The current round is the last assistant turn when no user message + -- follows it: the request continues that round, and the provider + -- requires the final assistant turn to keep its thinking. Its blocks + -- are never stripped here; a provider that can drop mismatched blocks + -- handles them. + local current_round = #messages + 1 + for index = #messages, 1, -1 do + local role = messages[index].role + if role == prompt.ROLE.USER then + break + end + if in_turn(messages[index]) then + current_round = index + elseif current_round <= #messages then + break + end + end + local last_stale: integer? = nil - for index, msg in ipairs(messages) do + for index = 1, current_round - 1 do + local msg = messages[index] if carries(msg) and not is_current(msg) then last_stale = index end diff --git a/src/llm/test/prompt_test.lua b/src/llm/test/prompt_test.lua index 8ccbf678..2d55720f 100644 --- a/src/llm/test/prompt_test.lua +++ b/src/llm/test/prompt_test.lua @@ -121,6 +121,7 @@ local function define_tests() builder:add_user("second") builder:add_function_call("lookup", "{}", "call_open") builder.messages[#builder.messages].metadata = { thinking_blocks = block("b") } + builder:add_user("later") local dropped, err = builder:drop_stale_thinking(function() return false end) test.is_nil(dropped) test.contains(tostring(err), "call_open") @@ -137,6 +138,7 @@ local function define_tests() builder:add_function_call("lookup", "{}", "call_done") builder:add_function_call("fetch", "{}", "call_open") builder:add_function_result("lookup", "found", "call_done") + builder:add_user("later") local dropped, err = builder:drop_stale_thinking(function() return false end) test.is_nil(dropped) test.contains(tostring(err), "call_open") @@ -151,6 +153,7 @@ local function define_tests() prompt.text("Looking it up"), { type = "function_call", id = "call_part", name = "lookup", arguments = "{}" }, }, nil, { thinking_blocks = block("a") }) + builder:add_user("later") local dropped, err = builder:drop_stale_thinking(function() return false end) test.is_nil(dropped) test.contains(tostring(err), "call_part") @@ -166,11 +169,56 @@ local function define_tests() builder:add_function_call("fetch", "{}", "call_2") builder:add_function_result("lookup", "found", "call_1") builder:add_function_result("fetch", "fetched", "call_2") + builder:add_user("later") local _, err = builder:drop_stale_thinking(function() return false end) test.is_nil(err) test.is_nil(builder:get_messages()[2].metadata.thinking_blocks) end) + local function current_round() + local builder = prompt.new() + builder:add_user("first") + builder:add_message(prompt.ROLE.ASSISTANT, { prompt.text("old answer") }, nil, + { thinking_blocks = block("earlier") }) + builder:add_user("second") + builder:add_message(prompt.ROLE.ASSISTANT, { prompt.text("Looking it up") }, nil, + { thinking_blocks = block("round") }) + builder:add_function_call("lookup", "{}", "call_1") + builder:add_function_result("lookup", "found", "call_1") + return builder + end + + it("keeps the current round's blocks when its results end the history", function() + local builder = current_round() + builder:add_developer("Reminder", { preserve_position = true }) + local _, err = builder:drop_stale_thinking(function() return false end) + test.is_nil(err) + local messages = builder:get_messages() + test.is_nil(messages[2].metadata.thinking_blocks) + test.eq(thinking_of(messages, 4), "round") + end) + + it("keeps the current round's blocks while its calls await results", function() + local builder = prompt.new() + builder:add_user("first") + builder:add_message(prompt.ROLE.ASSISTANT, { prompt.text("Looking it up") }, nil, + { thinking_blocks = block("round") }) + builder:add_function_call("lookup", "{}", "call_open") + local _, err = builder:drop_stale_thinking(function() return false end) + test.is_nil(err) + test.eq(thinking_of(builder:get_messages(), 2), "round") + end) + + it("strips the same turn once a later user turn exists", function() + local builder = current_round() + builder:add_user("third") + local _, err = builder:drop_stale_thinking(function() return false end) + test.is_nil(err) + local messages = builder:get_messages() + test.is_nil(messages[2].metadata.thinking_blocks) + test.is_nil(messages[4].metadata.thinking_blocks) + end) + it("leaves the caller's message tables unchanged when it strips", function() local persisted = { role = prompt.ROLE.ASSISTANT, content = { prompt.text("old") }, metadata = { thinking_blocks = block("a"), turn = 1 } } From 3ba177f75362b3d996d2b78be0e1bbe40b616718 Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 17:56:22 -0500 Subject: [PATCH 17/37] fix(agent): keep the current tool round's thinking and refresh the source lock Stale-thinking stripping leaves the current tool round intact. The package source lock records wippy/llm 0.5.13 resolved from the local llm source, matching the manifest requirement. --- src/agent/src/README.md | 5 +++++ src/agent/src/wippy.lock | 3 +-- src/agent/test/agent_test.lua | 26 ++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/agent/src/README.md b/src/agent/src/README.md index 10b7e3d4..f3a4d3b4 100644 --- a/src/agent/src/README.md +++ b/src/agent/src/README.md @@ -347,6 +347,11 @@ thinking blocks itself: `thinking_is_current = function(msg, fingerprint) ... end`, which replaces the default comparison. +The current tool round (the last assistant turn when no user message follows it) +keeps its blocks even when they are stale: the provider requires the final +assistant turn of a tool round to keep its thinking, so those blocks are left to +the provider's drop-on-mismatch handling, and only earlier turns are stripped. + Stripping works on copies: the host's message tables keep their blocks, and the stored fingerprint never reaches the provider. When a stale turn still has a function call without a result, the step fails with that error and sends no diff --git a/src/agent/src/wippy.lock b/src/agent/src/wippy.lock index ec258083..f47f0303 100644 --- a/src/agent/src/wippy.lock +++ b/src/agent/src/wippy.lock @@ -3,8 +3,7 @@ directories: src: . modules: - name: wippy/llm - version: 0.4.7 - hash: 2e55afcf55c441a8e79e3737691e5721d06138b9a3a46c7fb87199db6a5dd426 + version: 0.5.13 replacements: - from: wippy/llm to: ../../llm/src diff --git a/src/agent/test/agent_test.lua b/src/agent/test/agent_test.lua index 474df698..13fd840c 100644 --- a/src/agent/test/agent_test.lua +++ b/src/agent/test/agent_test.lua @@ -1618,6 +1618,7 @@ local function define_tests() metadata = { thinking_blocks = blocks("open"), prefix_fingerprint = "previous" } }, { role = prompt.ROLE.FUNCTION_CALL, content = {}, function_call = { id = "call_open", name = "calculator", arguments = "{}" } }, + { role = prompt.ROLE.USER, content = { prompt.text("Never mind") } }, }) local requests = {} agent._llm = recording_llm(requests) @@ -1627,6 +1628,31 @@ local function define_tests() test.eq(#requests, 0) end) + it("keeps the current tool round's blocks and strips earlier stale turns", function() + local builder = prompt.new({ + { role = prompt.ROLE.USER, content = { prompt.text("Earlier question") } }, + { role = prompt.ROLE.ASSISTANT, content = { prompt.text("Earlier answer") }, + metadata = { thinking_blocks = blocks("earlier"), prefix_fingerprint = "previous" } }, + { role = prompt.ROLE.USER, content = { prompt.text("Compute it") } }, + { role = prompt.ROLE.ASSISTANT, content = { prompt.text("Computing") }, + metadata = { thinking_blocks = blocks("round"), prefix_fingerprint = "previous" } }, + { role = prompt.ROLE.FUNCTION_CALL, content = {}, + function_call = { id = "call_1", name = "calculator", arguments = "{}" } }, + { role = prompt.ROLE.FUNCTION_RESULT, name = "calculator", function_call_id = "call_1", + content = { prompt.text("4") } }, + }) + local requests = {} + agent._llm = recording_llm(requests) + assert(agent.new(copy_spec()):step(builder, { drop_stale_thinking = true })) + local assistants = {} + for _, message in ipairs(requests[1]) do + if message.role == prompt.ROLE.ASSISTANT then table.insert(assistants, message) end + end + test.eq(#assistants, 2) + test.is_nil(assistants[1].metadata.thinking_blocks) + test.eq(assistants[2].metadata.thinking_blocks[1].thinking, "round") + end) + local function compacting_llm(captured) return { ERROR_TYPE = { CONTEXT_WINDOW = "context_window" }, From b0b529fa47f2e678d7f7f209850d93eab6982697 Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 18:33:17 -0500 Subject: [PATCH 18/37] fix(llm): a user message joining the tool-result turn keeps the round current drop_stale_thinking scans back over the trailing non-assistant messages; the last assistant turn stays the current round unless a user message follows it with no function result in between. Steering messages and the image message get_messages inserts after a tool result join the tool-result turn, so the round keeps its thinking. --- src/llm/src/README.md | 2 +- src/llm/src/prompt.lua | 29 ++++++++++++++---------- src/llm/test/prompt_test.lua | 43 +++++++++++++++++++++++++++++++++++- 3 files changed, 60 insertions(+), 14 deletions(-) diff --git a/src/llm/src/README.md b/src/llm/src/README.md index 86571bec..44438db6 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -242,7 +242,7 @@ local builder, err = builder:drop_stale_thinking(function(msg) end) ``` -An empty `thinking_blocks` list counts as no blocks; messages without blocks are not consulted. The current round, the last assistant turn when no user message follows it (the history ends with its function results, or its calls are still pending), is never stripped and never consulted: the provider requires the final assistant turn of a tool round to keep its thinking. Stale blocks there are left to the provider's own drop-on-mismatch handling; once a later user turn exists, the turn is strippable like any other. An assistant turn is the run of consecutive assistant and function call messages. If a turn whose blocks would be removed still has a function call without a result (a function call message or a `function_call` content part), the call fails and leaves the history unchanged: the provider needs those blocks to continue the tool round. +An empty `thinking_blocks` list counts as no blocks; messages without blocks are not consulted. The current round, the last assistant turn while the request continues it, is never stripped and never consulted: the provider requires the final assistant turn of a tool round to keep its thinking. The round continues while only its function results, notes and user messages that join the tool-result turn follow it (a steering message, or the image message `get_messages` inserts after a tool result with `_images`); a user message after the turn with no function result in between ends it. Stale blocks in the current round are left to the provider's own drop-on-mismatch handling; once a later assistant turn exists, or the turn is answered without tools and a user message follows, it is strippable like any other. An assistant turn is the run of consecutive assistant and function call messages. If a turn whose blocks would be removed still has a function call without a result (a function call message or a `function_call` content part), the call fails and leaves the history unchanged: the provider needs those blocks to continue the tool round. ### Builder Operations diff --git a/src/llm/src/prompt.lua b/src/llm/src/prompt.lua index bac83767..ad9d503f 100644 --- a/src/llm/src/prompt.lua +++ b/src/llm/src/prompt.lua @@ -435,21 +435,26 @@ function prompt.new(messages: {Message}?) end end - -- The current round is the last assistant turn when no user message - -- follows it: the request continues that round, and the provider - -- requires the final assistant turn to keep its thinking. Its blocks - -- are never stripped here; a provider that can drop mismatched blocks - -- handles them. + -- The current round is the last assistant turn while the request still + -- continues it: the provider requires the final assistant turn of a + -- tool round to keep its thinking. A user message after the round's + -- tool results joins that tool-result turn (steering, tool images), so + -- only a user message with no tool result after the turn ends it. The + -- round's blocks are never stripped here; a provider that can drop + -- mismatched blocks handles them. local current_round = #messages + 1 - for index = #messages, 1, -1 do + local seen_result, seen_user = false, false + local index = #messages + while index >= 1 and not in_turn(messages[index]) do local role = messages[index].role - if role == prompt.ROLE.USER then - break - end - if in_turn(messages[index]) then + if role == prompt.ROLE.FUNCTION_RESULT then seen_result = true end + if role == prompt.ROLE.USER then seen_user = true end + index = index - 1 + end + if index >= 1 and not (seen_user and not seen_result) then + while index >= 1 and in_turn(messages[index]) do current_round = index - elseif current_round <= #messages then - break + index = index - 1 end end diff --git a/src/llm/test/prompt_test.lua b/src/llm/test/prompt_test.lua index 2d55720f..01fd4d1b 100644 --- a/src/llm/test/prompt_test.lua +++ b/src/llm/test/prompt_test.lua @@ -138,6 +138,7 @@ local function define_tests() builder:add_function_call("lookup", "{}", "call_done") builder:add_function_call("fetch", "{}", "call_open") builder:add_function_result("lookup", "found", "call_done") + builder:add_assistant("Partial summary") builder:add_user("later") local dropped, err = builder:drop_stale_thinking(function() return false end) test.is_nil(dropped) @@ -169,6 +170,7 @@ local function define_tests() builder:add_function_call("fetch", "{}", "call_2") builder:add_function_result("lookup", "found", "call_1") builder:add_function_result("fetch", "fetched", "call_2") + builder:add_assistant("Summary") builder:add_user("later") local _, err = builder:drop_stale_thinking(function() return false end) test.is_nil(err) @@ -188,6 +190,44 @@ local function define_tests() return builder end + it("keeps the round when a user message joins its tool-result turn", function() + local builder = current_round() + builder:add_user("Use metric units") + local _, err = builder:drop_stale_thinking(function() return false end) + test.is_nil(err) + local messages = builder:get_messages() + test.is_nil(messages[2].metadata.thinking_blocks) + test.eq(thinking_of(messages, 4), "round") + end) + + it("keeps the round when its tool result carries images", function() + local builder = prompt.new() + builder:add_user("first") + builder:add_message(prompt.ROLE.ASSISTANT, { prompt.text("Taking a screenshot") }, nil, + { thinking_blocks = block("round") }) + builder:add_function_call("screenshot", "{}", "call_1") + builder:add_function_result("screenshot", + '{"result":"captured","_images":[{"url":"https://example.com/shot.png"}]}', "call_1") + local processed = prompt.new(builder:get_messages()) + local last = processed.messages[#processed.messages] + test.eq(last.role, prompt.ROLE.USER) + test.eq(last.content[1].type, "image") + local _, err = processed:drop_stale_thinking(function() return false end) + test.is_nil(err) + test.eq(thinking_of(processed:get_messages(), 2), "round") + end) + + it("strips a stale text-only turn that a user message follows", function() + local builder = prompt.new() + builder:add_user("first") + builder:add_message(prompt.ROLE.ASSISTANT, { prompt.text("answer") }, nil, + { thinking_blocks = block("stale") }) + builder:add_user("second") + local _, err = builder:drop_stale_thinking(function() return false end) + test.is_nil(err) + test.is_nil(builder:get_messages()[2].metadata.thinking_blocks) + end) + it("keeps the current round's blocks when its results end the history", function() local builder = current_round() builder:add_developer("Reminder", { preserve_position = true }) @@ -209,8 +249,9 @@ local function define_tests() test.eq(thinking_of(builder:get_messages(), 2), "round") end) - it("strips the same turn once a later user turn exists", function() + it("strips the same turn once a later assistant turn closes the round", function() local builder = current_round() + builder:add_assistant("Final answer") builder:add_user("third") local _, err = builder:drop_stale_thinking(function() return false end) test.is_nil(err) From 7d60edf5ac790bb19344928b0affe99fcd6e4d6a Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 18:33:17 -0500 Subject: [PATCH 19/37] test(agent): keep the current round when a user message joins its results Covers stale-thinking stripping when a user message follows the round's tool results and documents the rule. --- src/agent/src/README.md | 10 ++++++---- src/agent/test/agent_test.lua | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/agent/src/README.md b/src/agent/src/README.md index f3a4d3b4..7918d330 100644 --- a/src/agent/src/README.md +++ b/src/agent/src/README.md @@ -347,10 +347,12 @@ thinking blocks itself: `thinking_is_current = function(msg, fingerprint) ... end`, which replaces the default comparison. -The current tool round (the last assistant turn when no user message follows it) -keeps its blocks even when they are stale: the provider requires the final -assistant turn of a tool round to keep its thinking, so those blocks are left to -the provider's drop-on-mismatch handling, and only earlier turns are stripped. +The current tool round (the last assistant turn, while only its tool results, +notes and user messages joining the tool-result turn follow it) keeps its blocks +even when they are stale: the provider requires the final assistant turn of a +tool round to keep its thinking, so those blocks are left to the provider's +drop-on-mismatch handling, and only earlier turns are stripped. A user message +after an assistant turn with no tool result in between ends that turn. Stripping works on copies: the host's message tables keep their blocks, and the stored fingerprint never reaches the provider. When a stale turn still has a diff --git a/src/agent/test/agent_test.lua b/src/agent/test/agent_test.lua index 13fd840c..a2c32da9 100644 --- a/src/agent/test/agent_test.lua +++ b/src/agent/test/agent_test.lua @@ -1628,6 +1628,24 @@ local function define_tests() test.eq(#requests, 0) end) + it("keeps the current tool round's blocks when a user message joins its results", function() + local builder = prompt.new({ + { role = prompt.ROLE.USER, content = { prompt.text("Compute it") } }, + { role = prompt.ROLE.ASSISTANT, content = { prompt.text("Computing") }, + metadata = { thinking_blocks = blocks("round"), prefix_fingerprint = "previous" } }, + { role = prompt.ROLE.FUNCTION_CALL, content = {}, + function_call = { id = "call_1", name = "calculator", arguments = "{}" } }, + { role = prompt.ROLE.FUNCTION_RESULT, name = "calculator", function_call_id = "call_1", + content = { prompt.text("4") } }, + { role = prompt.ROLE.USER, content = { prompt.text("Use metric units") } }, + }) + local requests = {} + agent._llm = recording_llm(requests) + assert(agent.new(copy_spec()):step(builder, { drop_stale_thinking = true })) + local answer = assert(sent_answer(requests)) + test.eq(answer.metadata.thinking_blocks[1].thinking, "round") + end) + it("keeps the current tool round's blocks and strips earlier stale turns", function() local builder = prompt.new({ { role = prompt.ROLE.USER, content = { prompt.text("Earlier question") } }, From 2d6feed964c8d57d1540c61f077cb2f8c496c9cd Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 18:54:10 -0500 Subject: [PATCH 20/37] fix(llm): developer and positional notes end an answered assistant turn drop_stale_thinking counts any message that renders as user content (user, developer, positional system note) when deciding whether the last assistant turn is still the current round; hoisted system messages and cache markers do not. --- src/agent/src/README.md | 6 ++++-- src/llm/src/README.md | 2 +- src/llm/src/prompt.lua | 13 ++++++++---- src/llm/test/prompt_test.lua | 40 ++++++++++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 7 deletions(-) diff --git a/src/agent/src/README.md b/src/agent/src/README.md index 7918d330..6722aaf9 100644 --- a/src/agent/src/README.md +++ b/src/agent/src/README.md @@ -351,8 +351,10 @@ The current tool round (the last assistant turn, while only its tool results, notes and user messages joining the tool-result turn follow it) keeps its blocks even when they are stale: the provider requires the final assistant turn of a tool round to keep its thinking, so those blocks are left to the provider's -drop-on-mismatch handling, and only earlier turns are stripped. A user message -after an assistant turn with no tool result in between ends that turn. +drop-on-mismatch handling, and only earlier turns are stripped. User content +(a user message, a developer message or a positional system note) after an +assistant turn with no tool result in between ends that turn; ordinary system +messages, which drivers hoist into the top-level system, do not. Stripping works on copies: the host's message tables keep their blocks, and the stored fingerprint never reaches the provider. When a stale turn still has a diff --git a/src/llm/src/README.md b/src/llm/src/README.md index 44438db6..dee16632 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -242,7 +242,7 @@ local builder, err = builder:drop_stale_thinking(function(msg) end) ``` -An empty `thinking_blocks` list counts as no blocks; messages without blocks are not consulted. The current round, the last assistant turn while the request continues it, is never stripped and never consulted: the provider requires the final assistant turn of a tool round to keep its thinking. The round continues while only its function results, notes and user messages that join the tool-result turn follow it (a steering message, or the image message `get_messages` inserts after a tool result with `_images`); a user message after the turn with no function result in between ends it. Stale blocks in the current round are left to the provider's own drop-on-mismatch handling; once a later assistant turn exists, or the turn is answered without tools and a user message follows, it is strippable like any other. An assistant turn is the run of consecutive assistant and function call messages. If a turn whose blocks would be removed still has a function call without a result (a function call message or a `function_call` content part), the call fails and leaves the history unchanged: the provider needs those blocks to continue the tool round. +An empty `thinking_blocks` list counts as no blocks; messages without blocks are not consulted. The current round, the last assistant turn while the request continues it, is never stripped and never consulted: the provider requires the final assistant turn of a tool round to keep its thinking. The round continues while only its function results and user content that joins the tool-result turn follow it (a steering message, a developer or positional note, or the image message `get_messages` inserts after a tool result with `_images`). User content after the turn with no function result in between ends it: a user message, a developer message or a positional system note, which drivers render as a new user turn. Ordinary system messages, which drivers hoist into the top-level system, and cache markers do not. Stale blocks in the current round are left to the provider's own drop-on-mismatch handling; once a later assistant turn exists, or the turn is answered without tools and a user message follows, it is strippable like any other. An assistant turn is the run of consecutive assistant and function call messages. If a turn whose blocks would be removed still has a function call without a result (a function call message or a `function_call` content part), the call fails and leaves the history unchanged: the provider needs those blocks to continue the tool round. ### Builder Operations diff --git a/src/llm/src/prompt.lua b/src/llm/src/prompt.lua index ad9d503f..c0027bcd 100644 --- a/src/llm/src/prompt.lua +++ b/src/llm/src/prompt.lua @@ -437,9 +437,11 @@ function prompt.new(messages: {Message}?) -- The current round is the last assistant turn while the request still -- continues it: the provider requires the final assistant turn of a - -- tool round to keep its thinking. A user message after the round's - -- tool results joins that tool-result turn (steering, tool images), so - -- only a user message with no tool result after the turn ends it. The + -- tool round to keep its thinking. User content (user, developer and + -- positional notes) after the round's tool results joins that + -- tool-result turn (steering, tool images, reminders), so only user + -- content with no tool result after the turn ends it. System messages + -- that drivers hoist into the top-level system do not. The -- round's blocks are never stripped here; a provider that can drop -- mismatched blocks handles them. local current_round = #messages + 1 @@ -448,7 +450,10 @@ function prompt.new(messages: {Message}?) while index >= 1 and not in_turn(messages[index]) do local role = messages[index].role if role == prompt.ROLE.FUNCTION_RESULT then seen_result = true end - if role == prompt.ROLE.USER then seen_user = true end + if role == prompt.ROLE.USER or role == prompt.ROLE.DEVELOPER + or prompt.is_positional_note(messages[index]) then + seen_user = true + end index = index - 1 end if index >= 1 and not (seen_user and not seen_result) then diff --git a/src/llm/test/prompt_test.lua b/src/llm/test/prompt_test.lua index 01fd4d1b..2d8fe6b3 100644 --- a/src/llm/test/prompt_test.lua +++ b/src/llm/test/prompt_test.lua @@ -217,6 +217,46 @@ local function define_tests() test.eq(thinking_of(processed:get_messages(), 2), "round") end) + local function answered() + local builder = prompt.new() + builder:add_user("first") + builder:add_message(prompt.ROLE.ASSISTANT, { prompt.text("answer") }, nil, + { thinking_blocks = block("stale") }) + return builder + end + + it("strips a stale text-only turn that a developer note follows", function() + local builder = answered() + builder:add_developer("Lifecycle note") + local _, err = builder:drop_stale_thinking(function() return false end) + test.is_nil(err) + test.is_nil(builder:get_messages()[2].metadata.thinking_blocks) + end) + + it("strips a stale text-only turn that a positional system note follows", function() + local builder = answered() + builder:add_system("Stopped by the user", { preserve_position = true }) + local _, err = builder:drop_stale_thinking(function() return false end) + test.is_nil(err) + test.is_nil(builder:get_messages()[2].metadata.thinking_blocks) + end) + + it("keeps the round when a developer note follows its tool results", function() + local builder = current_round() + builder:add_developer("Reminder") + local _, err = builder:drop_stale_thinking(function() return false end) + test.is_nil(err) + test.eq(thinking_of(builder:get_messages(), 4), "round") + end) + + it("keeps the last turn current when only a hoisted system message follows", function() + local builder = answered() + builder:add_system("Mode: review") + local _, err = builder:drop_stale_thinking(function() return false end) + test.is_nil(err) + test.eq(thinking_of(builder:get_messages(), 2), "stale") + end) + it("strips a stale text-only turn that a user message follows", function() local builder = prompt.new() builder:add_user("first") From 855ee6bac49db947843406820a7f68bda2b462dc Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 19:20:35 -0500 Subject: [PATCH 21/37] fix(agent): fingerprint the request a checkpoint replacement sends A checkpoint replacement history recomputes prefix_fingerprint before its thinking blocks are judged, so the returned fingerprint matches the prefix actually sent. The lifecycle contract states that per-step lifecycle messages change the prefix unless the host persists them in place as positional notes. --- src/agent/src/README.md | 4 +++- src/agent/src/_index.yaml | 12 ++++++---- src/agent/src/agent.lua | 28 ++++++++++++++++++------ src/agent/test/agent_test.lua | 41 +++++++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 12 deletions(-) diff --git a/src/agent/src/README.md b/src/agent/src/README.md index 6722aaf9..3ccab060 100644 --- a/src/agent/src/README.md +++ b/src/agent/src/README.md @@ -318,7 +318,9 @@ messages in the conversation, which Claude and Bedrock hoist into the top-level system, and each tool's name, description, schema and native definition (`meta.claude`). Registry ids and registry metadata (comment, tags, title) do not reach the provider and are not part of it. It is stable across steps while -the sent system text and tools stay the same. +the sent system text and tools stay the same. It describes the request actually sent: +when a checkpoint callback replaces the history, the fingerprint is recomputed +from the replacement, and stale thinking in the replacement is judged against it. The projection is the Claude and Bedrock prefix. Other drivers bind a different one: OpenAI moves developer messages into `instructions`, and Google filters tool diff --git a/src/agent/src/_index.yaml b/src/agent/src/_index.yaml index 9c562df1..1a10c827 100644 --- a/src/agent/src/_index.yaml +++ b/src/agent/src/_index.yaml @@ -445,10 +445,14 @@ entries: get_history, or get_prompt with the supplied host. Returned messages are appended to the current prompt only by phases that - run before an LLM step (activate and before_step today). They are - per-step context: developer messages sit after the conversation cache - breakpoint, so they never alter the cached prompt prefix. Durable memory, - checkpoint, or audit writes belong inside the binding implementation. + run before an LLM step (activate and before_step today). The provider + prefix includes every message sent: a lifecycle message the host sends + for one step and then drops or rebuilds changes the prefix of the next + request, restarting the prompt cache and invalidating later thinking + blocks. Hosts persist returned messages in place as positional notes + (metadata.preserve_position) so later requests replay them unchanged. + Durable memory, checkpoint, or audit writes belong inside the binding + implementation. Multiple bindings may be attached by multiple traits; the compiler orders them by priority and declaration order. diff --git a/src/agent/src/agent.lua b/src/agent/src/agent.lua index cac6ed18..3d61b505 100644 --- a/src/agent/src/agent.lua +++ b/src/agent/src/agent.lua @@ -652,13 +652,23 @@ function agent:step(prompt_builder: any, runtime_options: any): (table?, string? return final_messages end + -- The fingerprint describes the request actually sent, so a checkpoint + -- replacement history recomputes it before its blocks are judged. + local function fingerprint_of(messages: {any}): (string?, any) + local projected = { system_message } + for _, message in ipairs(messages) do table.insert(projected, message) end + local digest, digest_err = hash.sha256(canonical.encode(prefix_projection(projected, options.tools))) + if digest_err then + return nil, output.errors.generate():kind(output.ERROR_TYPE.INVALID_REQUEST) + :message("Prefix fingerprint failed: " .. tostring(digest_err)):build() + end + return digest, nil + end + local conversation = prompt_builder:get_messages() - local projected = { system_message } - for _, message in ipairs(conversation) do table.insert(projected, message) end - local fingerprint, fingerprint_err = hash.sha256(canonical.encode(prefix_projection(projected, options.tools))) - if fingerprint_err then - return nil, output.errors.generate():kind(output.ERROR_TYPE.INVALID_REQUEST) - :message("Prefix fingerprint failed: " .. tostring(fingerprint_err)):build() + local fingerprint, fingerprint_err = fingerprint_of(conversation) + if not fingerprint then + return nil, fingerprint_err end -- Thinking blocks are bound to the prefix they were produced under. With @@ -695,7 +705,11 @@ function agent:step(prompt_builder: any, runtime_options: any): (table?, string? compact = function(info) local replacement, compact_err = runtime_options.checkpoint.compact(info) if not replacement then return nil, compact_err end - local retained, retained_err = current_thinking(replacement:get_messages()) + local replacement_messages = replacement:get_messages() + local replacement_fingerprint, replacement_err = fingerprint_of(replacement_messages) + if not replacement_fingerprint then return nil, replacement_err end + fingerprint = replacement_fingerprint + local retained, retained_err = current_thinking(replacement_messages) if not retained then return nil, retained_err end return build_messages(retained) end, diff --git a/src/agent/test/agent_test.lua b/src/agent/test/agent_test.lua index a2c32da9..61eb45ed 100644 --- a/src/agent/test/agent_test.lua +++ b/src/agent/test/agent_test.lua @@ -1698,6 +1698,47 @@ local function define_tests() return nil end + it("returns the fingerprint of the request a checkpoint replacement sends", function() + local function replacement_history() + local replacement = prompt.new() + replacement:add_system("Context memory: release plan v2") + replacement:add_user("Retained task") + return replacement + end + local captured = {} + agent._llm = compacting_llm(captured) + local builder = prompt.new() + builder:add_user("Large conversation") + local compacted = assert(agent.new(copy_spec()):step(builder, { + checkpoint = { token_threshold = 1, compact = function() return replacement_history() end }, + })) + test.is_nil(captured.err) + + agent._llm = recording_llm({}) + local direct = assert(agent.new(copy_spec()):step(replacement_history())) + local plain_builder = prompt.new() + plain_builder:add_user("Large conversation") + local plain = assert(agent.new(copy_spec()):step(plain_builder)) + + test.eq(compacted.prefix_fingerprint, direct.prefix_fingerprint) + test.neq(compacted.prefix_fingerprint, plain.prefix_fingerprint) + end) + + it("keeps the fingerprint of a checkpointed step that does not compact", function() + agent._llm = { + ERROR_TYPE = { CONTEXT_WINDOW = "context_window" }, + generate = function() return { result = "done", finish_reason = "stop" } end + } + local builder = prompt.new() + builder:add_user("Small conversation") + local checkpointed = assert(agent.new(copy_spec()):step(builder, { + checkpoint = { token_threshold = 1000, compact = function() return nil, "unused" end }, + })) + agent._llm = recording_llm({}) + local plain = assert(agent.new(copy_spec()):step(builder)) + test.eq(checkpointed.prefix_fingerprint, plain.prefix_fingerprint) + end) + it("strips stale blocks from a checkpoint replacement history", function() local captured = {} agent._llm = compacting_llm(captured) From 66666858b962dbeeca4f11f78c1e01920dfecbd5 Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 19:45:24 -0500 Subject: [PATCH 22/37] fix(agent)!: recall memory without a host opt-in Memory recall runs whenever the agent has a memory contract, as before this branch. The recall note is sent at the conversation tail as a positional developer note and returned in context_notes on normal, truncated and cancelled results for the host to persist in place. The persist_context_notes runtime option is removed. BREAKING CHANGE: step responses return the memory recall note in context_notes instead of memory_prompt; hosts persist each context note after the messages they passed and before the step output. --- src/agent/src/README.md | 11 ++-- src/agent/src/agent.lua | 7 --- src/agent/test/agent_memory_test.lua | 87 +++++++++++++--------------- src/agent/test/agent_test.lua | 68 +++++++++++----------- 4 files changed, 79 insertions(+), 94 deletions(-) diff --git a/src/agent/src/README.md b/src/agent/src/README.md index 3ccab060..d3540d1b 100644 --- a/src/agent/src/README.md +++ b/src/agent/src/README.md @@ -224,7 +224,6 @@ local response, err = runner:step(builder, { stream_target = { reply_to = process.self(), topic = "stream" }, tool_call = "auto", disable_memory_recall = false, - persist_context_notes = true, -- the host persists response.context_notes in place drop_stale_thinking = true, -- strip blocks bound to another prefix before the request thinking_is_current = nil, -- optional function(msg, fingerprint) -> boolean previous_memory_ids = { "mem_1", "mem_2" }, @@ -304,11 +303,11 @@ note that is sent once and then dropped or rebuilt changes the prefix before the output it produced; that restarts the prompt cache and invalidates every later thinking block, not only the cache. -Memory recall runs only when the host sets `persist_context_notes = true`. A host -that does not persist the notes would send a note once and drop it from the next -request, which edits the history before the output it produced; such a host gets -no recall. The recall cooldown reads `metadata.memory_ids` from the persisted -notes. +Memory recall runs whenever the agent has a memory contract and recall is not +disabled; its note is a context note. A host that does not persist the notes +sends each note once and drops it from the next request, which edits the history +before the output it produced. The recall cooldown reads `metadata.memory_ids` +from the persisted notes. ### Prefix fingerprint diff --git a/src/agent/src/agent.lua b/src/agent/src/agent.lua index 3d61b505..9e0c5831 100644 --- a/src/agent/src/agent.lua +++ b/src/agent/src/agent.lua @@ -336,13 +336,6 @@ local function perform_memory_recall(self: any, prompt_builder: any, runtime_opt return nil, nil end - -- The recall note is sent at the conversation tail, so it is part of the - -- prefix every later request replays. Only a host that persists the - -- returned context notes at that position keeps the prefix stable. - if not (runtime_options and runtime_options.persist_context_notes == true) then - return nil, nil - end - local messages = prompt_builder:get_messages() local options = get_memory_options(self.memory_contract) diff --git a/src/agent/test/agent_memory_test.lua b/src/agent/test/agent_memory_test.lua index 3baf21cb..a7f78804 100644 --- a/src/agent/test/agent_memory_test.lua +++ b/src/agent/test/agent_memory_test.lua @@ -3,9 +3,6 @@ local prompt = require("prompt") local json = require("json") local time = require("time") --- Recall runs only for hosts that persist the context notes a step returns. -local HOST = { persist_context_notes = true } - local function define_tests() describe("Agent Memory Processing - Detailed Logic", function() local mock_llm @@ -210,7 +207,7 @@ local function define_tests() local builder = mock_prompt.new() builder:add_user("first") builder:add_assistant("second") - local result, err = instance:step(builder, HOST) + local result, err = instance:step(builder) test.is_nil(err) test.not_nil(result) test.eq(#mock_contract._open_calls, 0) @@ -242,7 +239,7 @@ local function define_tests() prompt_builder:add_function_result("calculator", "Result: 4", "call_123") prompt_builder:add_user("What about 3 + 3?") - test_agent:step(prompt_builder, HOST) + test_agent:step(prompt_builder) -- Inspect what was passed to memory contract local call_log = memory_contract:get_call_log() @@ -282,7 +279,7 @@ local function define_tests() prompt_builder:add_assistant("Response 2") prompt_builder:add_user("Message 3") - test_agent:step(prompt_builder, HOST) + test_agent:step(prompt_builder) local call_log = memory_contract:get_call_log() local recent_actions = (call_log[1] :: any).recent_actions @@ -319,7 +316,7 @@ local function define_tests() prompt_builder:add_user("User message 2") prompt_builder:add_assistant("Assistant response 2") - test_agent:step(prompt_builder, HOST) + test_agent:step(prompt_builder) local call_log = memory_contract:get_call_log() local recent_actions = (call_log[1] :: any).recent_actions @@ -356,7 +353,7 @@ local function define_tests() table.insert(messages, { role = "assistant", content = {} }) -- Empty content table.insert(messages, { role = "user", content = {{ type = "text", text = "" }} }) -- Empty text - test_agent:step(prompt_builder, HOST) + test_agent:step(prompt_builder) local call_log = memory_contract:get_call_log() local recent_actions = (call_log[1] :: any).recent_actions @@ -400,7 +397,7 @@ local function define_tests() { type = "image", source = { type = "url", url = "https://example.com/a.png" } } })) - test_agent:step(prompt_builder, HOST) + test_agent:step(prompt_builder) local call_log = memory_contract:get_call_log() test.eq(#call_log, 1) @@ -444,7 +441,7 @@ local function define_tests() prompt_builder:add_user("Current message") - test_agent:step(prompt_builder, HOST) + test_agent:step(prompt_builder) local call_log = memory_contract:get_call_log() local previous_memories = (call_log[1] :: any).previous_memories @@ -484,7 +481,7 @@ local function define_tests() messages[#messages].metadata = { memory_ids = { "mem_" .. i } } end - test_agent:step(prompt_builder, HOST) + test_agent:step(prompt_builder) local call_log = memory_contract:get_call_log() @@ -526,7 +523,6 @@ local function define_tests() -- Pass runtime previous_memory_ids - FIXED: Use correct 2-parameter signature local runtime_options = { - persist_context_notes = true, previous_memory_ids = { "mem_from_runtime_1", "mem_from_runtime_2" } } @@ -574,7 +570,7 @@ local function define_tests() prompt_builder:add_user("Message " .. i) end - test_agent:step(prompt_builder, HOST) + test_agent:step(prompt_builder) local call_log = memory_contract:get_call_log() local constraints = (call_log[1] :: any).constraints @@ -626,7 +622,7 @@ local function define_tests() prompt_builder:add_user("Message " .. i) end - local result = test_agent:step(prompt_builder, HOST) + local result = test_agent:step(prompt_builder) test.not_nil(result) local call_log = memory_contract:get_call_log() @@ -666,7 +662,7 @@ local function define_tests() prompt_builder:add_assistant("Response 1") prompt_builder:add_user("Message 2") - local result = test_agent:step(prompt_builder, HOST) + local result = test_agent:step(prompt_builder) test.not_nil(result.memory_recall) test.not_nil(memory_contract:get_call_log()) @@ -695,7 +691,7 @@ local function define_tests() prompt_builder:add_user("Message 1") prompt_builder:add_assistant("Response 1") - local result = test_agent:step(prompt_builder, HOST) + local result = test_agent:step(prompt_builder) test.is_nil(result.memory_recall) test.eq(#memory_contract:get_call_log(), 0) @@ -735,7 +731,7 @@ local function define_tests() prompt_builder:add_user("Message 2") prompt_builder:add_assistant("Response 2") - local result = test_agent:step(prompt_builder, HOST) + local result = test_agent:step(prompt_builder) test.is_nil(result.memory_recall) -- Should not recall due to cooldown test.eq(#memory_contract:get_call_log(), 0) @@ -772,7 +768,7 @@ local function define_tests() prompt_builder:add_user("Message 2") prompt_builder:add_assistant("Response 2") - local result = test_agent:step(prompt_builder, HOST) + local result = test_agent:step(prompt_builder) test.not_nil(result.memory_recall) -- Should recall after cooldown test.eq(#memory_contract:get_call_log(), 1) @@ -801,7 +797,7 @@ local function define_tests() prompt_builder:add_user("Message 2") -- Disable memory recall via runtime options - FIXED: Use correct 2-parameter signature - local result = test_agent:step(prompt_builder, { disable_memory_recall = true, persist_context_notes = true }) + local result = test_agent:step(prompt_builder, { disable_memory_recall = true }) test.is_nil(result.memory_recall) test.eq(#memory_contract:get_call_log(), 0) @@ -821,7 +817,7 @@ local function define_tests() builder:add_user("Rebuild the Word handout and finish every edit.") builder:add_assistant("The first edit is complete.") builder:add_function_result("edit_document", "Second edit stored", "edit-2") - assert(instance:step(builder, HOST)) + assert(instance:step(builder)) local recall = test_messages[#test_messages] test.eq(recall.role, "developer") @@ -867,7 +863,7 @@ local function define_tests() prompt_builder:add_assistant("Response 1") prompt_builder:add_user("Message 2") - local result = test_agent:step(prompt_builder, HOST) + local result = test_agent:step(prompt_builder) test.not_nil(result.memory_recall) test.eq(#result.context_notes, 1) @@ -908,7 +904,7 @@ local function define_tests() prompt_builder:add_assistant("Response 1") prompt_builder:add_user("Message 2") - local result = test_agent:step(prompt_builder, HOST) + local result = test_agent:step(prompt_builder) test.is_nil(result.memory_recall) -- No memories returned test.is_nil(result.context_notes) @@ -940,7 +936,7 @@ local function define_tests() prompt_builder:add_assistant("Response 1") prompt_builder:add_user("Trigger memory") - test_agent:step(prompt_builder, HOST) + test_agent:step(prompt_builder) -- Check that memory was injected into LLM messages local llm_messages = test_messages @@ -991,7 +987,6 @@ local function define_tests() -- Pass runtime context - FIXED: Use correct 2-parameter signature local runtime_options = { - persist_context_notes = true, context = { session_id = "test_session_123", shared_key = "from_runtime_context" -- Should override contract context @@ -1034,7 +1029,7 @@ local function define_tests() prompt_builder:add_assistant("Response 1") prompt_builder:add_user("Message 2") - test_agent:step(prompt_builder, HOST) + test_agent:step(prompt_builder) local call_log = memory_contract:get_call_log() test.eq(#call_log, 1) @@ -1061,7 +1056,7 @@ local function define_tests() }) do local builder = mock_prompt.new() builder:add_user("Recall this run's memories") - local result, err = instance:step(builder, { context = identity, persist_context_notes = true }) + local result, err = instance:step(builder, { context = identity }) test.is_nil(err) test.not_nil(result) local opened = mock_contract._open_calls[#mock_contract._open_calls] @@ -1101,7 +1096,7 @@ local function define_tests() prompt_builder:add_user("Message 2") -- Should not crash, just skip memory recall - local result = test_agent:step(prompt_builder, HOST) + local result = test_agent:step(prompt_builder) test.not_nil(result) test.is_nil(result.memory_recall) @@ -1133,7 +1128,7 @@ local function define_tests() prompt_builder:add_user("Message 2") -- Should not crash, just skip memory recall - local result = test_agent:step(prompt_builder, HOST) + local result = test_agent:step(prompt_builder) test.not_nil(result) test.is_nil(result.memory_recall) @@ -1168,7 +1163,7 @@ local function define_tests() prompt_builder:add_user("Message 2") -- Should handle gracefully - local result = test_agent:step(prompt_builder, HOST) + local result = test_agent:step(prompt_builder) test.not_nil(result) test.is_nil(result.memory_recall) @@ -1212,7 +1207,7 @@ local function define_tests() prompt_builder:add_assistant("Response 1") prompt_builder:add_user("Message 2") - local result = test_agent:step(prompt_builder, HOST) + local result = test_agent:step(prompt_builder) -- Should handle large dataset test.not_nil(result) @@ -1263,7 +1258,7 @@ local function define_tests() it("reports a provider recall error and still generates", function() mock_contract._current_instance = recalling(nil, "vector store offline") - local result, err = agent.new(recall_spec):step(conversation(), HOST) + local result, err = agent.new(recall_spec):step(conversation()) test.is_nil(err) test.eq(result.result, "Mock LLM response") @@ -1276,7 +1271,7 @@ local function define_tests() it("reports a provider that cannot be opened", function() mock_contract._current_error = "binding not found" - local result, err = agent.new(recall_spec):step(conversation(), HOST) + local result, err = agent.new(recall_spec):step(conversation()) test.is_nil(err) test.not_nil(result.memory_recall_error:find("binding not found", 1, true)) @@ -1286,7 +1281,7 @@ local function define_tests() mock_contract._current_instance = nil mock_contract._current_error = nil - local result, err = agent.new(recall_spec):step(conversation(), HOST) + local result, err = agent.new(recall_spec):step(conversation()) test.is_nil(err) test.not_nil(result.memory_recall_error) @@ -1295,7 +1290,7 @@ local function define_tests() it("reports a memory contract that cannot be resolved", function() agent._contract = { get = function() return nil, "contract registry unavailable" end } - local result, err = agent.new(recall_spec):step(conversation(), HOST) + local result, err = agent.new(recall_spec):step(conversation()) test.is_nil(err) test.not_nil(result.memory_recall_error:find("contract registry unavailable", 1, true)) @@ -1311,7 +1306,7 @@ local function define_tests() } for _, case in ipairs(malformed) do mock_contract._current_instance = recalling(case.value, nil) - local result, err = agent.new(recall_spec):step(conversation(), HOST) + local result, err = agent.new(recall_spec):step(conversation()) test.is_nil(err, case.label) test.not_nil(result.memory_recall_error, case.label) test.is_false(memory_prompt_sent(), case.label) @@ -1320,18 +1315,18 @@ local function define_tests() it("reports no error for an empty recall or a skipped recall", function() mock_contract._current_instance = recalling({ memories = {} }, nil) - local result = agent.new(recall_spec):step(conversation(), HOST) + local result = agent.new(recall_spec):step(conversation()) test.is_nil(result.memory_recall_error) - result = agent.new(recall_spec):step(conversation(), { disable_memory_recall = true, persist_context_notes = true }) + result = agent.new(recall_spec):step(conversation(), { disable_memory_recall = true }) test.is_nil(result.memory_recall_error) local short = mock_prompt.new() short:add_user("Only message") - result = agent.new(recall_spec):step(short, HOST) + result = agent.new(recall_spec):step(short) test.is_nil(result.memory_recall_error) - result = agent.new({ id = "plain", name = "Plain", prompt = "p", tools = {} }):step(conversation(), HOST) + result = agent.new({ id = "plain", name = "Plain", prompt = "p", tools = {} }):step(conversation()) test.is_nil(result.memory_recall_error) end) @@ -1339,7 +1334,7 @@ local function define_tests() mock_contract._current_instance = recalling(nil, "vector store offline") agent._llm = { generate = function() return nil, "provider overloaded" end } - local result, err = agent.new(recall_spec):step(conversation(), HOST) + local result, err = agent.new(recall_spec):step(conversation()) test.eq(err, "provider overloaded") test.not_nil(result) @@ -1354,7 +1349,7 @@ local function define_tests() message = "prompt exceeds context window", details = details }) agent._llm = { generate = function() return nil, cause, details end } - local result, err, admission = agent.new(recall_spec):step(conversation(), HOST) + local result, err, admission = agent.new(recall_spec):step(conversation()) test.eq(err, cause) test.eq(admission, details) @@ -1368,19 +1363,19 @@ local function define_tests() agent._llm = { generate = function() return { result = "", finish_reason = "length", tokens = {} } end } - local truncated = agent.new(recall_spec):step(conversation(), HOST) + local truncated = agent.new(recall_spec):step(conversation()) test.is_true(truncated.truncated) test.not_nil(truncated.memory_recall_error) agent._llm = { generate = function() return { result = "partial", finish_reason = "cancelled", tokens = {} } end } - local cancelled = agent.new(recall_spec):step(conversation(), HOST) + local cancelled = agent.new(recall_spec):step(conversation()) test.is_true(cancelled.stopped) test.not_nil(cancelled.memory_recall_error) agent._llm = mock_llm - local stopped = agent.new(recall_spec):step(conversation(), { stop_check = function() return true end, persist_context_notes = true }) + local stopped = agent.new(recall_spec):step(conversation(), { stop_check = function() return true end }) test.is_true(stopped.stopped) test.not_nil(stopped.memory_recall_error) end) @@ -1414,7 +1409,7 @@ local function define_tests() prompt_builder:add_assistant("First response") prompt_builder:add_user("Second message") - test_agent:step(prompt_builder, HOST) + test_agent:step(prompt_builder) -- Verify message order in LLM input local llm_messages = test_messages @@ -1468,7 +1463,7 @@ local function define_tests() local original_message_count = #prompt_builder:get_messages() - test_agent:step(prompt_builder, HOST) + test_agent:step(prompt_builder) -- Original prompt builder should be unchanged test.eq(#prompt_builder:get_messages(), original_message_count) diff --git a/src/agent/test/agent_test.lua b/src/agent/test/agent_test.lua index 61eb45ed..134231ce 100644 --- a/src/agent/test/agent_test.lua +++ b/src/agent/test/agent_test.lua @@ -974,7 +974,7 @@ local function define_tests() prompt_builder:add_assistant("Of course!") prompt_builder:add_user("What's 2 + 2?") - local result = test_agent:step(prompt_builder, { persist_context_notes = true }) + local result = test_agent:step(prompt_builder) test.not_nil(result) test.not_nil(result.memory_recall) @@ -988,33 +988,32 @@ local function define_tests() test.is_nil(result.memory_prompt) end) - it("does not consult memory when the host does not persist context notes", function() - local opened = 0 - agent._contract = { - get = function() - return { open = function() opened = opened + 1 end } - end - } - local sent: any = nil - agent._llm = { - generate = function(messages) - sent = messages - return { result = "ok", finish_reason = "stop" } - end + it("recalls without any host option and returns the note on every result", function() + local replies = { + { result = "ok", finish_reason = "stop" }, + { result = "partial", finish_reason = "length", + tool_calls = { { id = "call_1", name = "calculator", arguments = {} } } }, + { result = "partial", finish_reason = "cancelled" }, } - local test_agent = agent.new(compiled_spec_with_memory) - local prompt_builder = prompt.new() - prompt_builder:add_user("Hello") - prompt_builder:add_assistant("Hi there!") - prompt_builder:add_user("Can you help me calculate something?") + for _, reply in ipairs(replies) do + local sent: any = nil + agent._llm = { + generate = function(messages) + sent = messages + return reply + end + } + local test_agent = agent.new(compiled_spec_with_memory) + local prompt_builder = prompt.new() + prompt_builder:add_user("Hello") + prompt_builder:add_assistant("Hi there!") + prompt_builder:add_user("Can you help me calculate something?") - local result = test_agent:step(prompt_builder) + local result = assert(test_agent:step(prompt_builder)) - test.eq(opened, 0) - test.is_nil(result.memory_recall) - test.is_nil(result.context_notes) - for _, message in ipairs(sent) do - test.is_true(message.role ~= prompt.ROLE.DEVELOPER) + test.eq(#result.context_notes, 1, reply.finish_reason) + test.eq(sent[#sent], result.context_notes[1], reply.finish_reason) + test.is_true(result.context_notes[1].metadata.preserve_position, reply.finish_reason) end end) @@ -1032,7 +1031,7 @@ local function define_tests() prompt_builder:add_assistant("Hi there!") prompt_builder:add_user("Can you help me calculate something?") - local result = test_agent:step(prompt_builder, { persist_context_notes = true }) + local result = test_agent:step(prompt_builder) test.eq(#result.context_notes, 1) local n = #sent @@ -1049,7 +1048,7 @@ local function define_tests() -- Only one message - below min_conversation_length prompt_builder:add_user("Hello") - local result = test_agent:step(prompt_builder, { persist_context_notes = true }) + local result = test_agent:step(prompt_builder) test.not_nil(result) test.is_nil(result.memory_recall) @@ -1068,7 +1067,7 @@ local function define_tests() prompt_builder:add_user("What's 2 + 2?") -- Disable memory recall via runtime options - local result = test_agent:step(prompt_builder, { disable_memory_recall = true, persist_context_notes = true }) + local result = test_agent:step(prompt_builder, { disable_memory_recall = true }) test.not_nil(result) test.is_nil(result.memory_recall) @@ -1096,7 +1095,7 @@ local function define_tests() prompt_builder:add_user("Test memory") -- Should not fail, just skip memory recall - local result = test_agent:step(prompt_builder, { persist_context_notes = true }) + local result = test_agent:step(prompt_builder) test.not_nil(result) test.is_nil(result.memory_recall) @@ -1118,7 +1117,7 @@ local function define_tests() -- Add only one more message (cooldown = 2, so should not trigger) prompt_builder:add_user("New question") - local result = test_agent:step(prompt_builder, { persist_context_notes = true }) + local result = test_agent:step(prompt_builder) test.not_nil(result) test.is_nil(result.memory_recall) -- Should not recall due to cooldown @@ -1152,7 +1151,6 @@ local function define_tests() -- Pass session context via runtime options local result = test_agent:step(prompt_builder, { - persist_context_notes = true, context = { user_id = "test_user_123", session_id = "session_456" @@ -1425,7 +1423,7 @@ local function define_tests() local step1 = prompt.new() step1:add_user("Plan the release") step1:add_developer("Lifecycle note for step 1") - local result1, err1 = runner:step(step1, { context = {}, persist_context_notes = true }) + local result1, err1 = runner:step(step1, { context = {} }) test.is_nil(err1) local step2 = prompt.new() @@ -1436,7 +1434,7 @@ local function define_tests() step2:add_function_call("calculator", '{"expression":"2+2"}', "call_1") step2:add_function_result("calculator", "4", "call_1") step2:add_developer("Lifecycle note for step 2") - local _, err2 = runner:step(step2, { context = {}, persist_context_notes = true }) + local _, err2 = runner:step(step2, { context = {} }) test.is_nil(err2) test.eq(#requests, 2) @@ -1843,7 +1841,7 @@ local function define_tests() local history = prompt.new() history:add_user("Plan the release") - local first = assert(runner:step(history, { context = {}, persist_context_notes = true })) + local first = assert(runner:step(history, { context = {} })) test.eq(#first.context_notes, 1) for _, note in ipairs(first.context_notes) do history:add_message(note.role, note.content, nil, note.metadata) @@ -1851,7 +1849,7 @@ local function define_tests() history:add_function_call("calculator", '{"expression":"2+2"}', "call_1") history.messages[#history.messages].metadata = { thinking_blocks = first.metadata.thinking_blocks } history:add_function_result("calculator", "4", "call_1") - local second = assert(runner:step(history, { context = {}, persist_context_notes = true })) + local second = assert(runner:step(history, { context = {} })) test.eq(#second.context_notes, 1) local before = without_markers(requests[1]) From 1939565635aba04d3134cc2adec8fee6cd1970e2 Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 21:26:51 -0500 Subject: [PATCH 23/37] feat(agent): deliver each tool result to a per-call listener tool_caller:set_result_listener(fn) receives (call_id, result) once per call with the final normalised result as soon as that call finishes. Parallel execution collects responses in completion order via channel.select over pending responses and the cancel channel; sequential execution delivers before the next call starts. A listener failure stops the batch, cancels in-flight calls, skips after_execute wrappers and makes execute return nil and the error. Argument, context and executor preparation is shared by both strategies. --- src/agent/src/README.md | 36 +++ src/agent/src/tools/caller.lua | 378 ++++++++++++------------ src/agent/test/tools/authority_test.lua | 4 +- src/agent/test/tools/caller_test.lua | 222 +++++++++++++- 4 files changed, 445 insertions(+), 195 deletions(-) diff --git a/src/agent/src/README.md b/src/agent/src/README.md index 83786a3a..b763e615 100644 --- a/src/agent/src/README.md +++ b/src/agent/src/README.md @@ -329,6 +329,42 @@ for call_id, result in pairs(results) do end ``` +### Result Listener + +A host that persists each tool result as soon as its call finishes registers a +listener. It receives the call ID and that call's final result, the same table +the batch returns under that ID: + +```lua +caller:set_result_listener(function(call_id, result) + local _, err = store:save_observation(call_id, result) + if err then return nil, err end +end) + +local results, err = caller:execute(context, validated, cancel) +if err then + -- the listener failed; calls whose results it did not accept run again on recovery +end +``` + +- Each call ID is delivered exactly once, including invalid calls, argument + parse failures, start failures, tool errors and cancelled calls. Explicit + failure payloads (`success = false` with an `error` string) are already + reflected in `result.error`. +- Sequential execution delivers each result before the next call starts. + Parallel execution delivers results in completion order, so a fast call is + delivered while slower peers are still running. Calls that fail before they + start are delivered as soon as the failure is known. +- Delivery happens before `after_execute` wrappers run; the wrappers and the + return value still receive the complete batch keyed by call ID. +- A listener fails by raising or by returning a non-nil second value. The batch + then stops: no further call starts, in-flight calls are cancelled, + `after_execute` wrappers do not run, and `execute` returns `nil` and an error + naming the call ID whose delivery failed. Every result delivered before that + call was accepted by the listener. +- The listener must not mutate the result; it is the table the batch returns. +- `set_result_listener(nil)` removes the listener. + ### Exclusive Tools Tools with `meta.exclusive = true` in their registry entry cancel all other concurrent tool calls in the same batch. Only one exclusive tool is allowed per batch. diff --git a/src/agent/src/tools/caller.lua b/src/agent/src/tools/caller.lua index 43b2248e..3c42cb8e 100644 --- a/src/agent/src/tools/caller.lua +++ b/src/agent/src/tools/caller.lua @@ -227,6 +227,7 @@ function tool_caller.new(): any self.wrapper_controls = {} self.last_tool_calls = {} self.runtime_context_resolver = nil + self.result_listener = nil return self end @@ -273,6 +274,17 @@ function tool_caller:set_runtime_context_resolver(resolver: any?): any return self end +-- The listener receives (call_id, result) once per call as soon as that call's +-- final result is known. It fails by raising or by returning a non-nil second +-- value; execute then stops the batch and returns nil and the error. +function tool_caller:set_result_listener(listener: any?): any + if listener ~= nil and type(listener) ~= "function" then + error("result listener must be a function or nil") + end + self.result_listener = listener + return self +end + function tool_caller:get_wrapper_observations(): {ToolWrapperObservation} return self.wrapper_observations or {} end @@ -633,6 +645,18 @@ end local CANCELLED_ERROR = "Tool call was cancelled" +type BatchRun = { + results: {[any]: ToolExecResult}, + listener: any?, + error: string?, +} + +type PendingCall = { + command: any, + response: any, + tool_call: any, +} + local function cancelled_result(tool_call: any): any return { result = nil, error = CANCELLED_ERROR, cancelled = true, tool_call = tool_call } end @@ -642,61 +666,41 @@ local function cancellation_requested(cancel: any?): boolean return not channel.select({ cancel:case_receive(), default = true }).default end --- Waits for an async tool call; when `cancel` is ready first the call is cancelled. -local function await_command(command: any, tool_call: any, cancel: any?, stopped: boolean?): (any, boolean) - local response = command:response() - local ok - local observed_stop = stopped == true - if observed_stop then - local completed = channel.select({ response:case_receive(), default = true }) - if completed.default then - command:cancel() - return cancelled_result(tool_call), true - end - ok = completed.ok - elseif cancel then - local selected = channel.select({ response:case_receive(), cancel:case_receive() }) - if selected.channel == cancel then - observed_stop = true - local completed = channel.select({ response:case_receive(), default = true }) - if completed.default then - command:cancel() - return cancelled_result(tool_call), true - end - ok = completed.ok - else - ok = selected.ok - end - else - local _, received = response:receive() - ok = received - end - - if ok and not command:is_canceled() then - local payload, err = command:result() - if err then - return { result = nil, error = err, tool_call = tool_call }, observed_stop +-- Some tools return an explicit failure as data rather than the executor's +-- second return value. Preserve that data, but expose its error through the +-- same canonical channel used by hosts and after-execute wrappers. +local function normalize_result(entry: any): any + local result = entry.result + if not entry.error and type(result) == "table" and result.success == false + and type(result.error) == "string" and result.error ~= "" then + entry.error = result.error + end + return entry +end + +-- Stores the call's final result and delivers it to the listener. A listener +-- failure sets run.error, which stops the batch. +local function record_result(run: BatchRun, call_id: any, entry: any) + normalize_result(entry) + run.results[call_id] = entry + if run.listener then + local ok, raised, listener_err = pcall(run.listener, call_id, entry) + if not ok then + run.error = "Tool result listener failed for " .. tostring(call_id) .. ": " .. tostring(raised) + elseif listener_err ~= nil then + run.error = "Tool result listener failed for " .. tostring(call_id) .. ": " .. tostring(listener_err) end - return { result = payload and payload:data() or nil, error = nil, tool_call = tool_call }, observed_stop end - local _, err = command:result() - return { - result = nil, - error = "Tool execution failed or was canceled: " .. tostring(err), - tool_call = tool_call - }, observed_stop end -local function execute_single_tool(self: any, call_id: string, tool_call: any, context: table?, cancel: any?): (any, boolean?) - if cancellation_requested(cancel) then return cancelled_result(tool_call), true end - local registry_id = tool_call.registry_id +-- Resolves arguments, context and the authorised executor for one call. +-- Returns the executor and decoded arguments, or the call's failure result. +local function prepare_call(self: any, call_id: any, tool_call: any, context: table?, batch: any?): (any, any, any) local args = tool_call.args - local tool_context = tool_call.context or {} - if type(args) == "string" then local parsed_args, err = get_json().decode(args) if err then - return { + return nil, nil, { result = nil, error = "Failed to parse arguments: " .. tostring(err), tool_call = tool_call @@ -705,196 +709,196 @@ local function execute_single_tool(self: any, call_id: string, tool_call: any, c args = parsed_args end - local merged_context = merge_execution_context(tool_context as table?, context, call_id) + local merged_context = merge_execution_context(tool_call.context as table?, context, tostring(call_id)) - local runtime_context, runtime_err = resolve_runtime_context(self, call_id, tool_call) + local runtime_context, runtime_err = resolve_runtime_context(self, tostring(call_id), tool_call) if runtime_err then - return { - result = nil, - error = runtime_err, - tool_call = tool_call - } + return nil, nil, { result = nil, error = runtime_err, tool_call = tool_call } end for k, v in pairs(runtime_context or {}) do merged_context[k] = v end - -- Execute the tool - local ctx_executor, executor_err = execution_executor(self, merged_context, tostring(registry_id)) + local ctx_executor, executor_err = execution_executor(self, merged_context, tostring(tool_call.registry_id), batch) if not ctx_executor then - return {result = nil, error = executor_err, tool_call = tool_call} + return nil, nil, { result = nil, error = executor_err, tool_call = tool_call } end - if cancel then - if cancellation_requested(cancel) then return cancelled_result(tool_call), true end - local command, start_err = ctx_executor:async(tostring(registry_id), args) - if not command then - return { result = nil, error = start_err, tool_call = tool_call } - end - return await_command(command, tool_call, cancel) + return ctx_executor, args, nil +end + +-- Starts an async call. Returns the pending call, or the start failure result. +local function launch_call(ctx_executor: any, tool_call: any, args: any): (PendingCall?, any) + local command, start_err = ctx_executor:async(tostring(tool_call.registry_id), args) + if not command then + return nil, { result = nil, error = start_err, tool_call = tool_call } end - local result, err = ctx_executor:call(tostring(registry_id), args) + return { command = command, response = command:response(), tool_call = tool_call }, nil +end +local function command_result(pending: PendingCall, ok: boolean?): any + local command = pending.command + if ok and not command:is_canceled() then + local payload, err = command:result() + if err then + return { result = nil, error = err, tool_call = pending.tool_call } + end + return { result = payload and payload:data() or nil, error = nil, tool_call = pending.tool_call } + end + local _, err = command:result() return { - result = result, - error = err, - tool_call = tool_call + result = nil, + error = "Tool execution failed or was canceled: " .. tostring(err), + tool_call = pending.tool_call } end -local function execute_sequential(self: any, context: any, validated_tools: any, cancel: any?): any - local results = {} - local cancelled = false +local function cancel_pending(pending: {[any]: PendingCall}) + for call_id, call in pairs(pending) do + call.command:cancel() + pending[call_id] = nil + end +end - -- Handle nil or empty validated_tools - if not validated_tools then - return results +-- Records every pending call: responses that are already complete keep their +-- real outcome, the rest are cancelled. +local function settle_after_stop(run: BatchRun, pending: {[any]: PendingCall}) + for call_id, call in pairs(pending) do + if run.error then break end + pending[call_id] = nil + local completed = channel.select({ call.response:case_receive(), default = true }) + if completed.default then + call.command:cancel() + record_result(run, call_id, cancelled_result(call.tool_call)) + else + record_result(run, call_id, command_result(call, completed.ok)) + end end + cancel_pending(pending) +end - for call_id, tool_call in pairs(validated_tools) do - if not tool_call.valid then - results[call_id] = { - error = tool_call.error, - tool_call = tool_call - } - goto continue +-- Records pending calls in completion order. Returns true when cancellation was observed. +local function collect_pending(run: BatchRun, pending: {[any]: PendingCall}, cancel: any?, stopped: boolean): boolean + if stopped then + settle_after_stop(run, pending) + return true + end + while next(pending) ~= nil and not run.error do + local cases = {} + for _, call in pairs(pending) do + cases[#cases + 1] = call.response:case_receive() end + if cancel then cases[#cases + 1] = cancel:case_receive() end - if cancelled then - results[call_id] = cancelled_result(tool_call) - goto continue + local selected = channel.select(cases) + if cancel and selected.channel == cancel then + settle_after_stop(run, pending) + return true + end + for call_id, call in pairs(pending) do + if call.response == selected.channel then + pending[call_id] = nil + record_result(run, call_id, command_result(call, selected.ok)) + break + end end + end + cancel_pending(pending) + return false +end - local exec_result, observed_stop = execute_single_tool(self, tostring(call_id), tool_call, context as {}?, cancel) - results[call_id] = exec_result - cancelled = observed_stop == true +local function execute_sequential(self: any, run: BatchRun, context: table?, validated_tools: any, cancel: any?) + local cancelled = false - ::continue:: - end + for call_id, tool_call in pairs(validated_tools) do + local entry = nil + if not tool_call.valid then + entry = { error = tool_call.error, tool_call = tool_call } + elseif cancelled or cancellation_requested(cancel) then + cancelled = true + entry = cancelled_result(tool_call) + else + local ctx_executor, args, failure = prepare_call(self, call_id, tool_call, context, nil) + if failure then + entry = failure + elseif not cancel then + local result, err = ctx_executor:call(tostring(tool_call.registry_id), args) + entry = { result = result, error = err, tool_call = tool_call } + elseif cancellation_requested(cancel) then + cancelled = true + entry = cancelled_result(tool_call) + else + local pending, start_failure = launch_call(ctx_executor, tool_call, args) + if start_failure then + entry = start_failure + else + cancelled = collect_pending(run, { [call_id] = pending :: PendingCall }, cancel, false) + end + end + end - return results + if entry then record_result(run, call_id, entry) end + if run.error then return end + end end -local function execute_parallel(self: any, context: table?, validated_tools: any, cancel: any?): any - local results = {} - local commands = {} +local function execute_parallel(self: any, run: BatchRun, context: table?, validated_tools: any, cancel: any?) + local pending: {[any]: PendingCall} = {} local cancelled = false -- Parallel tools share one authority snapshot for this batch. The next -- execute() refreshes again; no provider call serializes each launch. local batch = {} - -- Start all valid tools for call_id, tool_call in pairs(validated_tools) do + local entry = nil if not tool_call.valid then - results[call_id] = { - error = tool_call.error, - tool_call = tool_call - } - goto continue - end - - if cancelled or cancellation_requested(cancel) then + entry = { error = tool_call.error, tool_call = tool_call } + elseif cancelled or cancellation_requested(cancel) then cancelled = true - results[call_id] = cancelled_result(tool_call) - goto continue - end - - local registry_id = tool_call.registry_id - local args = tool_call.args - local tool_context = tool_call.context or {} - - if type(args) == "string" then - local parsed_args, err = get_json().decode(args) - if err then - results[call_id] = { - error = "Failed to parse arguments: " .. tostring(err), - tool_call = tool_call - } - goto continue + entry = cancelled_result(tool_call) + else + local ctx_executor, args, failure = prepare_call(self, call_id, tool_call, context, batch) + if failure then + entry = failure + elseif cancellation_requested(cancel) then + cancelled = true + entry = cancelled_result(tool_call) + else + local call, start_failure = launch_call(ctx_executor, tool_call, args) + if start_failure then + entry = start_failure + else + pending[call_id] = call :: PendingCall + end end - args = parsed_args - end - - local merged_context = merge_execution_context(tool_context as table?, context, tostring(call_id)) - - local runtime_context, runtime_err = resolve_runtime_context(self, tostring(call_id), tool_call) - if runtime_err then - results[call_id] = { - result = nil, - error = runtime_err, - tool_call = tool_call - } - goto continue - end - for k, v in pairs(runtime_context or {}) do - merged_context[k] = v end - -- Start async execution - local ctx_executor, executor_err = execution_executor(self, merged_context, tostring(registry_id), batch) - if not ctx_executor then - results[call_id] = {result = nil, error = executor_err, tool_call = tool_call} - goto continue - end - if cancellation_requested(cancel) then - cancelled = true - results[call_id] = cancelled_result(tool_call) - goto continue - end - local command, start_err = ctx_executor:async(tostring(registry_id), args) - if not command then - results[call_id] = { - result = nil, - error = start_err, - tool_call = tool_call - } - goto continue + if entry then record_result(run, call_id, entry) end + if run.error then + cancel_pending(pending) + return end - - commands[call_id] = { - command = command, - tool_call = tool_call - } - - ::continue:: - end - - -- Preserve completed outcomes even after Stop; cancel only pending peers. - for call_id, cmd_info in pairs(commands) do - local result, observed_stop = await_command(cmd_info.command, cmd_info.tool_call, cancel, cancelled) - results[call_id] = result - cancelled = cancelled or observed_stop == true end - return results + collect_pending(run, pending, cancel, cancelled) end -- `cancel`, when given, is a channel whose closing cancels the in-flight calls; --- each cancelled call reports `cancelled = true`. -function tool_caller:execute(context: any, validated_tools: any, cancel: any?): any +-- each cancelled call reports `cancelled = true`. Returns the results keyed by +-- call ID, or nil and an error when the result listener fails. +function tool_caller:execute(context: any, validated_tools: any, cancel: any?): (any, string?) self.wrapper_controls = {} - -- Handle nil validated_tools - if not validated_tools then - validated_tools = {} - end + local run: BatchRun = { results = {}, listener = self.result_listener, error = nil } - local results if self.strategy == STRATEGY.PARALLEL then - results = execute_parallel(self :: any, context as table?, validated_tools, cancel) + execute_parallel(self :: any, run, context as table?, validated_tools or {}, cancel) else - results = execute_sequential(self :: any, context, validated_tools, cancel) + execute_sequential(self :: any, run, context as table?, validated_tools or {}, cancel) end - - -- Some tools return an explicit failure as data rather than the executor's - -- second return value. Preserve that data, but expose its error through the - -- same canonical channel used by hosts and after-execute wrappers. - for _, entry in pairs(results) do - local result = entry.result - if not entry.error and type(result) == "table" and result.success == false - and type(result.error) == "string" and result.error ~= "" then - entry.error = result.error - end + if run.error then + return nil, run.error end + local results = run.results local after_payload: ToolWrapperApplyRequest = { phase = AFTER_EXECUTE :: ToolWrapperPhase, host = self.wrapper_context.host, diff --git a/src/agent/test/tools/authority_test.lua b/src/agent/test/tools/authority_test.lua index 79050824..24ee1469 100644 --- a/src/agent/test/tools/authority_test.lua +++ b/src/agent/test/tools/authority_test.lua @@ -51,8 +51,10 @@ local function run_tests() end function executor:async(id, args) local result = self:call(id, args) + local response = channel.new(1) + response:send(true) return { - response = function() return {receive = function() return {}, true end} end, + response = function() return response end, is_canceled = function() return false end, result = function() return {data = function() return result end} end, } diff --git a/src/agent/test/tools/caller_test.lua b/src/agent/test/tools/caller_test.lua index dda74b87..48d3fd95 100644 --- a/src/agent/test/tools/caller_test.lua +++ b/src/agent/test/tools/caller_test.lua @@ -17,6 +17,7 @@ local function define_tests() local wrapper_calls local wrapper_behaviors local execution_contexts + local cancelled_commands -- Mock tool schemas local tool_schemas = { @@ -97,6 +98,7 @@ local function define_tests() wrapper_calls = {} wrapper_behaviors = {} execution_contexts = {} + cancelled_commands = {} -- Create mock modules local mock_json = { @@ -174,15 +176,19 @@ local function define_tests() final_error = "Tool execution failed" end + local response = result_data and result_data.response + if not response then + response = channel.new(1) + response:send(true) + end + local canceled = false return { - response = function() - return { - receive = function() - return {}, true - end - } + response = function() return response end, + cancel = function() + canceled = true + table.insert(cancelled_commands, registry_id) end, - is_canceled = function() return false end, + is_canceled = function() return canceled end, result = function() if final_error then return nil, final_error @@ -287,6 +293,7 @@ local function define_tests() wrapper_calls = nil wrapper_behaviors = nil execution_contexts = nil + cancelled_commands = nil end) describe("explicit failure results", function() @@ -1197,6 +1204,207 @@ local function define_tests() end) end) + describe("Result listener", function() + local STRATEGIES = { "sequential", "parallel" } + + local function tool(call_id, registry_id, args) + return { call_id = call_id, name = call_id, registry_id = registry_id, args = args or {}, valid = true } + end + + local function recording_listener() + local notes = { order = {}, results = {}, errors = {}, counts = {} } + notes.listener = function(call_id, result) + table.insert(notes.order, call_id) + notes.results[call_id] = result + notes.errors[call_id] = result.error + notes.counts[call_id] = (notes.counts[call_id] or 0) + 1 + end + return notes + end + + it("accepts a function or nil and rejects other values", function() + local caller = tool_caller.new() + test.eq(caller:set_result_listener(function() end), caller) + test.eq(caller:set_result_listener(nil), caller) + local ok, err = pcall(function() caller:set_result_listener("listener") end) + test.is_false(ok) + test.contains(tostring(err), "result listener must be a function or nil") + end) + + it("notifies every outcome once with the normalised result the batch returns", function() + for _, strategy in ipairs(STRATEGIES) do + tool_results["test:explicit"] = { result = { success = false, error = "automation not found" } } + local notes = recording_listener() + local caller = tool_caller.new():set_strategy(strategy):set_result_listener(notes.listener) + local invalid = tool("invalid", "test:invalid") + invalid.valid = false + invalid.error = "Tool validation failed" + local results = caller:execute({}, { + ok = tool("ok", "test:calculator"), + invalid = invalid, + parse = tool("parse", "test:calculator", '{"invalid": json}'), + failed = tool("failed", "test:failing_tool"), + denied = tool("denied", "test:denied_tool"), + explicit = tool("explicit", "test:explicit"), + }, channel.new()) + + test.eq(#notes.order, 6, strategy) + test.eq(count_table_elements(results), 6, strategy) + for call_id, entry in pairs(results) do + test.eq(notes.counts[call_id], 1, strategy .. " " .. call_id) + test.is_true(rawequal(notes.results[call_id], entry), strategy .. " " .. call_id) + test.eq(notes.errors[call_id], entry.error, strategy .. " " .. call_id) + end + test.eq(results.ok.result, 42) + test.is_nil(results.ok.error) + test.eq(results.invalid.error, "Tool validation failed") + test.contains(results.parse.error, "Failed to parse arguments") + test.eq(results.failed.error, "Tool execution failed") + test.eq(results.denied.error, "not allowed: test:denied_tool") + test.eq(notes.errors.explicit, "automation not found", strategy) + end + end) + + it("notifies a fast parallel call before a slower peer finishes", function() + local slow_response = channel.new(1) + tool_results["test:slow"] = { response = slow_response, result = "slow" } + local notes = channel.new() + local done = channel.new(1) + local caller = tool_caller.new():set_strategy(tool_caller.STRATEGY.PARALLEL) + caller:set_result_listener(function(call_id, result) + notes:send({ call_id = call_id, result = result }) + end) + -- Array keys put the slow call first in iteration order. + local slow = tool("slow", "test:slow") + local fast = tool("fast", "test:calculator") + coroutine.spawn(function() done:send(caller:execute({}, { slow, fast })) end) + + local guard = time.timer("2s") + local first = channel.select({ notes:case_receive(), guard:channel():case_receive() }) + test.eq(first.channel, notes, "the fast call must be delivered while the slow call is pending") + test.eq(first.value.result.tool_call.call_id, "fast") + test.eq(first.value.result.result, 42) + + slow_response:send(true) + local second = channel.select({ notes:case_receive(), guard:channel():case_receive() }) + test.eq(second.channel, notes) + test.eq(second.value.result.tool_call.call_id, "slow") + test.eq(second.value.result.result, "slow") + local finished = channel.select({ done:case_receive(), guard:channel():case_receive() }) + guard:stop() + test.eq(finished.channel, done) + test.eq(finished.value[1].result, "slow") + test.eq(finished.value[2].result, 42) + end) + + it("notifies each sequential call before the next one starts", function() + for _, cancel in ipairs({ false, true }) do + local events = {} + tool_results["test:calculator"] = { execute = function(_, ctx) + table.insert(events, "run:" .. ctx.call_id) + return 42 + end } + local caller = tool_caller.new():set_result_listener(function(call_id) + table.insert(events, "notify:" .. call_id) + end) + caller:execute({}, { + a = tool("a", "test:calculator"), + b = tool("b", "test:calculator"), + c = tool("c", "test:calculator"), + }, cancel and channel.new() or nil) + test.eq(#events, 6) + for i = 1, #events, 2 do + local call_id = string.match(tostring(events[i]), "^run:(.+)$") + test.not_nil(call_id, events[i]) + test.eq(events[i + 1], "notify:" .. call_id) + end + end + end) + + it("notifies cancelled calls once when the batch is cancelled before start", function() + for _, strategy in ipairs(STRATEGIES) do + local notes = recording_listener() + local caller = tool_caller.new():set_strategy(strategy):set_result_listener(notes.listener) + local cancel = channel.new() + cancel:close() + local results = caller:execute({}, { + a = tool("a", "test:calculator"), + b = tool("b", "test:weather"), + }, cancel) + test.eq(#notes.order, 2, strategy) + for _, call_id in ipairs({ "a", "b" }) do + test.eq(notes.counts[call_id], 1, strategy) + test.is_true(results[call_id].cancelled, strategy) + test.is_true(rawequal(notes.results[call_id], results[call_id]), strategy) + end + end + end) + + it("notifies a completed peer and the cancelled pending call once each", function() + local cancel = channel.new() + tool_results["test:held"] = { response = channel.new(), result = "never" } + tool_results["test:done"] = { execute = function() + cancel:close() + return "committed" + end } + local notes = recording_listener() + local caller = tool_caller.new():set_strategy(tool_caller.STRATEGY.PARALLEL) + caller:set_result_listener(notes.listener) + local results = caller:execute({}, { tool("held", "test:held"), tool("done", "test:done") }, cancel) + test.eq(#notes.order, 2) + test.eq(notes.counts[1], 1) + test.eq(notes.counts[2], 1) + test.is_true(results[1].cancelled) + test.eq(results[2].result, "committed") + test.is_nil(results[2].cancelled) + test.eq(cancelled_commands[1], "test:held") + test.eq(#cancelled_commands, 1) + end) + + it("fails the batch on a listener error and stops the remaining calls", function() + local failures = { + returned = function() return nil, "store unavailable" end, + raised = function() error("store unavailable") end, + } + for _, strategy in ipairs(STRATEGIES) do + for kind, fail in pairs(failures) do + wrapper_calls = {} + cancelled_commands = {} + local runs = 0 + local notified = {} + tool_results["test:calculator"] = { execute = function() + runs = runs + 1 + return 42 + end } + local caller = tool_caller.new():set_strategy(strategy) + caller:set_wrapper_context({ host = { kind = "session", session_id = "s" } }) + caller:set_tool_wrappers({ { id = "audit", phases = { tool_caller.PHASE.AFTER_EXECUTE }, + binding = "test.wrapper:audit" } }) + caller:set_result_listener(function(call_id, result) + table.insert(notified, call_id) + return fail(call_id, result) + end) + local results, err = caller:execute({}, { + a = tool("a", "test:calculator"), + b = tool("b", "test:calculator"), + c = tool("c", "test:calculator"), + }) + local label = strategy .. " " .. kind + test.is_nil(results, label) + test.contains(tostring(err), "Tool result listener failed for " .. notified[1], label) + test.contains(tostring(err), "store unavailable", label) + test.eq(#notified, 1, label) + test.eq(#wrapper_calls, 0, label .. ": after_execute wrappers see only complete batches") + if strategy == "sequential" then + test.eq(runs, 1, label .. ": no further call starts") + else + test.eq(#cancelled_commands, 2, label .. ": undelivered calls are cancelled") + end + end + end + end) + end) + describe("Edge Cases and Error Handling", function() it("should handle execution with empty validated tools", function() local caller = tool_caller.new() From c5cb5d3d807d18179e8574a51f420d47a00bb5ec Mon Sep 17 00:00:00 2001 From: KickSide Date: Tue, 3 Nov 2026 21:49:03 -0500 Subject: [PATCH 24/37] fix(agent): cancel only unfinished calls when a result listener fails cancel_pending takes responses that have already arrived and cancels only commands that are still running, so a finished command is never cancelled. --- src/agent/src/README.md | 9 ++-- src/agent/src/tools/caller.lua | 21 ++++++-- src/agent/test/tools/caller_test.lua | 75 +++++++++++++++++++++++++--- 3 files changed, 89 insertions(+), 16 deletions(-) diff --git a/src/agent/src/README.md b/src/agent/src/README.md index b763e615..6157df4a 100644 --- a/src/agent/src/README.md +++ b/src/agent/src/README.md @@ -358,10 +358,11 @@ end - Delivery happens before `after_execute` wrappers run; the wrappers and the return value still receive the complete batch keyed by call ID. - A listener fails by raising or by returning a non-nil second value. The batch - then stops: no further call starts, in-flight calls are cancelled, - `after_execute` wrappers do not run, and `execute` returns `nil` and an error - naming the call ID whose delivery failed. Every result delivered before that - call was accepted by the listener. + then stops: no further call starts, calls still running are cancelled, results + of calls that finished but were not yet delivered are dropped without + cancelling them, `after_execute` wrappers do not run, and `execute` returns + `nil` and an error naming the call ID whose delivery failed. Every result + delivered before that call was accepted by the listener. - The listener must not mutate the result; it is the table the batch returns. - `set_result_listener(nil)` removes the listener. diff --git a/src/agent/src/tools/caller.lua b/src/agent/src/tools/caller.lua index 3c42cb8e..af35922b 100644 --- a/src/agent/src/tools/caller.lua +++ b/src/agent/src/tools/caller.lua @@ -752,10 +752,21 @@ local function command_result(pending: PendingCall, ok: boolean?): any } end +-- Takes the call's response if it has already arrived. +local function poll_response(call: PendingCall): (boolean, boolean?) + local completed = channel.select({ call.response:case_receive(), default = true }) + if completed.default then return false, nil end + return true, completed.ok +end + +-- Cancels calls that are still running. Responses that have already arrived +-- are taken and dropped; their commands are finished and are not cancelled. local function cancel_pending(pending: {[any]: PendingCall}) for call_id, call in pairs(pending) do - call.command:cancel() pending[call_id] = nil + if not poll_response(call) then + call.command:cancel() + end end end @@ -765,12 +776,12 @@ local function settle_after_stop(run: BatchRun, pending: {[any]: PendingCall}) for call_id, call in pairs(pending) do if run.error then break end pending[call_id] = nil - local completed = channel.select({ call.response:case_receive(), default = true }) - if completed.default then + local completed, ok = poll_response(call) + if completed then + record_result(run, call_id, command_result(call, ok)) + else call.command:cancel() record_result(run, call_id, cancelled_result(call.tool_call)) - else - record_result(run, call_id, command_result(call, completed.ok)) end end cancel_pending(pending) diff --git a/src/agent/test/tools/caller_test.lua b/src/agent/test/tools/caller_test.lua index 48d3fd95..4e46b561 100644 --- a/src/agent/test/tools/caller_test.lua +++ b/src/agent/test/tools/caller_test.lua @@ -1372,10 +1372,15 @@ local function define_tests() cancelled_commands = {} local runs = 0 local notified = {} - tool_results["test:calculator"] = { execute = function() - runs = runs + 1 - return 42 - end } + local function counted(response) + return { response = response, execute = function() + runs = runs + 1 + return 42 + end } + end + tool_results["test:calculator"] = counted(nil) + tool_results["test:held_b"] = counted(channel.new()) + tool_results["test:held_c"] = counted(channel.new()) local caller = tool_caller.new():set_strategy(strategy) caller:set_wrapper_context({ host = { kind = "session", session_id = "s" } }) caller:set_tool_wrappers({ { id = "audit", phases = { tool_caller.PHASE.AFTER_EXECUTE }, @@ -1386,8 +1391,8 @@ local function define_tests() end) local results, err = caller:execute({}, { a = tool("a", "test:calculator"), - b = tool("b", "test:calculator"), - c = tool("c", "test:calculator"), + b = tool("b", "test:held_b"), + c = tool("c", "test:held_c"), }) local label = strategy .. " " .. kind test.is_nil(results, label) @@ -1398,11 +1403,67 @@ local function define_tests() if strategy == "sequential" then test.eq(runs, 1, label .. ": no further call starts") else - test.eq(#cancelled_commands, 2, label .. ": undelivered calls are cancelled") + test.eq(notified[1], "a", label) + test.eq(#cancelled_commands, 2, label .. ": unfinished calls are cancelled") end end end end) + + it("does not cancel parallel calls that completed but were not delivered", function() + local caller = tool_caller.new():set_strategy("parallel") + local notified = 0 + caller:set_result_listener(function() + notified = notified + 1 + return nil, "store unavailable" + end) + local results, err = caller:execute({}, { + a = tool("a", "test:calculator"), + b = tool("b", "test:weather"), + c = tool("c", "test:non_exclusive"), + }) + test.is_nil(results) + test.contains(tostring(err), "store unavailable") + test.eq(notified, 1) + test.eq(#cancelled_commands, 0, "a completed command must not be cancelled") + end) + + it("starts no call when the listener fails during the parallel launch", function() + for _, first in ipairs({ "invalid", "parse" }) do + wrapper_calls = {} + local runs = 0 + tool_results["test:calculator"] = { execute = function() + runs = runs + 1 + return 42 + end } + local failed = tool("failed", "test:calculator", '{"invalid": json}') + if first == "invalid" then + failed.valid = false + failed.error = "Tool validation failed" + end + local caller = tool_caller.new():set_strategy("parallel") + caller:set_wrapper_context({ host = { kind = "session", session_id = "s" } }) + caller:set_tool_wrappers({ { id = "audit", phases = { tool_caller.PHASE.AFTER_EXECUTE }, + binding = "test.wrapper:audit" } }) + local notified = {} + caller:set_result_listener(function(call_id) + table.insert(notified, call_id) + return nil, "store unavailable" + end) + -- Array keys record the failed entry before any call launches. + local results, err = caller:execute({}, { + failed, + tool("a", "test:calculator"), + tool("b", "test:calculator"), + }) + test.is_nil(results, first) + test.contains(tostring(err), "Tool result listener failed for 1", first) + test.contains(tostring(err), "store unavailable", first) + test.eq(#notified, 1, first) + test.eq(runs, 0, first .. ": no call starts") + test.eq(#wrapper_calls, 0, first .. ": after_execute wrappers do not run") + end + end) end) describe("Edge Cases and Error Handling", function() From d064342289d0911feed05c2d70d6f46dd9149d62 Mon Sep 17 00:00:00 2001 From: KickSide Date: Wed, 4 Nov 2026 08:23:01 -0500 Subject: [PATCH 25/37] fix(llm,agent): make thinking replay and request shaping safe Stale current-round blocks: the agent marks them and Claude and Bedrock remove marked blocks when the final thinking object cannot carry a binding. Interleaved responses: extraction records block positions on all four paths and consolidation rebuilds the response order; stored rows without positions keep their order. OpenAI-compatible tool rounds stop at a non-tool message once every call has its result, so a positional note between rounds stays in place. One blank rule across drivers: blank user, system and developer text never reaches a driver, and an assistant turn with no visible text and no call renders as nothing. Claude invents no text for it. Effective output is planned once before the card limit and admission, from the route and the driver's output policy; mappers read it and adjusted.max_tokens.sent is the wire value. Requests with no mapped turn fail before HTTP on Claude, Bedrock, Google and OpenAI. drop_stale_thinking handles string content. The prefix fingerprint covers only what reaches the wire. --- src/agent/src/README.md | 18 +- src/agent/src/agent.lua | 43 +- src/agent/test/_index.yaml | 18 + src/agent/test/replay_safety_test.lua | 226 +++++ src/llm/src/README.md | 23 +- src/llm/src/_index.yaml | 2 + src/llm/src/bedrock/_index.yaml | 4 + src/llm/src/bedrock/client.lua | 8 + src/llm/src/bedrock/generate.lua | 7 +- src/llm/src/bedrock/mapper.lua | 163 ++- src/llm/src/bedrock/structured_output.lua | 7 +- src/llm/src/claude/_index.yaml | 4 + src/llm/src/claude/client.lua | 16 +- src/llm/src/claude/generate.lua | 7 +- src/llm/src/claude/mapper.lua | 129 ++- src/llm/src/claude/structured_output.lua | 11 +- src/llm/src/discovery/providers.lua | 35 +- src/llm/src/google/generate.lua | 4 + src/llm/src/google/mapper.lua | 3 +- src/llm/src/google/structured_output.lua | 4 + src/llm/src/llm.lua | 22 +- src/llm/src/openai/generate.lua | 4 + src/llm/src/openai/mapper.lua | 38 +- src/llm/src/openai/structured_output.lua | 4 + src/llm/src/openai_compat/_index.yaml | 2 + src/llm/src/openai_compat/mapper.lua | 25 +- .../src/openai_compat/structured_output.lua | 3 +- src/llm/src/output.lua | 3 + src/llm/src/prompt.lua | 142 ++- src/llm/src/util/route.lua | 48 + src/llm/src/util/thinking.lua | 172 ++++ src/llm/test/_index.yaml | 33 + src/llm/test/bedrock/mapper_test.lua | 3 +- src/llm/test/claude/mapper_test.lua | 3 +- src/llm/test/google/mapper_test.lua | 4 +- src/llm/test/llm_test.lua | 1 + src/llm/test/native_errors_test.lua | 8 +- src/llm/test/replay_safety_test.lua | 950 ++++++++++++++++++ src/llm/test/route_flow_test.lua | 14 +- 39 files changed, 2022 insertions(+), 189 deletions(-) create mode 100644 src/agent/test/replay_safety_test.lua create mode 100644 src/llm/test/replay_safety_test.lua diff --git a/src/agent/src/README.md b/src/agent/src/README.md index 2deb9ceb..1d34de8f 100644 --- a/src/agent/src/README.md +++ b/src/agent/src/README.md @@ -314,9 +314,10 @@ from the persisted notes. `prefix_fingerprint` is a sha256 over the system text and tools as drivers send them: the compiled system prompt together with ordinary (non-positional) system messages in the conversation, which Claude and Bedrock hoist into the top-level -system, and each tool's name, description, schema and native definition -(`meta.claude`). Registry ids and registry metadata (comment, tags, title) do -not reach the provider and are not part of it. It is stable across steps while +system, without blank text, and each tool's name, description, schema and native +definition (a `meta.claude` that names a `type`). Registry ids and other registry +metadata (comment, tags, title, a `meta.claude` without a `type`) do not reach the +provider and are not part of it. It is stable across steps while the sent system text and tools stay the same. It describes the request actually sent: when a checkpoint callback replaces the history, the fingerprint is recomputed from the replacement, and stale thinking in the replacement is judged against it. @@ -339,7 +340,9 @@ thinking blocks itself: 1. The host stores `response.prefix_fingerprint` as `metadata.prefix_fingerprint` on every row it persists from that step that carries `thinking_blocks` (the assistant row; function call messages built through `add_function_call` carry - no metadata, so blocks for a tool round live on its assistant row). + no metadata, so blocks for a tool round live on its assistant row). It stores + `response.metadata.content_positions` on the same row when present, which keeps + a response that interleaves thinking, text and calls in its order on replay. 2. On later steps the host passes `drop_stale_thinking = true`. Before the request, the agent removes the blocks of every message up to and including the last one whose `metadata.prefix_fingerprint` differs from the current @@ -352,7 +355,12 @@ The current tool round (the last assistant turn, while only its tool results, notes and user messages joining the tool-result turn follow it) keeps its blocks even when they are stale: the provider requires the final assistant turn of a tool round to keep its thinking, so those blocks are left to the provider's -drop-on-mismatch handling, and only earlier turns are stripped. User content +drop-on-mismatch handling, and only earlier turns are stripped. The agent marks +them `metadata.stale_thinking` on the copies it sends: when the route's final +thinking object cannot carry a binding (`thinking_effort` 0 on a route not declared +`always`, a `none` route, `between_tools`), Claude and Bedrock leave them out of the +request instead of sending them unbound, which an enforcing model rejects on every +retry. User content (a user message, a developer message or a positional system note) after an assistant turn with no tool result in between ends that turn; ordinary system messages, which drivers hoist into the top-level system, do not. diff --git a/src/agent/src/agent.lua b/src/agent/src/agent.lua index 9e0c5831..bff21aaa 100644 --- a/src/agent/src/agent.lua +++ b/src/agent/src/agent.lua @@ -447,31 +447,39 @@ local function has_tools_with_schemas(tools: any): boolean end -- What providers bind cached prefixes and thinking blocks to, as drivers send --- it: the system text (the compiled prompt plus conversation system messages, --- which Claude and Bedrock hoist into the top-level system) and each tool's --- name, description, schema and native definition. Registry ids and registry --- metadata never reach the provider. The model is left out: providers drop --- blocks a different model cannot read on their own. +-- it: the visible system text (the compiled prompt plus conversation system +-- messages, which Claude and Bedrock hoist into the top-level system) and each +-- tool's name, description, schema and native definition. A meta.claude is a +-- native definition only when it names a type; registry ids and other registry +-- metadata never reach the provider. The model is left out: providers drop blocks +-- a different model cannot read on their own. local function prefix_projection(messages: {any}, tools: {any}?): table local system = {} for _, message in ipairs(messages) do if message.role == prompt.ROLE.SYSTEM and not prompt.is_positional_note(message) then if type(message.content) == "string" then - table.insert(system, message.content) + if not prompt.is_blank(message.content) then + table.insert(system, message.content) + end elseif type(message.content) == "table" then for _, part in ipairs(message.content) do - table.insert(system, part.type == "text" and part.text or part) + if part.type ~= "text" then + table.insert(system, part) + elseif not prompt.is_blank(part.text) then + table.insert(system, part.text) + end end end end end local projected_tools = {} for _, tool in ipairs(tools or {}) do + local native = type(tool.meta) == "table" and tool.meta.claude or nil table.insert(projected_tools, { name = tool.name, description = tool.description, schema = tool.schema, - native = type(tool.meta) == "table" and tool.meta.claude or nil, + native = type(native) == "table" and native.type and native or nil, }) end return { system = system, tools = projected_tools } @@ -674,16 +682,25 @@ function agent:step(prompt_builder: any, runtime_options: any): (table?, string? end local is_current = runtime_options.thinking_is_current local working = prompt.new(messages) - local _, strip_err = working:drop_stale_thinking(function(msg) - if is_current then - return is_current(msg, fingerprint) == true + -- Stripping and marking read the same verdict, so a host predicate decides + -- each message once. + local verdicts = {} + local function current(msg) + if verdicts[msg] == nil then + if is_current then + verdicts[msg] = is_current(msg, fingerprint) == true + else + verdicts[msg] = msg.metadata.prefix_fingerprint == fingerprint + end end - return msg.metadata.prefix_fingerprint == fingerprint - end) + return verdicts[msg] + end + local _, strip_err = working:drop_stale_thinking(current) if strip_err then return nil, output.errors.generate():kind(output.ERROR_TYPE.INVALID_REQUEST) :message(tostring(strip_err)):build() end + working:mark_stale_thinking(current) return working:get_messages(), nil end diff --git a/src/agent/test/_index.yaml b/src/agent/test/_index.yaml index 21ebb601..2660dc44 100644 --- a/src/agent/test/_index.yaml +++ b/src/agent/test/_index.yaml @@ -99,6 +99,24 @@ entries: tool_caller: wippy.agent.tools:caller method: run_tests + # app:replay_safety_test + - name: replay_safety_test + kind: function.lua + meta: + name: Agent Replay Safety Test + type: test + group: Agent Framework + source: file://replay_safety_test.lua + modules: [json] + imports: + prompt: wippy.llm:prompt + agent: wippy.agent:agent + llm: wippy.llm:llm + claude_generate: wippy.llm.claude:generate + bedrock_generate: wippy.llm.bedrock:generate + test: wippy.test:test + method: run_tests + # app:context_test - name: context_test kind: function.lua diff --git a/src/agent/test/replay_safety_test.lua b/src/agent/test/replay_safety_test.lua new file mode 100644 index 00000000..9284e0d2 --- /dev/null +++ b/src/agent/test/replay_safety_test.lua @@ -0,0 +1,226 @@ +local test = require("test") +local json = require("json") +local llm = require("llm") +local prompt = require("prompt") +local agent = require("agent") +local claude_generate = require("claude_generate") +local bedrock_generate = require("bedrock_generate") + +local BUDGET_POLICY = { thinking = "budget", default_max_tokens = 2000 } + +local function text(value) return { { type = "text", text = value } } end +local function blocks(name) return { { type = "thinking", thinking = "", signature = "sig-" .. name } } end + +local function install(route): any + local sent: any = {} + claude_generate._client = { ENDPOINTS = { MESSAGES = "/messages" }, request = function(_, payload, options) + table.insert(sent, { payload = payload, betas = options and options.beta_features }) + return { content = { { type = "text", text = "ok" } }, stop_reason = "end_turn", + usage = { input_tokens = 1, output_tokens = 1 }, metadata = {} } + end } + bedrock_generate._client = { converse = function(_, payload) + table.insert(sent, { payload = payload }) + return { output = { message = { role = "assistant", content = { { text = "ok" } } } }, stopReason = "end_turn", + usage = { inputTokens = 1, outputTokens = 1 }, metadata = {} } + end } + llm._usage_tracker = false + local card = { name = "m", id = "m", capabilities = {}, class = {}, max_tokens = 200000, output_tokens = 64000, providers = { route } } + llm._models = { get_by_name = function() return card, nil end, get_by_class = function() return {}, nil end } + llm._providers = { + driver_declares_legacy_reasoning_flag = function() return false end, + driver_output_policy = function() return BUDGET_POLICY, nil end, + open = function(id) + local handler = id:match("^bedrock") and bedrock_generate or claude_generate + return { generate = function(_, args) return handler.handler(args) end } + end, + } + return sent +end + +local function spec(id, effort, prompt_text) + return { id = id, name = id, description = id, model = "m", max_tokens = 4096, thinking_effort = effort, + prompt = prompt_text == nil and ("You are " .. id) or prompt_text, + tools = { lookup = { name = "lookup", description = "d", schema = { type = "object", properties = {} }, + registry_id = "app:lookup", context = {} } }, + memory = {}, prompt_funcs = {}, step_funcs = {}, tool_wrappers = {} } +end + +-- Agent A thinks and hands off inside a tool round; B runs the next step. +local function handoff_history() + local answer = { role = "assistant", content = text("handing off"), metadata = { + thinking_blocks = blocks("A"), prefix_fingerprint = "fingerprint-of-agent-a" } } + local builder = prompt.new({ { role = "user", content = text("q1") }, answer }) + builder:add_function_call("handoff", '{"to":"b"}', "c1") + builder:add_function_result("handoff", "ok", "c1") + return builder, answer +end + +local function has(content, field) + for _, part in ipairs(content) do + if part[field] ~= nil then return true end + end + return false +end + +local function claude_thinking(content) + for _, part in ipairs(content) do + if part.type == "thinking" then return true end + end + return false +end + +local ROUTES = { + { driver = "Claude", id = "claude_route", between = { thinking = { type = "between_tools" } } }, + { driver = "Bedrock", id = "bedrock_route", + between = { additionalModelRequestFields = { thinking = { type = "between_tools" } } } }, +} + +local function carries_blocks(route, sent) + local turn = sent[1].payload.messages[2] + if route.driver == "Claude" then return claude_thinking(turn.content) end + return has(turn.content, "reasoningContent") +end + +local function sent_thinking(route, sent) + if route.driver == "Claude" then return sent[1].payload.thinking end + local fields = sent[1].payload.additionalModelRequestFields + return fields and fields.thinking +end + +local function define_tests() + describe("Agent replay safety", function() + after_each(function() + llm._providers = nil + llm._models = nil + llm._usage_tracker = nil + claude_generate._client = nil + bedrock_generate._client = nil + end) + + for _, route in ipairs(ROUTES) do + local UNBINDABLE = { + { name = "default budget", thinking = nil, effort = 0 }, + { name = "adaptive at effort 0", thinking = "adaptive", effort = 0 }, + { name = "none", thinking = "none", effort = 0 }, + { name = "none with a positive effort", thinking = "none", effort = 40 }, + } + for _, case in ipairs(UNBINDABLE) do + it("sends no stale block on " .. route.driver .. ": " .. case.name, function() + local sent = install({ id = route.id, provider_model = "wire", thinking = case.thinking }) + local builder, answer = handoff_history() + assert(agent.new(spec("agent_b", case.effort)):step(builder, { drop_stale_thinking = true })) + test.is_false(carries_blocks(route, sent)) + test.is_nil(sent_thinking(route, sent)) + test.eq(answer.metadata.thinking_blocks[1].signature, "sig-A") + test.is_nil(answer.metadata.stale_thinking) + end) + end + + it("sends no stale block on " .. route.driver .. ": between_tools", function() + local sent = install({ id = route.id, provider_model = "wire", thinking = "adaptive", + options = { request = route.between } }) + local builder = handoff_history() + assert(agent.new(spec("agent_b", 30)):step(builder, { drop_stale_thinking = true })) + test.is_false(carries_blocks(route, sent)) + test.eq(sent_thinking(route, sent).type, "between_tools") + end) + + local BINDABLE = { + { name = "always at effort 0", thinking = "always", effort = 0 }, + { name = "adaptive above effort 0", thinking = "adaptive", effort = 30 }, + } + for _, case in ipairs(BINDABLE) do + it("keeps the stale block bound on " .. route.driver .. ": " .. case.name, function() + local sent = install({ id = route.id, provider_model = "wire", thinking = case.thinking }) + local builder = handoff_history() + assert(agent.new(spec("agent_b", case.effort)):step(builder, { drop_stale_thinking = true })) + test.is_true(carries_blocks(route, sent)) + test.eq(sent_thinking(route, sent).block_binding.prefix_mismatch_behavior, "drop_block") + end) + end + + it("keeps a current block on an effort-0 " .. route.driver .. " step", function() + local sent = install({ id = route.id, provider_model = "wire" }) + local probe = prompt.new() + probe:add_user("x") + local fingerprint = assert(agent.new(spec("agent_b", 0)):step(probe, {})).prefix_fingerprint + sent = install({ id = route.id, provider_model = "wire" }) + local builder, answer = handoff_history() + answer.metadata.prefix_fingerprint = fingerprint + assert(agent.new(spec("agent_b", 0)):step(builder, { drop_stale_thinking = true })) + test.is_true(carries_blocks(route, sent)) + end) + + it("sends no empty system block for an empty prompt on " .. route.driver, function() + for _, empty in ipairs({ "", " \n" }) do + local sent = install({ id = route.id, provider_model = "wire" }) + local builder = prompt.new() + builder:add_user("q") + assert(agent.new(spec("agent_b", 0, empty)):step(builder, {})) + test.is_nil(sent[1].payload.system) + end + end) + end + + it("marks only what the host's predicate rejects", function() + local sent = install({ id = "claude_route", provider_model = "wire" }) + local builder = handoff_history() + assert(agent.new(spec("agent_b", 0)):step(builder, { + drop_stale_thinking = true, + thinking_is_current = function() return true end, + })) + test.is_true(claude_thinking(sent[1].payload.messages[2].content)) + end) + + it("marks nothing without drop_stale_thinking", function() + local sent = install({ id = "claude_route", provider_model = "wire" }) + local builder = handoff_history() + assert(agent.new(spec("agent_b", 0)):step(builder, {})) + test.is_true(claude_thinking(sent[1].payload.messages[2].content)) + end) + + describe("prefix fingerprint", function() + local function fingerprint(tool_meta, prompt_text) + install({ id = "claude_route", provider_model = "wire" }) + local compiled = spec("agent_b", 0, prompt_text) + compiled.tools.lookup.meta = tool_meta + local builder = prompt.new() + builder:add_user("q") + return assert(agent.new(compiled):step(builder, {})).prefix_fingerprint + end + + it("ignores meta.claude without a type, which never reaches the wire", function() + test.eq(fingerprint({ claude = { display_width_px = 10 } }), fingerprint({})) + end) + + it("ignores a meta.claude that is not a table", function() + test.eq(fingerprint({ claude = "native" }), fingerprint({})) + end) + + it("covers meta.claude with a type, which replaces the tool on the wire", function() + test.is_false(fingerprint({ claude = { type = "computer_20251124" } }) == fingerprint({})) + test.is_false(fingerprint({ claude = { type = "computer_20251124", display_width_px = 10 } }) + == fingerprint({ claude = { type = "computer_20251124", display_width_px = 20 } })) + end) + + it("ignores blank system text, which never reaches the wire", function() + test.eq(fingerprint({}, ""), fingerprint({}, " \n")) + end) + + it("ignores a blank conversation system message", function() + install({ id = "claude_route", provider_model = "wire" }) + local plain = prompt.new() + plain:add_user("q") + local with_blank = prompt.new({ + { role = "system", content = text(" ") }, + { role = "user", content = text("q") }, + }) + local compiled = agent.new(spec("agent_b", 0)) + test.eq(assert(compiled:step(with_blank, {})).prefix_fingerprint, + assert(compiled:step(plain, {})).prefix_fingerprint) + end) + end) + end) +end + +return test.run_cases(define_tests) diff --git a/src/llm/src/README.md b/src/llm/src/README.md index 517ddef9..8cb27740 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -117,6 +117,8 @@ Keys a driver builds structurally cannot be set this way; a `request` naming one A dotted entry reserves that nested field and every parent of it: a structured-output request may still set `text.verbosity` or `output_config.effort`, but not replace `text` or `output_config` with a value that is not an object. Generate leaves the structured-output fields open. +The output a request sends is decided once per route, before the card's `output_tokens` cap and context admission. Budget thinking (declared, or the Claude and Bedrock default at a positive effort) needs `max_tokens` above its budget, so a smaller or unset value rises to budget + 1024. The card cap then applies to the result and the budget shrinks to fit, keeping 1024 tokens for the answer where the 1024 minimum budget allows; a cap of 1024 or less sends no thinking. A changed caller value is reported as `metadata.adjusted.max_tokens = { requested, sent }` where `sent` is the wire value, a shrunk budget as `metadata.adjusted.thinking_budget`, and dropped thinking as `metadata.adjusted.thinking_effort`; `strict` rejects all three. Admission reserves that output, including the 2000 tokens Claude and Bedrock send for an unset `max_tokens`. A driver declares those two facts in its binding's `meta.output_policy` (`thinking`, `default_max_tokens`); without one, only the card cap on a caller value applies. + `request` is raw wire data: route facts, the card's `output_tokens` cap and `metadata.adjusted` describe the contract options, not the request. A route that declares `sampling: false` and sends `request.temperature` sends that temperature. Maps merge into the mapped body rather than replacing it, so a `request.thinking` on a budget route keeps the mapped `budget_tokens`; declare the matching route facts instead of overriding them through `request`. `request` applies to `generate` and `structured_output`; `embed`, `evaluate` and `status` do not send it. `request` belongs to the route: route `options.request` from the catalog always applies, while a caller `request` on a resolved call is rejected with `invalid_request`, like caller `accepts`. Direct `provider_id` calls may pass `request`: @@ -131,13 +133,15 @@ local response, err = llm.generate("Hello", { ### Thinking block binding -Claude and Bedrock store each response's `thinking` blocks as `{ type, thinking, signature }` and `redacted_thinking` blocks as `{ type, data }` in `metadata.thinking_blocks`, in response order, and replay every stored block on later turns in its position (Bedrock: as `reasoningContent.reasoningText` and `reasoningContent.redactedContent`). Claude replays each block unchanged except for fields an earlier stored shape carried that its type does not accept (`data` on `thinking`, `thinking` and `signature` on `redacted_thinking`). A Claude thinking block is bound to the conversation that produced it: the top-level system prompt, the tools, and every earlier message. After an agent handoff, a prompt or tool change, or a history edit, models that enforce the check reject a request that replays such a block with a 400. +Claude and Bedrock store each response's `thinking` blocks as `{ type, thinking, signature }` and `redacted_thinking` blocks as `{ type, data }` in `metadata.thinking_blocks`, in response order (Bedrock: replayed as `reasoningContent.reasoningText` and `reasoningContent.redactedContent`). A response that is thinking, then at most one text block, then tool calls replays in that order with no record. Any other response, such as thinking between text and tool calls, also carries `metadata.content_positions` and a `position` on each thinking block; a host that stores `content_positions` next to `thinking_blocks` on the assistant row (or the first function call row of the response when it has no text) gets the exact response order back, with each text block separate. Rows without `content_positions`, including every row stored before it existed, replay as all the thinking blocks of the row, then its text as one block, then its calls; so do rows whose text no longer equals the recorded text. Positions never reach the wire, and other drivers ignore them. Claude replays each block unchanged except for fields an earlier stored shape carried that its type does not accept (`data` on `thinking`, `thinking` and `signature` on `redacted_thinking`). A Claude thinking block is bound to the conversation that produced it: the top-level system prompt, the tools, and every earlier message. After an agent handoff, a prompt or tool change, or a history edit, models that enforce the check reject a request that replays such a block with a 400. When a request replays at least one thinking block and sends a thinking object of type `adaptive` or `enabled` (a positive thinking_effort, or `thinking = always` at any effort), the driver sets `thinking.block_binding.prefix_mismatch_behavior = "drop_block"` and adds the `thinking-binding-controls-2026-08-01` beta. The API then drops the first mismatched block and every later thinking block instead of rejecting the request. The drop applies to that request only and the history keeps the blocks, so after one change to the system prompt, tools or an earlier message, every later request again drops every thinking block from the first one bound to the old prefix onward, until the history before the change no longer carries thinking blocks; such conversations continue without that earlier reasoning, and `metadata.input_transformations` shows the drops on every response. Setting `prefix_mismatch_behavior` also opts the request into enforcement: on models that enforce the check only for organizations created on or after 2026-08-31, an older organization's mismatched blocks reached the model before and are now dropped. A `block_binding` the thinking object already carries is kept as set, and any request whose thinking object carries `block_binding` sends the beta. The Claude driver sends one `anthropic-beta` header: the configured `beta_features`, then the betas of a configured `anthropic-beta` context header (any letter case), then request betas, each named once. Bedrock appends the beta to the `anthropic_beta` list of `additionalModelRequestFields` and `/input_transformations` to `additionalModelResponseFieldPaths`, each named once. Plain generation and structured output behave the same. The driver decides the binding after merging `options.request`, from the thinking object as sent: a `request.thinking.type = "between_tools"` gets no binding and no beta, and a `request.thinking.block_binding` is kept as set and sends the beta. A request without a thinking object (`adaptive` or `budget` at zero or absent effort, or `thinking = "none"`) carries no binding: `block_binding` exists only inside an adaptive or enabled thinking object, and the driver never turns thinking on to carry it, since on several models an omitted thinking object means no thinking. Such a request replays its blocks unbound, so keep that conversation append-only. `always` is the form that carries the binding at zero effort. -Claude Sonnet 5.5 `thinking.type = "between_tools"` rejects `block_binding`, so a between_tools request has no drop_block recovery. Routes that must survive agent handoffs or history edits use `thinking: always` with a low default effort (`options.thinking_effort: 10`, the `low` level; 20 through 50 map to `medium`); keep between_tools for append-only routes. between_tools also rejects the `xhigh` and `max` levels (thinking_effort 80 and above) and per-message effort changes. +A message whose `metadata.stale_thinking` is true carries blocks the caller knows are bound to another prefix (`mark_stale_thinking`, which the agent runner calls). Claude and Bedrock remove those blocks when the final thinking object, after `options.request` and for that route, cannot carry a binding, so the request succeeds and later requests do not repeat a rejection. An adaptive or enabled thinking object keeps them, bound with `drop_block`. Only the request changes; the stored history keeps its blocks. + +Claude Sonnet 5.5 `thinking.type = "between_tools"` rejects `block_binding`, so a between_tools request has no drop_block recovery for blocks nothing marks as stale. Routes that must survive agent handoffs or history edits use `thinking: always` with a low default effort (`options.thinking_effort: 10`, the `low` level; 20 through 50 map to `medium`); keep between_tools for append-only routes. between_tools also rejects the `xhigh` and `max` levels (thinking_effort 80 and above) and per-message effort changes. Responses to a bound request report each dropped block in `metadata.input_transformations`, for example `{ type = "thinking_dropped", path = "messages.1.content.0", reason = "prefix_binding_mismatch" }`. `prefix_binding_mismatch` means the conversation before the block changed; `model_binding_mismatch` means the conversation moved to a model that cannot read the block. Other values exist and later checks add more, for example the reason `organization_binding_mismatch` (a block from another organization) and the entry type `thinking_mismatch_allowed` (a block that failed the check but still reached the model); ignore entry types and reasons you do not recognize. An empty array means nothing was dropped. Streaming Claude responses report the array from `message_start`. After a mid-stream server-side fallback the API sends the final array again in the closing `message_delta`, in a location that is not documented; the driver does not capture that final array yet, so such a stream reports the array from `message_start`. Bedrock reports it from `messageStop`. @@ -250,6 +254,17 @@ Developer messages, positional or not, render the same way on Claude and Bedrock Bedrock sends developer messages as user text instead of `system` blocks. A Bedrock conversation stored before this change therefore renders a different prefix once after upgrading: its cache restarts and its earlier thinking blocks no longer match. The provider rejects such blocks unless the request lets it drop them; hosts that keep thinking blocks across the upgrade strip them with `drop_stale_thinking`. +### Blank Content + +One rule decides what is blank: text with no non-whitespace character. + +- Blank text in user, system and developer messages never reaches a driver. A message left with no content goes entirely, and an image beside blank text stays. Tool results are sent as they are. +- An assistant turn, after consolidating its assistant and function call rows, with no visible text and no call renders as nothing on every driver, its thinking blocks included. Claude invents no text for it. Whitespace text in a turn with a call stays on Claude and Bedrock, because the replay has to match the response. + +A stored conversation with such a row renders a different prefix once after upgrading (Claude sent the text `intercepted`, Bedrock the thinking blocks alone, OpenAI-compatible an empty assistant message): its cache restarts from that row and later thinking blocks no longer match. + +A request whose mapped conversation is empty fails with `invalid_request` before any HTTP request on Claude, Bedrock, Google and OpenAI. A system-role note as the only content stays a valid request on OpenAI, as does any system message on OpenAI-compatible. + ### Thinking Blocks Assistant and function call messages carry provider reasoning in `metadata.thinking_blocks`. Each block is bound to the prefix before it and chained to the previous block, so blocks can be dropped only from the front of the history. `drop_stale_thinking` takes a predicate that reports whether a message's blocks are still valid and removes the blocks of every message up to and including the last one the predicate rejects: @@ -260,7 +275,9 @@ local builder, err = builder:drop_stale_thinking(function(msg) end) ``` -An empty `thinking_blocks` list counts as no blocks; messages without blocks are not consulted. The current round, the last assistant turn while the request continues it, is never stripped and never consulted: the provider requires the final assistant turn of a tool round to keep its thinking. The round continues while only its function results and user content that joins the tool-result turn follow it (a steering message, a developer or positional note, or the image message `get_messages` inserts after a tool result with `_images`). User content after the turn with no function result in between ends it: a user message, a developer message or a positional system note, which drivers render as a new user turn. Ordinary system messages, which drivers hoist into the top-level system, and cache markers do not. Stale blocks in the current round are left to the provider's own drop-on-mismatch handling; once a later assistant turn exists, or the turn is answered without tools and a user message follows, it is strippable like any other. An assistant turn is the run of consecutive assistant and function call messages. If a turn whose blocks would be removed still has a function call without a result (a function call message or a `function_call` content part), the call fails and leaves the history unchanged: the provider needs those blocks to continue the tool round. +`mark_stale_thinking` takes the same predicate and sets `metadata.stale_thinking` on copies of every message that carries blocks, up to and including the last one the predicate rejects. Call it after `drop_stale_thinking` to mark the current round's stale blocks, which that call keeps; the driver removes marked blocks when the request cannot bind them. + +An empty `thinking_blocks` list counts as no blocks; messages without blocks are not consulted. The current round, the last assistant turn while the request continues it, is never stripped and never consulted: the provider requires the final assistant turn of a tool round to keep its thinking. The round continues while only its function results and user content that joins the tool-result turn follow it (a steering message, a developer or positional note, or the image message `get_messages` inserts after a tool result with `_images`). User content after the turn with no function result in between ends it: a user message, a developer message or a positional system note, which drivers render as a new user turn. Ordinary system messages, which drivers hoist into the top-level system, and cache markers do not. Stale blocks in the current round are left to the provider's own drop-on-mismatch handling, or to the driver when they are marked and the request cannot bind them; once a later assistant turn exists, or the turn is answered without tools and a user message follows, it is strippable like any other. An assistant turn is the run of consecutive assistant and function call messages. If a turn whose blocks would be removed still has a function call without a result (a function call message or a `function_call` content part), the call fails and leaves the history unchanged: the provider needs those blocks to continue the tool round. ### Builder Operations diff --git a/src/llm/src/_index.yaml b/src/llm/src/_index.yaml index 5790f6c0..c9a96afe 100644 --- a/src/llm/src/_index.yaml +++ b/src/llm/src/_index.yaml @@ -269,6 +269,7 @@ entries: "temperature": {"type": "number", "minimum": 0, "maximum": 2}, "max_tokens": {"type": "integer", "minimum": 1}, "thinking_effort": {"type": "integer", "minimum": 0, "maximum": 100}, + "thinking_budget_tokens": {"type": "integer", "minimum": 1024, "description": "Budget thinking tokens as planned by llm before the request; drivers called directly derive it from thinking_effort"}, "request": {"type": "object", "additionalProperties": true, "description": "Provider wire fields deep-merged into the request body after option mapping; request values win, maps merge, arrays replace. Keys the driver builds (messages, tools, ...) are rejected. Set on routes in the catalog; callers may set it only on direct provider_id calls"}, "tool_choice_fallback": {"type": "string", "enum": ["auto"], "description": "Permits sending a forced tool_choice as 'auto' to a model that cannot be forced; for callers that enforce tool use themselves. The response metadata.tool_choice reports the substitution"}, "model_profile": { @@ -561,6 +562,7 @@ entries: "temperature": {"type": "number", "minimum": 0, "maximum": 2}, "max_tokens": {"type": "integer", "minimum": 1}, "thinking_effort": {"type": "integer", "minimum": 0, "maximum": 100}, + "thinking_budget_tokens": {"type": "integer", "minimum": 1024, "description": "Budget thinking tokens as planned by llm before the request; drivers called directly derive it from thinking_effort"}, "request": {"type": "object", "additionalProperties": true, "description": "Provider wire fields deep-merged into the request body after option mapping; request values win, maps merge, arrays replace. Keys the driver builds (messages, tools, ...) are rejected. Set on routes in the catalog; callers may set it only on direct provider_id calls"}, "model_profile": { "type": "object", diff --git a/src/llm/src/bedrock/_index.yaml b/src/llm/src/bedrock/_index.yaml index 0cbea71e..45bb410e 100644 --- a/src/llm/src/bedrock/_index.yaml +++ b/src/llm/src/bedrock/_index.yaml @@ -121,6 +121,7 @@ entries: imports: sigv4: wippy.llm.bedrock:sigv4 bedrock_credentials: wippy.llm.bedrock:credentials + thinking: wippy.llm.util:thinking transport: wippy.llm.util:transport - name: mapper @@ -240,6 +241,9 @@ entries: meta: name: AWS Bedrock LLM Driver comment: AWS Bedrock provider using Converse API for text generation and InvokeModel for embeddings + output_policy: + thinking: budget + default_max_tokens: 2000 tags: - bedrock - aws diff --git a/src/llm/src/bedrock/client.lua b/src/llm/src/bedrock/client.lua index 5c42a188..b0bb8b1c 100644 --- a/src/llm/src/bedrock/client.lua +++ b/src/llm/src/bedrock/client.lua @@ -4,6 +4,7 @@ local base64 = require("base64") local env = require("env") local ctx = require("ctx") local transport = require("transport") +local thinking = require("thinking") local sigv4 = require("sigv4") local credentials = require("bedrock_credentials") @@ -416,6 +417,7 @@ function bedrock_client.process_converse_stream(stream_response, callbacks) local error_reported = false local metadata = stream_response.metadata or {} local redacted_bytes: {[number]: string} = {} + local layout = {} local function handle_event(event): string? -- Converse stream events use top-level keys instead of a "type" field @@ -434,6 +436,7 @@ function bedrock_client.process_converse_stream(stream_response, callbacks) content_blocks[index] = start_block if start_block.toolUse then + layout[index] = { kind = "tool_use", id = start_block.toolUse.toolUseId } tool_calls[index] = { id = start_block.toolUse.toolUseId or "", name = start_block.toolUse.name or "", @@ -448,6 +451,8 @@ function bedrock_client.process_converse_stream(stream_response, callbacks) if delta.text then local text_chunk = tostring(delta.text) full_content = full_content .. text_chunk + layout[index] = layout[index] or { kind = "text", text = "" } + layout[index].text = layout[index].text .. text_chunk on_content(text_chunk) elseif delta.reasoningContent then -- ConverseStream opens reasoning blocks with their first delta, not contentBlockStart. @@ -462,9 +467,11 @@ function bedrock_client.process_converse_stream(stream_response, callbacks) end thinking_blocks[index] = thinking_blocks[index] or { type = "redacted_thinking", data = "" } redacted_bytes[index] = (redacted_bytes[index] or "") .. bytes + layout[index] = { kind = "thinking", block = thinking_blocks[index] } else local block = thinking_blocks[index] or { type = "thinking", thinking = "", signature = "" } thinking_blocks[index] = block + layout[index] = { kind = "thinking", block = block } if rc.text then on_thinking(tostring(rc.text)) block.thinking = block.thinking .. rc.text @@ -536,6 +543,7 @@ function bedrock_client.process_converse_stream(stream_response, callbacks) content = full_content, tool_calls = final_tool_calls, thinking = transport.in_index_order(thinking_blocks), + content_positions = thinking.record_positions(transport.in_index_order(layout)), finish_reason = finish_reason, usage = usage, metadata = metadata diff --git a/src/llm/src/bedrock/generate.lua b/src/llm/src/bedrock/generate.lua index 317d166d..5dfe234b 100644 --- a/src/llm/src/bedrock/generate.lua +++ b/src/llm/src/bedrock/generate.lua @@ -99,6 +99,9 @@ function generate_handler.handler(contract_args) } local mapped = generate_handler._mapper.map_messages(contract_args.messages) + if #mapped.messages == 0 then + return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(output.NO_TURN_MESSAGE):build() + end local inference_config, additional_fields, adjusted, options_err = generate_handler._mapper.map_options(contract_args.options or {}, contract_args.accepts) if options_err then return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(options_err):build() @@ -109,7 +112,7 @@ function generate_handler.handler(contract_args) end if not inference_config.maxTokens then - inference_config.maxTokens = 2000 + inference_config.maxTokens = mapper.DEFAULT_MAX_TOKENS end local converse_payload = { @@ -159,7 +162,7 @@ function generate_handler.handler(contract_args) } -- The binding depends on the thinking object as sent, so it is the last change to the payload. - local bind_err = generate_handler._mapper.bind_thinking(converse_payload) + local bind_err = generate_handler._mapper.bind_thinking(converse_payload, mapped.stale_thinking) if bind_err then return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(bind_err):build() end diff --git a/src/llm/src/bedrock/mapper.lua b/src/llm/src/bedrock/mapper.lua index 0115613f..28bd9cc0 100644 --- a/src/llm/src/bedrock/mapper.lua +++ b/src/llm/src/bedrock/mapper.lua @@ -24,6 +24,10 @@ mapper.CAPABILITY = { -- Wire body keys the driver builds structurally; options.request cannot set them. mapper.REQUEST_KEYS = { "messages", "system", "toolConfig" } +-- Output sent for a request that leaves max_tokens unset; the driver binding's +-- meta.output_policy declares the same value so admission can reserve it. +mapper.DEFAULT_MAX_TOKENS = 2000 + local MAX_CACHE_CHECKPOINTS = 4 -- Converse stopReason -> contract finish_reason @@ -192,7 +196,7 @@ local function last_cacheable_message_position(messages) local content: any = messages[message_index].content or {} for block_index = #content, 1, -1 do local block = content[block_index] - if block and ((block.text and block.text ~= "") or block.toolUse or block.toolResult + if block and ((block.text and not prompt.is_blank(block.text)) or block.toolUse or block.toolResult or block.image or block.document) then return { message = message_index, block = block_index } end @@ -240,18 +244,76 @@ local function reasoning_content(block) return nil end +-- Merges consecutive same-role messages into one turn. +local function consolidated_turns(messages) + local consolidated = {} + for _, msg in ipairs(messages) do + if #consolidated > 0 and consolidated[#consolidated].role == msg.role then + for _, block in ipairs(msg.content) do + table.insert(consolidated[#consolidated].content, block) + end + else + table.insert(consolidated, msg) + end + end + return consolidated +end + +-- An assistant turn with no visible text and no call renders as nothing, thinking +-- included: there is nothing in it to replay against. +local function without_blank_turns(messages) + local kept = {} + for _, msg in ipairs(messages) do + local visible = msg.role ~= "assistant" + for _, block in ipairs(msg.role == "assistant" and msg.content or {}) do + if block.text then + visible = visible or not prompt.is_blank(block.text) + elseif not block.reasoningContent and not block.cachePoint then + visible = true + end + end + if visible then + table.insert(kept, msg) + end + end + return kept +end + -- Map contract messages to Converse format function mapper.map_messages(contract_messages) if not contract_messages or #contract_messages == 0 then return { messages = {}, system = nil } end + contract_messages = prompt.without_blank_text(contract_messages) + local order = thinking.new_turn_order() + local stale_thinking = {} + + local function add_reasoning_blocks(content_blocks, msg) + if not (msg.metadata and msg.metadata.thinking_blocks) then + return + end + for _, thinking_block in ipairs(msg.metadata.thinking_blocks) do + local reasoning = reasoning_content(thinking_block) + if reasoning then + order.place_thinking(reasoning, thinking_block) + if msg.metadata.stale_thinking then + stale_thinking[reasoning] = true + end + table.insert(content_blocks, reasoning) + end + end + end + local converse_messages = {} local system_blocks = {} local system_cache_positions = {} local message_cache_positions = {} for _, msg in ipairs(contract_messages) do + if msg.role ~= prompt.ROLE.ASSISTANT and msg.role ~= prompt.ROLE.FUNCTION_CALL and msg.role ~= "cache_marker" then + order.leave_turn() + end if msg.role == prompt.ROLE.SYSTEM and not prompt.is_positional_note(msg) then if type(msg.content) == "string" then table.insert(system_blocks, { text = msg.content }) @@ -296,22 +358,18 @@ function mapper.map_messages(contract_messages) local arguments = normalize_tool_arguments(msg.function_call.arguments) local content_blocks = {} - if msg.metadata and msg.metadata.thinking_blocks then - for _, thinking_block in ipairs(msg.metadata.thinking_blocks) do - local reasoning = reasoning_content(thinking_block) - if reasoning then - table.insert(content_blocks, reasoning) - end - end - end + order.enter_row(msg) + add_reasoning_blocks(content_blocks, msg) - table.insert(content_blocks, { + local tool_use = { toolUse = { toolUseId = msg.function_call.id or "", name = msg.function_call.name or "", input = arguments } - }) + } + order.place_call(tool_use, msg.function_call.id) + table.insert(content_blocks, tool_use) table.insert(converse_messages, { role = "assistant", @@ -320,33 +378,37 @@ function mapper.map_messages(contract_messages) elseif msg.role == prompt.ROLE.ASSISTANT then local content_blocks = {} - if msg.metadata and msg.metadata.thinking_blocks then - for _, thinking_block in ipairs(msg.metadata.thinking_blocks) do - local reasoning = reasoning_content(thinking_block) - if reasoning then - table.insert(content_blocks, reasoning) - end + local layout = order.enter_row(msg) + add_reasoning_blocks(content_blocks, msg) + + if layout then + for _, entry in ipairs(layout.texts) do + local text_block = { text = entry.text } + order.place_text(text_block, entry.position) + table.insert(content_blocks, text_block) end end local content = msg.content if type(content) == "string" then - if content ~= "" then + if content ~= "" and not layout then table.insert(content_blocks, { text = content }) end elseif type(content) == "table" then for _, part in ipairs(content) do - if part.type == "text" and part.text and part.text ~= "" then + if part.type == "text" and part.text and part.text ~= "" and not layout then table.insert(content_blocks, { text = part.text }) elseif part.type == "function_call" then local args = normalize_tool_arguments(part.arguments) - table.insert(content_blocks, { + local tool_use = { toolUse = { toolUseId = part.id or "", name = part.name or "", input = args } - }) + } + order.place_call(tool_use, part.id) + table.insert(content_blocks, tool_use) elseif part.type == "image" then local img = convert_image_to_converse(part) if img then @@ -398,26 +460,25 @@ function mapper.map_messages(contract_messages) local message = (converse_messages :: any)[pos.message] local block_index = math.tointeger(pos.block) if message and block_index then - table.insert(message.content, block_index + 1, { cachePoint = { type = "default" } }) + local cache_point = { cachePoint = { type = "default" } } + order.place_behind(cache_point, message.content[block_index]) + table.insert(message.content, block_index + 1, cache_point) end end end - -- Consolidate consecutive same-role messages - local consolidated = {} - for _, msg in ipairs(converse_messages) do - if #consolidated > 0 and consolidated[#consolidated].role == msg.role then - for _, block in ipairs(msg.content) do - table.insert(consolidated[#consolidated].content, block) - end - else - table.insert(consolidated, msg) + local consolidated = consolidated_turns(converse_messages) + for _, msg in ipairs(consolidated) do + if msg.role == "assistant" then + msg.content = order.rebuild(msg.content) end end return { - messages = consolidated, - system = #system_blocks > 0 and system_blocks or nil + -- Converse takes alternating roles, so turns around a removed blank turn merge. + messages = consolidated_turns(without_blank_turns(consolidated)), + system = #system_blocks > 0 and system_blocks or nil, + stale_thinking = next(stale_thinking) and stale_thinking or nil } end @@ -511,8 +572,7 @@ function mapper.map_options(contract_options, accepts): (table, table, table, st additional_fields.output_config = { effort = thinking.effort_level(contract_options.thinking_effort) } end elseif mode == "budget" and contract_options.thinking_effort and contract_options.thinking_effort > 0 then - local thinking_budget = 1024 + (24000 - 1024) * (contract_options.thinking_effort / 100) - thinking_budget = math.floor(thinking_budget + 0.5) + local thinking_budget, max_tokens = route.budget_output(contract_options) additional_fields.thinking = { type = "enabled", @@ -526,10 +586,7 @@ function mapper.map_options(contract_options, accepts): (table, table, table, st inference_config.temperature = nil end - -- Ensure max_tokens accommodates thinking budget - if not inference_config.maxTokens or inference_config.maxTokens <= thinking_budget then - inference_config.maxTokens = thinking_budget + 1024 - end + inference_config.maxTokens = max_tokens end -- Pass through top_k as additional field @@ -556,9 +613,14 @@ end -- Binds the reasoning a Converse payload replays to drop_block. Bedrock takes the beta in -- the anthropic_beta model field and returns input_transformations only when the payload -- names it in additionalModelResponseFieldPaths. Returns an error when either list a --- bound payload extends is not a list. -function mapper.bind_thinking(payload): string? +-- bound payload extends is not a list. A thinking object that cannot carry a binding +-- sends none of the blocks in `stale_blocks` (from map_messages): unbound, the API +-- would reject them. +function mapper.bind_thinking(payload, stale_blocks): string? local fields = payload.additionalModelRequestFields + if not thinking.is_bindable(fields and fields.thinking) then + thinking.remove_blocks(payload.messages, stale_blocks) + end if not fields then return nil end @@ -661,25 +723,32 @@ function mapper.extract_response_content(converse_response, tool_names) end local text_blocks = {} + local entries = {} for _, block in ipairs(converse_response.output.message.content) do if block.text then content_text = content_text .. block.text table.insert(text_blocks, block.text) + table.insert(entries, { kind = "text", text = block.text }) elseif block.toolUse then + table.insert(entries, { kind = "tool_use", id = block.toolUse.toolUseId }) table.insert(tool_calls, { id = block.toolUse.toolUseId or "", name = block.toolUse.name or "", arguments = block.toolUse.input or {} }) elseif block.reasoningContent and block.reasoningContent.reasoningText then - table.insert(thinking_blocks, { + local stored = { type = "thinking", thinking = block.reasoningContent.reasoningText.text or "", signature = block.reasoningContent.reasoningText.signature or "" - }) + } + table.insert(entries, { kind = "thinking", block = stored }) + table.insert(thinking_blocks, stored) elseif block.reasoningContent and block.reasoningContent.redactedContent then - table.insert(thinking_blocks, { type = "redacted_thinking", data = block.reasoningContent.redactedContent }) + local stored = { type = "redacted_thinking", data = block.reasoningContent.redactedContent } + table.insert(entries, { kind = "thinking", block = stored }) + table.insert(thinking_blocks, stored) end end @@ -696,6 +765,7 @@ function mapper.extract_response_content(converse_response, tool_names) }) -- Clear the content_text since the "content" was actually the tool call content_text = "" + entries = {} break end end @@ -704,7 +774,8 @@ function mapper.extract_response_content(converse_response, tool_names) return { content = content_text, tool_calls = tool_calls, - thinking_blocks = thinking_blocks + thinking_blocks = thinking_blocks, + content_positions = thinking.record_positions(entries) } end @@ -761,6 +832,7 @@ function mapper.format_success_response(converse_response, name_to_id_map) result.metadata.thinking = thinking_content result.metadata.thinking_blocks = extracted.thinking_blocks + result.metadata.content_positions = extracted.content_positions return result end @@ -789,6 +861,7 @@ function mapper.format_streaming_response(stream_result, name_to_id_map, usage, result.metadata.thinking = thinking_content result.metadata.thinking_blocks = thinking_blocks + result.metadata.content_positions = stream_result.content_positions return result end diff --git a/src/llm/src/bedrock/structured_output.lua b/src/llm/src/bedrock/structured_output.lua index b30bf3d5..e73a8224 100644 --- a/src/llm/src/bedrock/structured_output.lua +++ b/src/llm/src/bedrock/structured_output.lua @@ -92,6 +92,9 @@ function structured_output_handler.handler(contract_args) end local mapped = structured_output_handler._mapper.map_messages(contract_args.messages) + if #mapped.messages == 0 then + return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(output.NO_TURN_MESSAGE):build() + end local inference_config, additional_fields, adjusted, options_err = structured_output_handler._mapper.map_options(contract_args.options or {}, contract_args.accepts) if options_err then return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(options_err):build() @@ -102,7 +105,7 @@ function structured_output_handler.handler(contract_args) end if not inference_config.maxTokens then - inference_config.maxTokens = 2000 + inference_config.maxTokens = mapper.DEFAULT_MAX_TOKENS end -- Build tool for structured output extraction @@ -144,7 +147,7 @@ function structured_output_handler.handler(contract_args) end -- The binding depends on the thinking object as sent, so it is the last change to the payload. - local bind_err = structured_output_handler._mapper.bind_thinking(converse_payload) + local bind_err = structured_output_handler._mapper.bind_thinking(converse_payload, mapped.stale_thinking) if bind_err then return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(bind_err):build() end diff --git a/src/llm/src/claude/_index.yaml b/src/llm/src/claude/_index.yaml index 2b288832..c3724650 100644 --- a/src/llm/src/claude/_index.yaml +++ b/src/llm/src/claude/_index.yaml @@ -40,6 +40,7 @@ entries: imports: mapper: wippy.llm.claude:mapper output: wippy.llm:output + thinking: wippy.llm.util:thinking transport: wippy.llm.util:transport # wippy.llm.claude:driver @@ -48,6 +49,9 @@ entries: meta: name: Claude LLM Driver comment: Claude implementation of LLM contracts with context-based configuration and streaming support + output_policy: + thinking: budget + default_max_tokens: 2000 tags: - claude - llm diff --git a/src/llm/src/claude/client.lua b/src/llm/src/claude/client.lua index cb84c358..0e09b338 100644 --- a/src/llm/src/claude/client.lua +++ b/src/llm/src/claude/client.lua @@ -3,6 +3,7 @@ local http_client = require("http_client") local env = require("env") local ctx = require("ctx") local transport = require("transport") +local thinking = require("thinking") type ClaudeConfig = { api_key: string?, @@ -223,7 +224,7 @@ function claude_client.request(endpoint_path, payload, options): (any, transport return parsed end -local function stream_result(full_content, tool_calls, thinking_blocks, finish_reason, usage, metadata) +local function stream_result(full_content, tool_calls, thinking_blocks, layout, finish_reason, usage, metadata) local final_tool_calls = {} for _, tool_call in ipairs(transport.in_index_order(tool_calls)) do if type(tool_call) == "table" and tool_call.id then @@ -235,6 +236,7 @@ local function stream_result(full_content, tool_calls, thinking_blocks, finish_r content = full_content, tool_calls = final_tool_calls, thinking = transport.in_index_order(thinking_blocks), + content_positions = thinking.record_positions(transport.in_index_order(layout)), finish_reason = finish_reason, usage = usage, metadata = metadata @@ -259,6 +261,7 @@ function claude_client.process_stream(stream_response, callbacks) local finish_reason = nil local usage = {} local content_blocks = {} + local layout = {} local leftover = "" local metadata = stream_response.metadata or {} @@ -331,6 +334,11 @@ function claude_client.process_stream(stream_response, callbacks) data = data.content_block.data } end + if thinking_blocks[data.index] then + layout[data.index] = { kind = "thinking", block = thinking_blocks[data.index] } + elseif data.content_block.type == "tool_use" then + layout[data.index] = { kind = "tool_use", id = data.content_block.id } + end end elseif event_type == "content_block_delta" then local index = data.index or 0 @@ -339,6 +347,8 @@ function claude_client.process_stream(stream_response, callbacks) if delta.type == "text_delta" then local text_chunk = delta.text or "" full_content = full_content .. text_chunk + layout[index] = layout[index] or { kind = "text", text = "" } + layout[index].text = layout[index].text .. text_chunk on_content(text_chunk) elseif delta.type == "thinking_delta" then local thinking_chunk = delta.thinking or "" @@ -395,7 +405,7 @@ function claude_client.process_stream(stream_response, callbacks) end end elseif event_type == "message_stop" then - local result = stream_result(full_content, tool_calls, thinking_blocks, finish_reason, usage, metadata) + local result = stream_result(full_content, tool_calls, thinking_blocks, layout, finish_reason, usage, metadata) on_done(result) return full_content, nil, result elseif event_type == "error" then @@ -411,7 +421,7 @@ function claude_client.process_stream(stream_response, callbacks) ::continue:: end - local result = stream_result(full_content, tool_calls, thinking_blocks, finish_reason, usage, metadata) + local result = stream_result(full_content, tool_calls, thinking_blocks, layout, finish_reason, usage, metadata) on_done(result) return full_content, nil, result end diff --git a/src/llm/src/claude/generate.lua b/src/llm/src/claude/generate.lua index 71c36e5d..81ac6e5b 100644 --- a/src/llm/src/claude/generate.lua +++ b/src/llm/src/claude/generate.lua @@ -101,6 +101,9 @@ function generate_handler.handler(contract_args) } local mapped_messages = generate_handler._mapper.map_messages(contract_args.messages) + if #mapped_messages.messages == 0 then + return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(output.NO_TURN_MESSAGE):build() + end local mapped_options, adjusted, options_err = generate_handler._mapper.map_options(contract_args.options or {}, contract_args.accepts) if options_err then return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(options_err):build() @@ -113,7 +116,7 @@ function generate_handler.handler(contract_args) local claude_payload = { model = contract_args.model, messages = mapped_messages.messages, - max_tokens = mapped_options.max_tokens or 2000 + max_tokens = mapped_options.max_tokens or mapper.DEFAULT_MAX_TOKENS } if mapped_messages.system then @@ -165,7 +168,7 @@ function generate_handler.handler(contract_args) end -- The binding depends on the thinking object as sent, so it is the last change to the payload. - request_options.beta_features = generate_handler._mapper.bind_thinking(claude_payload) + request_options.beta_features = generate_handler._mapper.bind_thinking(claude_payload, mapped_messages.stale_thinking) local response, request_err = generate_handler._client.request( generate_handler._client.ENDPOINTS.MESSAGES, diff --git a/src/llm/src/claude/mapper.lua b/src/llm/src/claude/mapper.lua index 8601a0fa..041d0472 100644 --- a/src/llm/src/claude/mapper.lua +++ b/src/llm/src/claude/mapper.lua @@ -55,6 +55,10 @@ mapper.HTTP_STATUS_MAP[502] = output.ERROR_TYPE.SERVER_ERROR mapper.HTTP_STATUS_MAP[503] = output.ERROR_TYPE.SERVER_ERROR mapper.HTTP_STATUS_MAP[529] = output.ERROR_TYPE.SERVER_ERROR +-- Output sent for a request that leaves max_tokens unset; the driver binding's +-- meta.output_policy declares the same value so admission can reserve it. +mapper.DEFAULT_MAX_TOKENS = 2000 + -- Claude API cache control limit local MAX_CACHE_BREAKPOINTS = 4 @@ -112,7 +116,7 @@ local function last_cacheable_message_position(messages) for block_index = #content, 1, -1 do local block = content[block_index] if block and ( - (block.type == "text" and block.text and block.text ~= "") + (block.type == "text" and not prompt.is_blank(block.text)) or block.type == "tool_use" or block.type == "tool_result" or block.type == "image" or block.type == "document" ) then @@ -230,7 +234,8 @@ local STALE_THINKING_FIELDS: {[string]: {[string]: boolean}} = { } -- A thinking or redacted_thinking block as the API takes it back: every field passes --- through unchanged except the stale fields of its type; other blocks yield nil. +-- through unchanged except the stale fields of its type and the recorded response +-- position, which only orders the replay; other blocks yield nil. local function thinking_block_shape(block) local stale = STALE_THINKING_FIELDS[block.type] if not stale then @@ -238,7 +243,7 @@ local function thinking_block_shape(block) end local shaped = {} for key, value in pairs(block) do - if not stale[key] then + if not stale[key] and key ~= "position" then shaped[key] = value end end @@ -375,13 +380,24 @@ local function consolidate_messages(messages) return result end -local function ensure_content_exists(messages) - for i = 1, #messages - 1 do - if not messages[i].content or #messages[i].content == 0 then - messages[i].content = {{ type = "text", text = "intercepted" }} +-- An assistant turn with no visible text and no call renders as nothing, thinking +-- included: there is nothing in it to replay against. +local function without_blank_turns(messages) + local kept = {} + for _, msg in ipairs(messages) do + local visible = msg.role ~= "assistant" + for _, block in ipairs(msg.role == "assistant" and msg.content or {}) do + if block.type == "text" then + visible = visible or not prompt.is_blank(block.text) + elseif block.type ~= "thinking" and block.type ~= "redacted_thinking" then + visible = true + end + end + if visible then + table.insert(kept, msg) end end - return messages + return kept end function mapper.map_messages(contract_messages) @@ -392,7 +408,25 @@ function mapper.map_messages(contract_messages) } end + contract_messages = prompt.without_blank_text(contract_messages) local tool_id = tool_id_mapper(contract_messages) + local order = thinking.new_turn_order() + local stale_thinking = {} + + local function add_thinking_blocks(content_blocks, msg) + if not (msg.metadata and msg.metadata.thinking_blocks) then + return + end + for _, thinking_block in ipairs(msg.metadata.thinking_blocks) do + local wire_block = thinking_block_shape(thinking_block) or thinking_block + order.place_thinking(wire_block, thinking_block) + if msg.metadata.stale_thinking then + stale_thinking[wire_block] = true + end + table.insert(content_blocks, wire_block) + end + end + local claude_messages = {} local system_blocks = {} local system_cache_positions = {} @@ -400,6 +434,9 @@ function mapper.map_messages(contract_messages) local in_system_phase = true for i, msg in ipairs(contract_messages) do + if msg.role ~= prompt.ROLE.ASSISTANT and msg.role ~= prompt.ROLE.FUNCTION_CALL and msg.role ~= "cache_marker" then + order.leave_turn() + end if prompt.is_positional_note(msg) or msg.role == prompt.ROLE.DEVELOPER then -- Notes join the user turn they follow as blocks after that turn's -- content. Existing blocks are never edited, so the rendering does @@ -467,18 +504,17 @@ function mapper.map_messages(contract_messages) local arguments = normalize_tool_arguments(msg.function_call.arguments) local content_blocks = {} - if msg.metadata and msg.metadata.thinking_blocks then - for _, thinking_block in ipairs(msg.metadata.thinking_blocks) do - table.insert(content_blocks, thinking_block_shape(thinking_block) or thinking_block) - end - end + order.enter_row(msg) + add_thinking_blocks(content_blocks, msg) - table.insert(content_blocks, { + local tool_use = { type = "tool_use", id = tool_id(msg.function_call.id), name = msg.function_call.name, input = arguments - }) + } + order.place_call(tool_use, msg.function_call.id) + table.insert(content_blocks, tool_use) table.insert(claude_messages, { role = "assistant", @@ -488,15 +524,20 @@ function mapper.map_messages(contract_messages) in_system_phase = false local content_blocks = {} - if msg.metadata and msg.metadata.thinking_blocks then - for _, thinking_block in ipairs(msg.metadata.thinking_blocks) do - table.insert(content_blocks, thinking_block_shape(thinking_block) or thinking_block) - end - end + local layout = order.enter_row(msg) + add_thinking_blocks(content_blocks, msg) local regular_content = process_content_array(msg.content) - if type(regular_content) == "string" and regular_content ~= "" then + if layout then + for _, entry in ipairs(layout.texts) do + local text_block = { type = "text", text = entry.text } + order.place_text(text_block, entry.position) + table.insert(content_blocks, text_block) + end + end + + if type(regular_content) == "string" and regular_content ~= "" and not layout then table.insert(content_blocks, { type = "text", text = regular_content @@ -505,13 +546,15 @@ function mapper.map_messages(contract_messages) for _, part in ipairs(regular_content) do if part.type == "function_call" then local arguments = normalize_tool_arguments(part.arguments) - table.insert(content_blocks, { + local tool_use = { type = "tool_use", id = tool_id(part.id), name = part.name, input = arguments - }) - elseif part.type == "text" and part.text and part.text ~= "" then + } + order.place_call(tool_use, part.id) + table.insert(content_blocks, tool_use) + elseif part.type == "text" and part.text and part.text ~= "" and not layout then table.insert(content_blocks, part) elseif part.type ~= "text" then table.insert(content_blocks, part) @@ -562,11 +605,17 @@ function mapper.map_messages(contract_messages) end claude_messages = consolidate_messages(claude_messages) - claude_messages = ensure_content_exists(claude_messages) + for _, msg in ipairs(claude_messages) do + if msg.role == "assistant" then + msg.content = order.rebuild(msg.content) + end + end + claude_messages = without_blank_turns(claude_messages) return { messages = claude_messages, - system = #system_blocks > 0 and system_blocks or nil + system = #system_blocks > 0 and system_blocks or nil, + stale_thinking = next(stale_thinking) and stale_thinking or nil } end @@ -671,8 +720,7 @@ function mapper.map_options(contract_options, accepts): (table, table, string?) claude_options.output_config = { effort = thinking.effort_level(contract_options.thinking_effort) } end elseif mode == "budget" and contract_options.thinking_effort and contract_options.thinking_effort > 0 then - local thinking_budget = 1024 + (24000 - 1024) * (contract_options.thinking_effort / 100) - thinking_budget = math.floor(thinking_budget + 0.5) + local thinking_budget, max_tokens = route.budget_output(contract_options) claude_options.thinking = { type = "enabled", @@ -686,9 +734,7 @@ function mapper.map_options(contract_options, accepts): (table, table, string?) claude_options.temperature = nil end - if not claude_options.max_tokens or claude_options.max_tokens <= thinking_budget then - claude_options.max_tokens = thinking_budget + 1024 - end + claude_options.max_tokens = max_tokens end return claude_options, adjusted @@ -708,8 +754,13 @@ local function replays_thinking(messages): boolean end -- Binds the thinking blocks a Messages payload replays to drop_block and returns the --- request betas its block_binding requires, or nil when the payload carries none. -function mapper.bind_thinking(payload): {string}? +-- request betas its block_binding requires, or nil when the payload carries none. A +-- thinking object that cannot carry a binding sends none of the blocks in +-- `stale_blocks` (from map_messages): unbound, the API would reject them. +function mapper.bind_thinking(payload, stale_blocks): {string}? + if not thinking.is_bindable(payload.thinking) then + thinking.remove_blocks(payload.messages, stale_blocks) + end if replays_thinking(payload.messages) then thinking.bind_replayed_blocks(payload.thinking) end @@ -723,6 +774,7 @@ function mapper.extract_response_content(claude_response) local content_text = "" local tool_calls = {} local thinking_blocks = {} + local entries = {} if not claude_response or not claude_response.content then return { @@ -735,21 +787,26 @@ function mapper.extract_response_content(claude_response) for _, block in ipairs(claude_response.content) do if block.type == "text" then content_text = content_text .. (block.text or "") + table.insert(entries, { kind = "text", text = block.text }) elseif block.type == "tool_use" then + table.insert(entries, { kind = "tool_use", id = block.id }) table.insert(tool_calls, { id = block.id, name = block.name or "", arguments = block.input or {} }) elseif block.type == "thinking" or block.type == "redacted_thinking" then - table.insert(thinking_blocks, thinking_block_shape(block)) + local shaped = thinking_block_shape(block) + table.insert(entries, { kind = "thinking", block = shaped }) + table.insert(thinking_blocks, shaped) end end return { content = content_text, tool_calls = tool_calls, - thinking_blocks = thinking_blocks + thinking_blocks = thinking_blocks, + content_positions = thinking.record_positions(entries) } end @@ -793,6 +850,7 @@ function mapper.format_success_response(claude_response, model, name_to_id_map) result.metadata.thinking = thinking_content result.metadata.thinking_blocks = extracted.thinking_blocks + result.metadata.content_positions = extracted.content_positions return result end @@ -822,6 +880,7 @@ function mapper.format_streaming_response(client_result, name_to_id_map, usage, result.metadata.thinking = thinking_content result.metadata.thinking_blocks = thinking_blocks + result.metadata.content_positions = client_result.content_positions return result end diff --git a/src/llm/src/claude/structured_output.lua b/src/llm/src/claude/structured_output.lua index 8c604618..9e9f0217 100644 --- a/src/llm/src/claude/structured_output.lua +++ b/src/llm/src/claude/structured_output.lua @@ -174,6 +174,9 @@ function structured_output_handler.handler(contract_args) end local mapped_messages = structured_output_handler._mapper.map_messages(contract_args.messages) + if #mapped_messages.messages == 0 then + return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(output.NO_TURN_MESSAGE):build() + end local mapped_options, adjusted, options_err = structured_output_handler._mapper.map_options(contract_args.options or {}, contract_args.accepts) if options_err then return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(options_err):build() @@ -190,7 +193,7 @@ function structured_output_handler.handler(contract_args) local native_payload = { model = contract_args.model, messages = mapped_messages.messages, - max_tokens = mapped_options.max_tokens or 2000 + max_tokens = mapped_options.max_tokens or mapper.DEFAULT_MAX_TOKENS } if mapped_messages.system then native_payload.system = mapped_messages.system @@ -208,7 +211,7 @@ function structured_output_handler.handler(contract_args) end -- The binding depends on the thinking object as sent, so it is the last change to the payload. - local native_betas = structured_output_handler._mapper.bind_thinking(native_payload) + local native_betas = structured_output_handler._mapper.bind_thinking(native_payload, mapped_messages.stale_thinking) local response, request_err = structured_output_handler._client.request( structured_output_handler._client.ENDPOINTS.MESSAGES, native_payload, @@ -259,7 +262,7 @@ function structured_output_handler.handler(contract_args) messages = mapped_messages.messages, tools = claude_tools, tool_choice = tool_choice, - max_tokens = mapped_options.max_tokens or 2000 + max_tokens = mapped_options.max_tokens or mapper.DEFAULT_MAX_TOKENS } if mapped_messages.system then @@ -283,7 +286,7 @@ function structured_output_handler.handler(contract_args) } -- The binding depends on the thinking object as sent, so it is the last change to the payload. - request_options.beta_features = structured_output_handler._mapper.bind_thinking(claude_payload) + request_options.beta_features = structured_output_handler._mapper.bind_thinking(claude_payload, mapped_messages.stale_thinking) local response, request_err = structured_output_handler._client.request( structured_output_handler._client.ENDPOINTS.MESSAGES, diff --git a/src/llm/src/discovery/providers.lua b/src/llm/src/discovery/providers.lua index 3e8c797f..0f113e59 100644 --- a/src/llm/src/discovery/providers.lua +++ b/src/llm/src/discovery/providers.lua @@ -109,11 +109,9 @@ function providers.open(provider_id, context_overrides) return instance end --- Whether the provider's driver binding declares that it owns the legacy --- reasoning_model_request meaning (contract.binding meta.legacy_reasoning_flag). --- Resolves the provider entry's data.driver.id the same way providers.open does, --- so a custom provider entry bound to a framework driver is covered too. -function providers.driver_declares_legacy_reasoning_flag(provider_id): (boolean?, string | error | nil) +-- The contract.binding entry behind a provider entry's data.driver.id, resolved the +-- way providers.open does; nil for a provider entry without a driver. +local function driver_binding(provider_id): (any?, string | error | nil) local provider_entry, err = providers._registry.get(provider_id) if err then return nil, error_context.prefix(err, "Registry error: ") @@ -124,7 +122,7 @@ function providers.driver_declares_legacy_reasoning_flag(provider_id): (boolean? local driver = provider_entry.data and provider_entry.data.driver if not driver or not driver.id then - return false, nil + return nil, nil end local binding_entry, binding_err = providers._registry.get(tostring(driver.id)) @@ -134,8 +132,31 @@ function providers.driver_declares_legacy_reasoning_flag(provider_id): (boolean? if not binding_entry then return nil, "Driver binding not found: " .. tostring(driver.id) end + return binding_entry, nil +end + + +-- Whether the provider's driver binding declares that it owns the legacy +-- reasoning_model_request meaning (contract.binding meta.legacy_reasoning_flag). +-- Resolves the provider entry's data.driver.id the same way providers.open does, +-- so a custom provider entry bound to a framework driver is covered too. +function providers.driver_declares_legacy_reasoning_flag(provider_id): (boolean?, string | error | nil) + local binding, err = driver_binding(provider_id) + if err then + return nil, err + end + return binding ~= nil and binding.meta ~= nil and binding.meta.legacy_reasoning_flag == true, nil +end - return binding_entry.meta ~= nil and binding_entry.meta.legacy_reasoning_flag == true, nil +-- What a driver sends when a request leaves the output unset +-- (contract.binding meta.output_policy): { thinking, default_max_tokens }. +-- A driver that declares none lets the provider decide, so nothing can be reserved. +function providers.driver_output_policy(provider_id): (table?, string | error | nil) + local binding, err = driver_binding(provider_id) + if err then + return nil, err + end + return binding and binding.meta and binding.meta.output_policy or nil, nil end return providers diff --git a/src/llm/src/google/generate.lua b/src/llm/src/google/generate.lua index b79f42a5..6e1c1ee2 100644 --- a/src/llm/src/google/generate.lua +++ b/src/llm/src/google/generate.lua @@ -30,6 +30,10 @@ function generate.handler(contract_args) model = contract_args.model }) + if #messages == 0 then + return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(output.NO_TURN_MESSAGE):build() + end + local payload = { contents = messages, generationConfig = {} diff --git a/src/llm/src/google/mapper.lua b/src/llm/src/google/mapper.lua index 4677b346..dab9d7e0 100644 --- a/src/llm/src/google/mapper.lua +++ b/src/llm/src/google/mapper.lua @@ -145,6 +145,7 @@ end function mapper.map_messages(contract_messages, options) options = options or {} + contract_messages = prompt.without_blank_text(contract_messages) local processed_messages = {} local system_instructions = {} local provider_call_ids = {} @@ -180,7 +181,7 @@ function mapper.map_messages(contract_messages, options) i = i + 1 elseif msg.role == "assistant" then local assistant_msg = mapper.standardize_content(msg.content) - if assistant_msg ~= "" then + if not prompt.is_blank(assistant_msg) then table.insert(processed_messages, { role = "model", parts = {{ text = assistant_msg }} }) end diff --git a/src/llm/src/google/structured_output.lua b/src/llm/src/google/structured_output.lua index 94ba4757..b375f826 100644 --- a/src/llm/src/google/structured_output.lua +++ b/src/llm/src/google/structured_output.lua @@ -75,6 +75,10 @@ function structured_output.handler(contract_args) model = contract_args.model }) + if #messages == 0 then + return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(output.NO_TURN_MESSAGE):build() + end + local payload = { contents = messages, generationConfig = { diff --git a/src/llm/src/llm.lua b/src/llm/src/llm.lua index 9840debb..249784fb 100644 --- a/src/llm/src/llm.lua +++ b/src/llm/src/llm.lua @@ -593,11 +593,25 @@ local function prepare_route(contract_args, provider_info: any, options, provide remove("thinking_effort") end - local limit = output_limit(card) + local policy, policy_err = providers_module.driver_output_policy(provider_info.id) + if policy_err then + return nil, native_error(policy_err, errors.INTERNAL, false) + end local requested = tonumber(contract_args.options.max_tokens) - if limit ~= nil and requested ~= nil and requested > limit then - adjusted.max_tokens = { requested = requested, sent = limit } - contract_args.options.max_tokens = limit + local plan = route.plan_output(requested, contract_args.options.thinking_effort, + policy and (accepts.thinking or policy.thinking), policy and policy.default_max_tokens, output_limit(card)) + if plan.max_tokens ~= requested then + contract_args.options.max_tokens = plan.max_tokens + if requested ~= nil then + adjusted.max_tokens = { requested = requested, sent = plan.max_tokens } + end + end + if plan.dropped_thinking then + remove("thinking_effort") + end + contract_args.options.thinking_budget_tokens = plan.budget + if plan.budget ~= plan.wanted_budget then + adjusted.thinking_budget = { requested = plan.wanted_budget, sent = plan.budget } end local strict_err = route.strict_error(tostring(provider_info.id), strict, adjusted) diff --git a/src/llm/src/openai/generate.lua b/src/llm/src/openai/generate.lua index 4df06549..b8295171 100644 --- a/src/llm/src/openai/generate.lua +++ b/src/llm/src/openai/generate.lua @@ -129,6 +129,10 @@ function generate_handler.handler(contract_args) model = contract_args.model }) + if #input_items == 0 then + return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(output.NO_TURN_MESSAGE):build() + end + local payload = { model = contract_args.model, input = input_items diff --git a/src/llm/src/openai/mapper.lua b/src/llm/src/openai/mapper.lua index 27da4494..39186462 100644 --- a/src/llm/src/openai/mapper.lua +++ b/src/llm/src/openai/mapper.lua @@ -102,20 +102,24 @@ local function to_output_content_part(part) return { type = "output_text", text = tostring(part.text or "") } end +-- Input parts of a message, or nil when none has anything to send. local function normalize_input_content(content) - if content == nil or content == "" then - return { { type = "input_text", text = "" } } + if prompt.is_blank(content) then + return nil end if type(content) == "string" then return { { type = "input_text", text = content } } end if type(content) == "table" then local parts = {} - for i, p in ipairs(content) do - parts[i] = to_input_content_part(p) + for _, p in ipairs(content) do + local part = to_input_content_part(p) + if part.type ~= "input_text" or not prompt.is_blank(part.text) then + table.insert(parts, part) + end end if #parts == 0 then - return { { type = "input_text", text = "" } } + return nil end return parts end @@ -123,7 +127,7 @@ local function normalize_input_content(content) end local function normalize_output_content(content) - if content == nil or content == "" then + if prompt.is_blank(content) then return nil end if type(content) == "string" then @@ -131,8 +135,11 @@ local function normalize_output_content(content) end if type(content) == "table" then local parts = {} - for i, p in ipairs(content) do - parts[i] = to_output_content_part(p) + for _, p in ipairs(content) do + local part = to_output_content_part(p) + if not prompt.is_blank(part.text) then + table.insert(parts, part) + end end if #parts == 0 then return nil end return parts @@ -175,6 +182,7 @@ end function openai_mapper.extract_instructions(contract_messages) if not contract_messages then return nil, {} end + contract_messages = prompt.without_blank_text(contract_messages) local parts = {} local consumed = {} for i, msg in ipairs(contract_messages) do @@ -197,17 +205,21 @@ function openai_mapper.map_messages(contract_messages, options) options = options or {} if not contract_messages then return {} end + contract_messages = prompt.without_blank_text(contract_messages) local _, consumed = openai_mapper.extract_instructions(contract_messages) local items = {} for i, msg in ipairs(contract_messages) do if not consumed[i] and msg.role ~= "cache_marker" then if msg.role == "user" or msg.role == "system" or msg.role == "developer" then - table.insert(items, { - type = "message", - role = msg.role, - content = normalize_input_content(msg.content) - }) + local content = normalize_input_content(msg.content) + if content then + table.insert(items, { + type = "message", + role = msg.role, + content = content + }) + end elseif msg.role == "assistant" then local content = normalize_output_content(msg.content) if content then diff --git a/src/llm/src/openai/structured_output.lua b/src/llm/src/openai/structured_output.lua index 4b4050e2..2ef05a52 100644 --- a/src/llm/src/openai/structured_output.lua +++ b/src/llm/src/openai/structured_output.lua @@ -107,6 +107,10 @@ function structured_output_handler.handler(contract_args) model = contract_args.model }) + if #input_items == 0 then + return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(output.NO_TURN_MESSAGE):build() + end + local payload = { model = contract_args.model, input = input_items, diff --git a/src/llm/src/openai_compat/_index.yaml b/src/llm/src/openai_compat/_index.yaml index 7e48b921..5c4fd1af 100644 --- a/src/llm/src/openai_compat/_index.yaml +++ b/src/llm/src/openai_compat/_index.yaml @@ -124,6 +124,7 @@ entries: - json imports: output: wippy.llm:output + prompt: wippy.llm:prompt route: wippy.llm.util:route # wippy.llm.openai_compat:organization @@ -189,6 +190,7 @@ entries: openai_client: wippy.llm.openai_compat:client openai_mapper: wippy.llm.openai_compat:mapper output: wippy.llm:output + prompt: wippy.llm:prompt wire_request: wippy.llm.util:wire_request method: handler pool: diff --git a/src/llm/src/openai_compat/mapper.lua b/src/llm/src/openai_compat/mapper.lua index 391fd0b8..c9ba8cba 100644 --- a/src/llm/src/openai_compat/mapper.lua +++ b/src/llm/src/openai_compat/mapper.lua @@ -1,6 +1,7 @@ local json = require("json") local output = require("output") local route = require("route") +local prompt = require("prompt") local openai_mapper = {} local MAX_CACHE_BREAKPOINTS = 4 @@ -158,6 +159,7 @@ end function openai_mapper.map_messages(contract_messages, options) options = options or {} + contract_messages = prompt.without_blank_text(contract_messages) local processed_messages = {} local i = 1 local is_claude = is_claude_model(options.model) @@ -207,6 +209,8 @@ function openai_mapper.map_messages(contract_messages, options) -- The prompt builder interleaves them: call_A, result_A, call_B, result_B local function_calls_found = {} local function_results_found = {} + local open_calls = 0 + local answered = {} local scan_end = i - 1 local j = i while j <= #contract_messages do @@ -214,12 +218,28 @@ function openai_mapper.map_messages(contract_messages, options) if current_msg.role == "function_call" and current_msg.function_call and current_msg.function_call.id then table.insert(function_calls_found, current_msg) + if not answered[current_msg.function_call.id] then + open_calls = open_calls + 1 + end scan_end = j elseif current_msg.role == "function_result" then table.insert(function_results_found, current_msg) + if current_msg.function_call_id and not answered[current_msg.function_call_id] then + answered[current_msg.function_call_id] = true + for _, found in ipairs(function_calls_found) do + if found.function_call.id == current_msg.function_call_id then + open_calls = open_calls - 1 + break + end + end + end scan_end = j elseif current_msg.role == "assistant" or current_msg.role == "user" then break + elseif current_msg.role ~= "function_call" and current_msg.role ~= "cache_marker" and open_calls == 0 then + -- With every call answered the round is complete: a later call is + -- the next round, and this message stays in place between them. + break end j = j + 1 end @@ -246,7 +266,10 @@ function openai_mapper.map_messages(contract_messages, options) assistant_msg.tool_calls = nil end - table.insert(processed_messages, assistant_msg) + -- A turn with no visible text and no call renders as nothing. + if assistant_msg.tool_calls or not prompt.is_blank(assistant_msg.content) then + table.insert(processed_messages, assistant_msg) + end -- Emit function results as tool messages in order for _, result_msg in ipairs(function_results_found) do diff --git a/src/llm/src/openai_compat/structured_output.lua b/src/llm/src/openai_compat/structured_output.lua index 7f09605a..9f15a3c9 100644 --- a/src/llm/src/openai_compat/structured_output.lua +++ b/src/llm/src/openai_compat/structured_output.lua @@ -1,3 +1,4 @@ +local prompt = require("prompt") local openai_client = require("openai_client") local openai_mapper = require("openai_mapper") local output = require("output") @@ -105,7 +106,7 @@ function structured_output_handler.handler(contract_args) local openai_payload = { model = contract_args.model, - messages = contract_args.messages, + messages = prompt.without_blank_text(contract_args.messages), response_format = { type = "json_schema", json_schema = { diff --git a/src/llm/src/output.lua b/src/llm/src/output.lua index 584ead3c..4c1a97f5 100644 --- a/src/llm/src/output.lua +++ b/src/llm/src/output.lua @@ -470,6 +470,9 @@ function output.stream_error_details(started: boolean, deferred: ErrorInfo?): { end -- Message injected into conversation when LLM output is truncated mid-tool-call +-- The driver error for a request whose mapped conversation holds nothing to answer. +output.NO_TURN_MESSAGE = "Request has no conversation turn: every message is instructions or blank" + output.TRUNCATION_MSG = "Your previous response was truncated because it exceeded the maximum token limit. The tool calls you attempted were incomplete and have been discarded. Please retry with a shorter response. Break your work into smaller steps if needed." -- Detect LLM responses cut off by the token limit that produced nothing diff --git a/src/llm/src/prompt.lua b/src/llm/src/prompt.lua index c0027bcd..032d96d8 100644 --- a/src/llm/src/prompt.lua +++ b/src/llm/src/prompt.lua @@ -53,6 +53,7 @@ type PromptBuilder = { add_function_result: (self: any, name: string, content: any, function_call_id: string?, options: FunctionResultOptions?) -> PromptBuilder, add_cache_marker: (self: any, marker_id: string?) -> PromptBuilder, drop_stale_thinking: (self: any, is_current: (msg: Message) -> boolean) -> (PromptBuilder?, string?), + mark_stale_thinking: (self: any, is_current: (msg: Message) -> boolean) -> PromptBuilder, get_messages: (self: any) -> {Message}, build: (self: any) -> {messages: {Message}}, clone: (self: any) -> PromptBuilder, @@ -147,10 +148,82 @@ function prompt.developer_instruction(text: string): string return "" .. text .. "" end +-- Whether text has nothing visible in it. This one rule decides which text blocks +-- reach a provider and which assistant turns render at all. +function prompt.is_blank(text: any): boolean + return text == nil or (type(text) == "string" and not text:find("%S")) +end + +local BLANKABLE_ROLES: {[string]: boolean} = { + [prompt.ROLE.USER] = true, + [prompt.ROLE.SYSTEM] = true, + [prompt.ROLE.DEVELOPER] = true, +} + +local function is_blank_part(part: any): boolean + if type(part) == "string" then return prompt.is_blank(part) end + if type(part) ~= "table" then return false end + return (part.type == prompt.CONTENT_TYPE.TEXT or part.type == "input_text") and prompt.is_blank(part.text) +end + +-- User, system and developer messages without their blank text. A message left +-- with no content goes entirely; the caller's tables stay as they are. +function prompt.without_blank_text(messages: {any}): {any} + local kept: {any} = table.create(#messages, 0) + for _, msg in ipairs(messages) do + local content = msg.content + if not BLANKABLE_ROLES[msg.role] then + table.insert(kept, msg) + elseif type(content) == "table" then + local parts: {any} = {} + for _, part in ipairs(content) do + if not is_blank_part(part) then table.insert(parts, part) end + end + if #parts == #content and #content > 0 then + table.insert(kept, msg) + elseif #parts > 0 then + local copy: any = {} + for key, value in pairs(msg) do copy[key] = value end + copy.content = parts + table.insert(kept, copy) + end + elseif not prompt.is_blank(content) then + table.insert(kept, msg) + end + end + return kept +end + --------------------------- -- Helper Functions --------------------------- +local function in_assistant_turn(msg: Message): boolean + return msg.role == prompt.ROLE.ASSISTANT or msg.role == prompt.ROLE.FUNCTION_CALL +end + +local function carries_thinking(msg: Message): boolean + if not in_assistant_turn(msg) or type(msg.metadata) ~= "table" then return false end + local blocks = msg.metadata.thinking_blocks + return type(blocks) == "table" and #blocks > 0 +end + +-- A copy of a message with a rewritten metadata table, so message tables the +-- caller shares with the builder keep theirs. +local function with_metadata(msg: Message, rewrite: (metadata: table) -> ()): Message + local copy: Message = {} :: Message + for key, value in pairs(msg :: table) do + (copy :: any)[key] = value + end + local metadata = {} + for key, value in pairs(msg.metadata :: table) do + metadata[key] = value + end + rewrite(metadata) + copy.metadata = metadata + return copy +end + -- Process function result content to extract images and clean text local function process_function_result_content(content: any): (any, {ContentPart}?) local parsed @@ -393,21 +466,11 @@ function prompt.new(messages: {Message}?) end end - local function in_turn(msg: Message): boolean - return msg.role == prompt.ROLE.ASSISTANT or msg.role == prompt.ROLE.FUNCTION_CALL - end - - local function carries(msg: Message): boolean - if not in_turn(msg) or type(msg.metadata) ~= "table" then return false end - local blocks = msg.metadata.thinking_blocks - return type(blocks) == "table" and #blocks > 0 - end - -- First unanswered call of each assistant turn, keyed by message index. local open_call: {[integer]: string} = {} local turn_start: integer? = nil for index, msg in ipairs(messages) do - if in_turn(msg) then + if in_assistant_turn(msg) then turn_start = turn_start or index else turn_start = nil @@ -420,9 +483,11 @@ function prompt.new(messages: {Message}?) name = msg.function_call and msg.function_call.name, }) else - for _, part in ipairs(msg.content or {}) do - if (part :: any).type == "function_call" then - table.insert(ids, { id = (part :: any).id or false, name = (part :: any).name }) + if type(msg.content) == "table" then + for _, part in ipairs(msg.content) do + if (part :: any).type == "function_call" then + table.insert(ids, { id = (part :: any).id or false, name = (part :: any).name }) + end end end end @@ -447,7 +512,7 @@ function prompt.new(messages: {Message}?) local current_round = #messages + 1 local seen_result, seen_user = false, false local index = #messages - while index >= 1 and not in_turn(messages[index]) do + while index >= 1 and not in_assistant_turn(messages[index]) do local role = messages[index].role if role == prompt.ROLE.FUNCTION_RESULT then seen_result = true end if role == prompt.ROLE.USER or role == prompt.ROLE.DEVELOPER @@ -457,7 +522,7 @@ function prompt.new(messages: {Message}?) index = index - 1 end if index >= 1 and not (seen_user and not seen_result) then - while index >= 1 and in_turn(messages[index]) do + while index >= 1 and in_assistant_turn(messages[index]) do current_round = index index = index - 1 end @@ -466,7 +531,7 @@ function prompt.new(messages: {Message}?) local last_stale: integer? = nil for index = 1, current_round - 1 do local msg = messages[index] - if carries(msg) and not is_current(msg) then + if carries_thinking(msg) and not is_current(msg) then last_stale = index end end @@ -478,12 +543,12 @@ function prompt.new(messages: {Message}?) turn_start = nil for index = 1, last_stale do local msg = messages[index] - if in_turn(msg) then + if in_assistant_turn(msg) then turn_start = turn_start or index else turn_start = nil end - if carries(msg) then + if carries_thinking(msg) then local open = turn_start and open_call[turn_start] if open then return nil, string.format( @@ -493,21 +558,38 @@ function prompt.new(messages: {Message}?) end end - -- Stripped messages are replaced by copies, so message tables the - -- caller shares with the builder keep their blocks. for _, index in ipairs(stale) do - local copy: Message = {} :: Message - for key, value in pairs(messages[index] :: table) do - (copy :: any)[key] = value + messages[index] = with_metadata(messages[index], function(metadata) + metadata.thinking_blocks = nil + end) + end + return self, nil + end + + -- Marks the blocks that stay in the request although they are stale, the ones + -- drop_stale_thinking keeps for the current round. Only the caller knows they + -- are stale and only the driver knows whether the thinking object it sends can + -- carry a binding, so the driver removes marked blocks when it cannot. Marks + -- follow the same front-only rule: every block up to the last stale one. + builder.mark_stale_thinking = function(self: any, is_current: (msg: Message) -> boolean) + local messages: {Message} = self.messages :: {Message} + local last_stale: integer? = nil + for index, msg in ipairs(messages) do + if carries_thinking(msg) and not is_current(msg) then + last_stale = index end - local metadata = {} - for key, value in pairs(messages[index].metadata :: table) do - if key ~= "thinking_blocks" then metadata[key] = value end + end + if not last_stale then + return self + end + for index = 1, last_stale do + if carries_thinking(messages[index]) then + messages[index] = with_metadata(messages[index], function(metadata) + metadata.stale_thinking = true + end) end - copy.metadata = metadata - messages[index] = copy end - return self, nil + return self end -- Get all messages in the current builder (with image processing) diff --git a/src/llm/src/util/route.lua b/src/llm/src/util/route.lua index 008349b7..76c33ecb 100644 --- a/src/llm/src/util/route.lua +++ b/src/llm/src/util/route.lua @@ -105,6 +105,54 @@ function route.clean_options(options: table): table return clean end +-- Budget thinking takes a share of the response: the budget grows with the effort +-- and the response keeps room for an answer behind it. +local MIN_THINKING_BUDGET = 1024 +local MAX_THINKING_BUDGET = 24000 +local ANSWER_ROOM = 1024 + +function route.thinking_budget(effort: number): number + return math.floor(MIN_THINKING_BUDGET + (MAX_THINKING_BUDGET - MIN_THINKING_BUDGET) * (effort / 100) + 0.5) +end + +-- The output a request sends, computed once per route before the card limit and +-- admission: `mode` is the effective thinking mode, `default_max_tokens` what the +-- driver sends for an unset max_tokens and `limit` the card's output limit. +-- budget is nil when the request has no budget thinking; dropped_thinking says the +-- limit leaves no room for the smallest budget. +function route.plan_output(max_tokens: number?, effort: number?, mode: string?, default_max_tokens: number?, limit: number?): table + local thinks = mode == "budget" and effort ~= nil and effort > 0 + local wanted_budget = thinks and route.thinking_budget(effort :: number) or nil + local output = max_tokens or default_max_tokens + if wanted_budget and (not max_tokens or max_tokens <= wanted_budget) then + output = wanted_budget + ANSWER_ROOM + end + if limit and output and output > limit then + output = limit + end + if not wanted_budget then + return { max_tokens = output } + end + if output <= MIN_THINKING_BUDGET then + return { max_tokens = math.min(max_tokens or default_max_tokens or output, output), dropped_thinking = true } + end + local budget = wanted_budget + if budget >= output then + budget = math.max(output - ANSWER_ROOM, MIN_THINKING_BUDGET) + end + return { max_tokens = output, budget = budget, wanted_budget = wanted_budget } +end + +-- The budget and output a budget-thinking request sends. llm plans them before the +-- request; a driver called directly with no plan gets the same arithmetic. +function route.budget_output(options: table): (number, number) + if options.thinking_budget_tokens then + return options.thinking_budget_tokens, options.max_tokens + end + local plan = route.plan_output(options.max_tokens, options.thinking_effort, "budget", nil, nil) + return plan.budget, plan.max_tokens +end + -- A driver capability table: { name, defaults = {fact = value}, supported = {fact = {value = true}} }. -- `supported` only needs entries for facts a driver restricts beyond the global -- enum; a fact absent from `supported` accepts any globally valid value. diff --git a/src/llm/src/util/thinking.lua b/src/llm/src/util/thinking.lua index 08f62f31..fd0219f0 100644 --- a/src/llm/src/util/thinking.lua +++ b/src/llm/src/util/thinking.lua @@ -17,11 +17,183 @@ function thinking.bind_replayed_blocks(config: {[string]: any}?) config.block_binding = { prefix_mismatch_behavior = "drop_block" } end +-- Whether a thinking object can carry block_binding. +function thinking.is_bindable(config: {[string]: any}?): boolean + return type(config) == "table" and BINDABLE_TYPES[config.type] == true +end + +-- Removes the wire blocks in `stale` from every message that holds one. +function thinking.remove_blocks(messages: {any}?, stale: {[any]: boolean}?) + if not messages or not stale or next(stale) == nil then + return + end + for _, message in ipairs(messages) do + if type(message.content) == "table" then + local kept = {} + for _, block in ipairs(message.content) do + if not stale[block] then table.insert(kept, block) end + end + if #kept ~= #message.content then + message.content = kept + end + end + end +end + -- Whether a thinking object carries block_binding, which the API admits only with BINDING_BETA. function thinking.is_bound(config: {[string]: any}?): boolean return type(config) == "table" and config.block_binding ~= nil end +-- Records where a response's blocks sit, given `entries` in response order: each +-- { kind = "thinking", block = }, { kind = "text", text = ... } or +-- { kind = "tool_use", id = ... }. A thinking block keeps its position on itself and +-- the others come back as the list for metadata.content_positions. A response that +-- is thinking, then at most one text, then calls replays in that order without any +-- record, so it gets none. +function thinking.record_positions(entries: {any}): {any}? + local present: {any} = {} + for _, entry in ipairs(entries) do + if entry.kind ~= "text" or (entry.text or "") ~= "" then + table.insert(present, entry) + end + end + + local stages: {[string]: number} = { thinking = 1, text = 2, tool_use = 3 } + local stage, texts = 1, 0 + local plain = true + for _, entry in ipairs(present) do + if stages[entry.kind] < stage then plain = false end + stage = stages[entry.kind] + if entry.kind == "text" then texts = texts + 1 end + end + if plain and texts <= 1 then + return nil + end + + local positions: {any} = {} + for position, entry in ipairs(present) do + if entry.kind == "thinking" then + entry.block.position = position + elseif entry.kind == "text" then + table.insert(positions, { type = "text", position = position, text = entry.text }) + else + table.insert(positions, { type = "tool_use", position = position, id = entry.id }) + end + end + return positions +end + +local function row_text(content: any): string + if type(content) == "string" then return content end + local text = "" + if type(content) == "table" then + for _, part in ipairs(content) do + if part.type == "text" and type(part.text) == "string" then + text = text .. part.text + end + end + end + return text +end + +-- A row's recorded layout, or nil when it has none or its text is no longer the +-- text the layout recorded: the row is what the host stands behind. +local function read_layout(msg: any): any? + local positions = type(msg.metadata) == "table" and msg.metadata.content_positions or nil + if type(positions) ~= "table" then + return nil + end + local layout = { texts = {}, calls = {} } + local text = "" + for _, entry in ipairs(positions) do + if entry.type == "text" then + table.insert(layout.texts, entry) + text = text .. entry.text + elseif entry.type == "tool_use" then + layout.calls[entry.id] = entry.position + end + end + if text ~= row_text(msg.content) then + return nil + end + return layout +end + +-- Tracks the wire blocks of assistant rows against their recorded layouts and +-- rebuilds response order once the rows are consolidated into one turn. +function thinking.new_turn_order() + local placed: {[any]: any} = {} + local active: any = nil + local order = {} + + -- Returns the row's own layout, whose text blocks replace the row's text. + function order.enter_row(msg: any): any? + local layout = read_layout(msg) + if layout then active = layout end + return layout + end + + function order.leave_turn() + active = nil + end + + function order.place_thinking(wire_block: any, stored_block: any) + if active and stored_block.position then + placed[wire_block] = { layout = active, position = stored_block.position } + end + end + + function order.place_text(wire_block: any, position: number) + if active then + placed[wire_block] = { layout = active, position = position } + end + end + + function order.place_call(wire_block: any, id: any) + local position = active and active.calls[id] + if position then + placed[wire_block] = { layout = active, position = position } + end + end + + -- A block that belongs right behind `anchor`, wherever the anchor ends up. + function order.place_behind(wire_block: any, anchor: any) + local place = placed[anchor] + if place then + placed[wire_block] = { layout = place.layout, position = place.position + 0.5 } + end + end + + -- The content with each run of blocks from one layout in response order. Blocks + -- with no recorded place keep theirs, and they end a run. + function order.rebuild(content: {any}): {any} + if next(placed) == nil then + return content + end + local rebuilt: {any} = {} + local index = 1 + while index <= #content do + local place = placed[content[index]] + if not place then + table.insert(rebuilt, content[index]) + index = index + 1 + else + local run: {any} = {} + while content[index] and placed[content[index]] and placed[content[index]].layout == place.layout do + table.insert(run, content[index]) + index = index + 1 + end + table.sort(run, function(a, b) return placed[a].position < placed[b].position end) + for _, block in ipairs(run) do table.insert(rebuilt, block) end + end + end + return rebuilt + end + + return order +end + function thinking.effort_level(effort: number): string if effort >= 100 then return "max" end if effort >= 80 then return "xhigh" end diff --git a/src/llm/test/_index.yaml b/src/llm/test/_index.yaml index bafc9a4e..f7ee3519 100644 --- a/src/llm/test/_index.yaml +++ b/src/llm/test/_index.yaml @@ -256,6 +256,39 @@ entries: test: wippy.test:test method: run_tests + # app:replay_safety_test + - name: replay_safety_test + kind: function.lua + meta: + name: Replay Safety Test + type: test + group: LLM / General + source: file://replay_safety_test.lua + modules: [json, registry] + imports: + llm: wippy.llm:llm + prompt: wippy.llm:prompt + providers: wippy.llm.discovery:providers + claude_mapper: wippy.llm.claude:mapper + claude_client: wippy.llm.claude:client + claude_generate: wippy.llm.claude:generate + claude_structured: wippy.llm.claude:structured_output + bedrock_mapper: wippy.llm.bedrock:mapper + bedrock_client: wippy.llm.bedrock:client + bedrock_generate: wippy.llm.bedrock:generate + bedrock_structured: wippy.llm.bedrock:structured_output + openai_mapper: wippy.llm.openai:mapper + openai_generate: wippy.llm.openai:generate + openai_structured: wippy.llm.openai:structured_output + compat_mapper: wippy.llm.openai_compat:mapper + compat_generate: wippy.llm.openai_compat:generate + compat_structured: wippy.llm.openai_compat:structured_output + google_mapper: wippy.llm.google:mapper + google_generate: wippy.llm.google:generate + google_structured: wippy.llm.google:structured_output + test: wippy.test:test + method: run_tests + # app:output_test - name: output_test kind: function.lua diff --git a/src/llm/test/bedrock/mapper_test.lua b/src/llm/test/bedrock/mapper_test.lua index 0cf48e3e..20e5d164 100644 --- a/src/llm/test/bedrock/mapper_test.lua +++ b/src/llm/test/bedrock/mapper_test.lua @@ -507,8 +507,7 @@ local function define_tests() }) test.not_nil(((result.messages[1].content :: any)[2] :: any).cachePoint) - test.not_nil(((result.messages[2].content :: any)[1] :: any).reasoningContent) - test.is_nil(result.messages[2].content[2]) + test.is_nil(result.messages[2]) end) it("should consolidate consecutive same-role messages", function() diff --git a/src/llm/test/claude/mapper_test.lua b/src/llm/test/claude/mapper_test.lua index 4adc3436..be1331eb 100644 --- a/src/llm/test/claude/mapper_test.lua +++ b/src/llm/test/claude/mapper_test.lua @@ -720,8 +720,7 @@ local function define_tests() }) test.not_nil((result.messages[1].content[1] :: any).cache_control) - test.eq(result.messages[2].content[1].type, "thinking") - test.is_nil(result.messages[2].content[1].cache_control) + test.is_nil(result.messages[2]) end) it("should keep the marked prefix stable when a dynamic developer note follows", function() diff --git a/src/llm/test/google/mapper_test.lua b/src/llm/test/google/mapper_test.lua index c80dd124..c4704ee0 100644 --- a/src/llm/test/google/mapper_test.lua +++ b/src/llm/test/google/mapper_test.lua @@ -499,9 +499,7 @@ local function define_tests() local google_messages, _ = mapper.map_messages(contract_messages) - tests.eq(#google_messages, 1) - local empty_str_parts = google_messages[1].parts :: any - tests.eq(empty_str_parts[1].text, "") + tests.eq(#google_messages, 0) end) it("should skip assistant messages with empty content", function() diff --git a/src/llm/test/llm_test.lua b/src/llm/test/llm_test.lua index 77bd05c5..0d37c877 100644 --- a/src/llm/test/llm_test.lua +++ b/src/llm/test/llm_test.lua @@ -232,6 +232,7 @@ local function define_tests() driver_declares_legacy_reasoning_flag = function(provider_id) return provider_id == "wippy.llm.openai:provider" end, + driver_output_policy = function() return nil, nil end, last_structured_output_args = nil, last_embed_args = nil, last_status_args = nil, diff --git a/src/llm/test/native_errors_test.lua b/src/llm/test/native_errors_test.lua index b03d55db..9c1518fd 100644 --- a/src/llm/test/native_errors_test.lua +++ b/src/llm/test/native_errors_test.lua @@ -32,7 +32,7 @@ return test.run_cases(function() local cause = errors.new({ kind = kind, retryable = kind == errors.INTERNAL, message = "provider refused", details = { operation = method } }) llm._providers = { - driver_declares_legacy_reasoning_flag = function() return false end, + driver_declares_legacy_reasoning_flag = function() return false end, driver_output_policy = function() return nil, nil end, open = function() return { [method] = function() return nil, cause end } end, } local result, err = call(method, { model = "test-model", provider_id = direct and "provider" or nil, fallback = false }) @@ -94,7 +94,7 @@ return test.run_cases(function() it("retains the exact text of untyped " .. method .. " provider failures", function() llm._providers = { - driver_declares_legacy_reasoning_flag = function() return false end, + driver_declares_legacy_reasoning_flag = function() return false end, driver_output_policy = function() return nil, nil end, open = function() return { [method] = function() return nil, "provider refused" end } end, } local result, err = call(method, { model = "test-model" }) @@ -159,7 +159,7 @@ return test.run_cases(function() {name = "large", max_tokens = 2000, providers = {{id = "provider", provider_model = "large"}}}, } end} llm._providers = { - driver_declares_legacy_reasoning_flag = function() return false end, + driver_declares_legacy_reasoning_flag = function() return false end, driver_output_policy = function() return nil, nil end, open = function() return {generate = function(_, args) if args.model == "small" then return nil, errors.new({kind = errors.INVALID, message = "context too large", @@ -303,7 +303,7 @@ return test.run_cases(function() } } end } llm._providers = { - driver_declares_legacy_reasoning_flag = function() return false end, + driver_declares_legacy_reasoning_flag = function() return false end, driver_output_policy = function() return nil, nil end, open = function(id) return { generate = function() return nil, errors.new({ kind = errors.RATE_LIMITED, retryable = true, diff --git a/src/llm/test/replay_safety_test.lua b/src/llm/test/replay_safety_test.lua new file mode 100644 index 00000000..acea50be --- /dev/null +++ b/src/llm/test/replay_safety_test.lua @@ -0,0 +1,950 @@ +local test = require("test") +local json = require("json") +local registry = require("registry") +local llm = require("llm") +local prompt = require("prompt") +local providers = require("providers") +local claude_mapper = require("claude_mapper") +local claude_client = require("claude_client") +local claude_generate = require("claude_generate") +local claude_structured = require("claude_structured") +local bedrock_mapper = require("bedrock_mapper") +local bedrock_client = require("bedrock_client") +local bedrock_generate = require("bedrock_generate") +local bedrock_structured = require("bedrock_structured") +local openai_mapper = require("openai_mapper") +local openai_generate = require("openai_generate") +local openai_structured = require("openai_structured") +local compat_mapper = require("compat_mapper") +local compat_generate = require("compat_generate") +local compat_structured = require("compat_structured") +local google_mapper = require("google_mapper") +local google_generate = require("google_generate") +local google_structured = require("google_structured") + +local BINDING_BETA = "thinking-binding-controls-2026-08-01" +local SCHEMA = { type = "object", properties = { a = { type = "number" } }, required = { "a" }, additionalProperties = false } + +local function text(value) return { { type = "text", text = value } } end +local function signed(name) return { type = "thinking", thinking = "", signature = name } end + +local CLAUDE_REPLY = { content = { { type = "text", text = "{\"a\":1}" }, + { type = "tool_use", name = "structured_output", id = "s", input = { a = 1 } } }, + stop_reason = "end_turn", usage = { input_tokens = 1, output_tokens = 1 } } +local BEDROCK_REPLY = { output = { message = { role = "assistant", content = { { text = "ok" }, + { toolUse = { toolUseId = "s", name = "structured_output", input = { a = 1 } } } } } }, + stopReason = "end_turn", usage = { inputTokens = 1, outputTokens = 1 } } + +local function capture_claude(handler): any + local sent: any = {} + handler._client = { ENDPOINTS = { MESSAGES = "/messages" }, request = function(_, payload, options) + table.insert(sent, { payload = payload, betas = options and options.beta_features }) + return json.decode((json.encode(CLAUDE_REPLY))) + end } + return sent +end + +local function capture_bedrock(handler): any + local sent: any = {} + handler._client = { converse = function(_, payload) + table.insert(sent, { payload = payload }) + return json.decode((json.encode(BEDROCK_REPLY))) + end } + return sent +end + +local function capture_google(handler): any + local sent: any = {} + local client_contract = { with_context = function(self) return self end, + open = function(self) return self end, + request = function(_, args) + table.insert(sent, { payload = args.payload or args }) + return { candidates = { { content = { parts = { { text = "{\"a\":1}" } } }, finishReason = "STOP" } }, + usageMetadata = { promptTokenCount = 1, candidatesTokenCount = 1 } } + end } + handler._contract = { get = function() return client_contract end } + handler._ctx = { all = function() return {} end, get = function() return "client" end } + return sent +end + +local function capture_openai(handler): any + local sent: any = {} + handler._client = { ENDPOINTS = { RESPONSES = "/responses", CHAT_COMPLETIONS = "/chat/completions" }, + request = function(_, payload) + table.insert(sent, { payload = payload }) + return { status = "completed", output = { { type = "message", content = { { type = "output_text", text = "{\"a\":1}" } } } }, + choices = { { message = { role = "assistant", content = "{\"a\":1}" }, finish_reason = "stop" } }, + usage = { input_tokens = 1, output_tokens = 1, prompt_tokens = 1, completion_tokens = 1 }, metadata = {} } + end } + return sent +end + +-- Wire order of an assistant turn as signatures, call ids and text. +local function claude_order(message) + local order = {} + for _, part in ipairs(message.content) do + table.insert(order, part.signature or part.id or part.text or part.type) + end + return order +end + +local function bedrock_order(message) + local order = {} + for _, part in ipairs(message.content) do + local reasoning = part.reasoningContent and part.reasoningContent.reasoningText + table.insert(order, reasoning and reasoning.signature or (part.toolUse and part.toolUse.toolUseId) + or part.text or (part.cachePoint and "cachePoint") or "other") + end + return order +end + +local function assert_order(actual, expected) + test.eq(table.concat(actual, "|"), table.concat(expected, "|")) +end + +local RESPONSE_ORDER = { "s1", "before ", "s2", "c1", "s3", "c2", "after" } + +local CLAUDE_RESPONSE = { + { type = "thinking", thinking = "", signature = "s1" }, + { type = "text", text = "before " }, + { type = "thinking", thinking = "", signature = "s2" }, + { type = "tool_use", id = "c1", name = "t", input = { a = 1 } }, + { type = "thinking", thinking = "", signature = "s3" }, + { type = "tool_use", id = "c2", name = "t", input = { a = 2 } }, + { type = "text", text = "after" }, +} + +local function reasoning_block(name) return { reasoningContent = { reasoningText = { text = "", signature = name } } } end + +local BEDROCK_RESPONSE = { + reasoning_block("s1"), + { text = "before " }, + reasoning_block("s2"), + { toolUse = { toolUseId = "c1", name = "t", input = { a = 1 } } }, + reasoning_block("s3"), + { toolUse = { toolUseId = "c2", name = "t", input = { a = 2 } } }, + { text = "after" }, +} + +local function sse_stream(events) + local body = "" + for _, event in ipairs(events) do + body = body .. "event: " .. event[1] .. "\ndata: " .. json.encode(event[2]) .. "\n\n" + end + local done = false + return { stream = { read = function() + if done then return nil end + done = true + return body + end }, metadata = {} } +end + +local function claude_stream_events() + local events = { { "message_start", { message = { usage = { input_tokens = 1 } } } } } + for index, block in ipairs(CLAUDE_RESPONSE) do + local at = index - 1 + if block.type == "thinking" then + table.insert(events, { "content_block_start", { index = at, content_block = { type = "thinking", thinking = "", signature = "" } } }) + table.insert(events, { "content_block_delta", { index = at, delta = { type = "signature_delta", signature = block.signature } } }) + elseif block.type == "text" then + table.insert(events, { "content_block_start", { index = at, content_block = { type = "text", text = "" } } }) + table.insert(events, { "content_block_delta", { index = at, delta = { type = "text_delta", text = block.text } } }) + else + table.insert(events, { "content_block_start", { index = at, content_block = { type = "tool_use", id = block.id, name = block.name } } }) + table.insert(events, { "content_block_delta", { index = at, delta = { type = "input_json_delta", partial_json = json.encode(block.input) } } }) + end + table.insert(events, { "content_block_stop", { index = at } }) + end + table.insert(events, { "message_delta", { delta = { stop_reason = "tool_use" }, usage = { output_tokens = 2 } } }) + table.insert(events, { "message_stop", {} }) + return events +end + +local function eventstream_message(event_type, body) + local function header(name, value) + return string.char(#name) .. name .. string.char(7) .. string.pack(">I2", #value) .. value + end + local headers = header(":event-type", event_type) .. header(":content-type", "application/json") + .. header(":message-type", "event") + local payload = json.encode(body) + local total = 12 + #headers + #payload + 4 + return string.pack(">I4I4I4", total, #headers, 0) .. headers .. payload .. string.pack(">I4", 0) +end + +local function bedrock_stream() + local chunks = { eventstream_message("messageStart", { role = "assistant" }) } + local function add(name, body) table.insert(chunks, eventstream_message(name, body)) end + for index, block in ipairs(BEDROCK_RESPONSE) do + local at = index - 1 + if block.reasoningContent then + add("contentBlockDelta", { contentBlockIndex = at, delta = { reasoningContent = { + signature = block.reasoningContent.reasoningText.signature } } }) + elseif block.text then + add("contentBlockDelta", { contentBlockIndex = at, delta = { text = block.text } }) + else + add("contentBlockStart", { contentBlockIndex = at, start = { toolUse = { toolUseId = block.toolUse.toolUseId, name = "t" } } }) + add("contentBlockDelta", { contentBlockIndex = at, delta = { toolUse = { input = json.encode(block.toolUse.input) } } }) + end + add("contentBlockStop", { contentBlockIndex = at }) + end + add("messageStop", { stopReason = "tool_use" }) + add("metadata", { usage = { inputTokens = 1, outputTokens = 2 } }) + local at = 0 + return { stream = { read = function() + at = at + 1 + return chunks[at] + end }, metadata = {} } +end + +-- The rows a host persists from one tool-using response. +local function stored_rows(result): any + local metadata = { + thinking_blocks = result.metadata.thinking_blocks, + content_positions = result.metadata.content_positions, + } + local rows = { + { role = "user", content = text("q") }, + { role = "assistant", content = text(result.result.content), metadata = metadata }, + } + for _, call in ipairs(result.result.tool_calls) do + table.insert(rows, { role = "function_call", content = {}, + function_call = { id = call.id, name = call.name, arguments = call.arguments } }) + end + for _, call in ipairs(result.result.tool_calls) do + table.insert(rows, { role = "function_result", name = call.name, function_call_id = call.id, content = text("r") }) + end + return rows +end + +local function claude_replay(rows, options) + local sent = capture_claude(claude_generate) + local _, err = claude_generate.handler({ model = "m", messages = rows, options = options or { max_tokens = 100 } }) + test.is_nil(err) + return sent[1].payload, sent[1].betas +end + +local function bedrock_replay(rows, options) + local sent = capture_bedrock(bedrock_generate) + local _, err = bedrock_generate.handler({ model = "m", messages = rows, options = options or { max_tokens = 100 } }) + test.is_nil(err) + return sent[1].payload +end + +local function claude_replay_order(result) + return claude_order(claude_replay(stored_rows(result)).messages[2]) +end + +local function bedrock_replay_order(result) + return bedrock_order(bedrock_replay(stored_rows(result)).messages[2]) +end + +local function has_block(list, predicate) + for _, message in ipairs(list) do + if type(message.content) == "table" then + for _, part in ipairs(message.content) do + if predicate(part) then return true end + end + end + end + return false +end + +local function stale_rows(): any + return { + { role = "user", content = text("q") }, + { role = "assistant", content = text("handing off"), metadata = { + thinking_blocks = { signed("s1") }, stale_thinking = true } }, + { role = "function_call", content = {}, function_call = { id = "c1", name = "handoff", arguments = {} } }, + { role = "function_result", name = "handoff", function_call_id = "c1", content = text("ok") }, + } +end + +local function is_claude_thinking(part) return part.type == "thinking" end +local function is_bedrock_reasoning(part) return part.reasoningContent ~= nil end + +local function policy_providers(handler, method, policy) + return { + driver_declares_legacy_reasoning_flag = function() return false end, + driver_output_policy = function() return policy, nil end, + open = function() + return { [method] = function(_, args) return handler.handler(args) end } + end, + } +end + +local function use_card(card) + llm._models = { get_by_name = function() return card, nil end, get_by_class = function() return {}, nil end } +end + +local BUDGET_POLICY = { thinking = "budget", default_max_tokens = 2000 } + +local function define_tests() + describe("Replay safety", function() + after_each(function() + llm._providers = nil + llm._models = nil + llm._usage_tracker = nil + end) + + describe("stale blocks without a bindable thinking object", function() + local UNBINDABLE = { + { name = "default budget at effort 0", accepts = nil, options = { max_tokens = 100 } }, + { name = "adaptive at effort 0", accepts = { thinking = "adaptive" }, options = { max_tokens = 100, thinking_effort = 0 } }, + { name = "none", accepts = { thinking = "none" }, options = { max_tokens = 100 } }, + } + for _, case in ipairs(UNBINDABLE) do + it("removes marked blocks on Claude: " .. case.name, function() + local sent = capture_claude(claude_generate) + assert(claude_generate.handler({ model = "m", messages = stale_rows(), options = case.options, accepts = case.accepts })) + test.is_false(has_block(sent[1].payload.messages, is_claude_thinking)) + test.is_nil(sent[1].betas) + assert_order(claude_order(sent[1].payload.messages[2]), { "handing off", "c1" }) + end) + it("removes marked blocks on Bedrock: " .. case.name, function() + local sent = capture_bedrock(bedrock_generate) + assert(bedrock_generate.handler({ model = "m", messages = stale_rows(), options = case.options, accepts = case.accepts })) + test.is_false(has_block(sent[1].payload.messages, is_bedrock_reasoning)) + assert_order(bedrock_order(sent[1].payload.messages[2]), { "handing off", "c1" }) + end) + end + + it("removes marked blocks on Claude between_tools", function() + local sent = capture_claude(claude_generate) + assert(claude_generate.handler({ model = "m", messages = stale_rows(), options = { + max_tokens = 100, thinking_effort = 50, request = { thinking = { type = "between_tools" } } }, + accepts = { thinking = "adaptive" } })) + test.eq(sent[1].payload.thinking.type, "between_tools") + test.is_false(has_block(sent[1].payload.messages, is_claude_thinking)) + end) + + it("removes marked blocks on Bedrock between_tools", function() + local sent = capture_bedrock(bedrock_generate) + assert(bedrock_generate.handler({ model = "m", messages = stale_rows(), options = { + max_tokens = 100, thinking_effort = 50, + request = { additionalModelRequestFields = { thinking = { type = "between_tools" } } } }, + accepts = { thinking = "adaptive" } })) + test.eq(sent[1].payload.additionalModelRequestFields.thinking.type, "between_tools") + test.is_false(has_block(sent[1].payload.messages, is_bedrock_reasoning)) + end) + + it("removes marked blocks in Claude and Bedrock structured output", function() + local sent = capture_claude(claude_structured) + assert(claude_structured.handler({ model = "m", messages = stale_rows(), schema = SCHEMA, options = { max_tokens = 100 } })) + test.is_false(has_block(sent[1].payload.messages, is_claude_thinking)) + sent = capture_claude(claude_structured) + assert(claude_structured.handler({ model = "m", messages = stale_rows(), schema = SCHEMA, + options = { max_tokens = 100 }, accepts = { structured_output = "native" } })) + test.is_false(has_block(sent[1].payload.messages, is_claude_thinking)) + sent = capture_bedrock(bedrock_structured) + assert(bedrock_structured.handler({ model = "m", messages = stale_rows(), schema = SCHEMA, options = { max_tokens = 100 } })) + test.is_false(has_block(sent[1].payload.messages, is_bedrock_reasoning)) + end) + + local BINDABLE = { + { name = "always at effort 0", accepts = { thinking = "always" }, options = { max_tokens = 100 } }, + { name = "adaptive above 0", accepts = { thinking = "adaptive" }, options = { max_tokens = 100, thinking_effort = 30 } }, + { name = "budget above 0", accepts = { thinking = "budget" }, options = { max_tokens = 100, thinking_effort = 30 } }, + } + for _, case in ipairs(BINDABLE) do + it("keeps marked blocks bound on Claude: " .. case.name, function() + local sent = capture_claude(claude_generate) + assert(claude_generate.handler({ model = "m", messages = stale_rows(), options = case.options, accepts = case.accepts })) + test.is_true(has_block(sent[1].payload.messages, is_claude_thinking)) + test.eq(sent[1].payload.thinking.block_binding.prefix_mismatch_behavior, "drop_block") + test.eq(sent[1].betas[1], BINDING_BETA) + end) + it("keeps marked blocks bound on Bedrock: " .. case.name, function() + local sent = capture_bedrock(bedrock_generate) + assert(bedrock_generate.handler({ model = "m", messages = stale_rows(), options = case.options, accepts = case.accepts })) + test.is_true(has_block(sent[1].payload.messages, is_bedrock_reasoning)) + test.eq(sent[1].payload.additionalModelRequestFields.thinking.block_binding.prefix_mismatch_behavior, "drop_block") + end) + end + + it("decides from the object after the overlay", function() + local sent = capture_claude(claude_generate) + assert(claude_generate.handler({ model = "m", messages = stale_rows(), accepts = { thinking = "adaptive" }, options = { + max_tokens = 100, thinking_effort = 30, request = { thinking = { type = "disabled" } } } })) + test.is_false(has_block(sent[1].payload.messages, is_claude_thinking)) + + sent = capture_claude(claude_generate) + assert(claude_generate.handler({ model = "m", messages = stale_rows(), accepts = { thinking = "none" }, options = { + max_tokens = 100, request = { thinking = { type = "adaptive" } } } })) + test.is_true(has_block(sent[1].payload.messages, is_claude_thinking)) + test.eq(sent[1].payload.thinking.block_binding.prefix_mismatch_behavior, "drop_block") + end) + + it("leaves unmarked blocks on a request without a thinking object", function() + local rows = stale_rows() + rows[2].metadata.stale_thinking = nil + local payload = claude_replay(rows) + test.is_true(has_block(payload.messages, is_claude_thinking)) + payload = bedrock_replay(stale_rows()) + test.is_false(has_block(payload.messages, is_bedrock_reasoning)) + end) + end) + + describe("interleaved response order", function() + it("replays Claude JSON exactly", function() + local result = claude_mapper.format_success_response({ content = CLAUDE_RESPONSE, stop_reason = "tool_use", + usage = { input_tokens = 1, output_tokens = 1 } }, "m", {}) + assert_order(claude_replay_order({ result = result.result, metadata = result.metadata }), RESPONSE_ORDER) + end) + + it("replays Claude stream exactly", function() + local _, stream_err, stream_result = claude_client.process_stream(sse_stream(claude_stream_events()), {}) + test.is_nil(stream_err) + local result = claude_mapper.format_streaming_response(stream_result, {}, stream_result.usage, stream_result.finish_reason, stream_result.metadata) + assert_order(claude_replay_order(result), RESPONSE_ORDER) + end) + + it("replays Bedrock JSON exactly", function() + local result = bedrock_mapper.format_success_response({ output = { message = { role = "assistant", content = BEDROCK_RESPONSE } }, + stopReason = "tool_use", usage = { inputTokens = 1, outputTokens = 1 } }, {}) + assert_order(bedrock_replay_order(result), RESPONSE_ORDER) + end) + + it("replays Bedrock stream exactly", function() + local _, stream_err, stream_result = bedrock_client.process_converse_stream(bedrock_stream(), {}) + test.is_nil(stream_err) + local result = bedrock_mapper.format_streaming_response(stream_result, {}, stream_result.usage, stream_result.finish_reason, stream_result.metadata) + assert_order(bedrock_replay_order(result), RESPONSE_ORDER) + end) + + it("sends no position field on the wire", function() + local result = claude_mapper.format_success_response({ content = CLAUDE_RESPONSE, stop_reason = "tool_use", + usage = { input_tokens = 1, output_tokens = 1 } }, "m", {}) + local payload = claude_replay(stored_rows(result)) + test.is_false(json.encode(payload):find("position", 1, true) ~= nil) + end) + + it("records nothing for a response the plain order reproduces", function() + local result = claude_mapper.format_success_response({ content = { + { type = "thinking", thinking = "", signature = "s1" }, + { type = "text", text = "only" }, + { type = "tool_use", id = "c1", name = "t", input = { a = 1 } }, + }, stop_reason = "tool_use", usage = { input_tokens = 1, output_tokens = 1 } }, "m", {}) + test.is_nil(result.metadata.content_positions) + test.is_nil(result.metadata.thinking_blocks[1].position) + end) + + it("keeps today's order for stored rows without positions", function() + local rows = { + { role = "user", content = text("q") }, + { role = "assistant", content = text("before after"), metadata = { + thinking_blocks = { signed("s1"), signed("s2"), signed("s3") } } }, + { role = "function_call", content = {}, function_call = { id = "c1", name = "t", arguments = {} } }, + { role = "function_call", content = {}, function_call = { id = "c2", name = "t", arguments = {} } }, + { role = "function_result", name = "t", function_call_id = "c1", content = text("r") }, + { role = "function_result", name = "t", function_call_id = "c2", content = text("r") }, + } + assert_order(claude_order(claude_replay(rows).messages[2]), { "s1", "s2", "s3", "before after", "c1", "c2" }) + assert_order(bedrock_order(bedrock_replay(rows).messages[2]), { "s1", "s2", "s3", "before after", "c1", "c2" }) + end) + + it("keeps the text and call order once the blocks are stripped", function() + local result = claude_mapper.format_success_response({ content = CLAUDE_RESPONSE, stop_reason = "tool_use", + usage = { input_tokens = 1, output_tokens = 1 } }, "m", {}) + local rows = stored_rows({ result = result.result, metadata = result.metadata }) + rows[2].metadata.thinking_blocks = nil + assert_order(claude_order(claude_replay(rows).messages[2]), { "before ", "c1", "c2", "after" }) + end) + + it("keeps the row's text when the host changed it", function() + local result = claude_mapper.format_success_response({ content = CLAUDE_RESPONSE, stop_reason = "tool_use", + usage = { input_tokens = 1, output_tokens = 1 } }, "m", {}) + local rows = stored_rows({ result = result.result, metadata = result.metadata }) + rows[2].content = text("edited") + assert_order(claude_order(claude_replay(rows).messages[2]), { "s1", "s2", "s3", "edited", "c1", "c2" }) + end) + + it("keeps two responses of one consolidated turn apart", function() + local first = claude_mapper.format_success_response({ content = { + { type = "text", text = "a" }, { type = "thinking", thinking = "", signature = "s1" }, { type = "text", text = "b" }, + }, stop_reason = "end_turn", usage = { input_tokens = 1, output_tokens = 1 } }, "m", {}) + local second = claude_mapper.format_success_response({ content = { + { type = "text", text = "c" }, { type = "thinking", thinking = "", signature = "s2" }, { type = "text", text = "d" }, + }, stop_reason = "end_turn", usage = { input_tokens = 1, output_tokens = 1 } }, "m", {}) + local function row(result) + return { role = "assistant", content = text(result.result.content), metadata = { + thinking_blocks = result.metadata.thinking_blocks, content_positions = result.metadata.content_positions } } + end + local rows = { { role = "user", content = text("q") }, row(first), row(second), { role = "user", content = text("next") } } + assert_order(claude_order(claude_replay(rows).messages[2]), { "a", "s1", "b", "c", "s2", "d" }) + end) + + it("keeps a Bedrock cache point behind the block it follows", function() + local result = bedrock_mapper.format_success_response({ output = { message = { role = "assistant", content = BEDROCK_RESPONSE } }, + stopReason = "tool_use", usage = { inputTokens = 1, outputTokens = 1 } }, {}) + local rows = stored_rows(result) + table.insert(rows, 3, { role = "cache_marker", marker_id = "m" }) + assert_order(bedrock_order(bedrock_replay(rows).messages[2]), + { "s1", "before ", "s2", "c1", "s3", "c2", "after", "cachePoint" }) + end) + + it("takes no text layout from a recovered text tool call", function() + local extracted = bedrock_mapper.extract_response_content({ output = { message = { content = { + reasoning_block("s1"), { text = "a" }, { text = '{"name":"t","arguments":{}}' }, + } } } }, { t = "app:t" }) + test.eq(#extracted.tool_calls, 1) + test.is_nil(extracted.content_positions) + end) + end) + + describe("OpenAI-compatible tool rounds", function() + local function note_rows() + return { + { role = "user", content = text("q") }, + { role = "assistant", content = text("look") }, + { role = "function_call", content = {}, function_call = { id = "c1", name = "t", arguments = "{}" } }, + { role = "function_result", name = "t", function_call_id = "c1", content = text("r1") }, + { role = "system", content = text("NOTE"), metadata = { preserve_position = true } }, + { role = "function_call", content = {}, function_call = { id = "c2", name = "t", arguments = "{}" } }, + { role = "function_result", name = "t", function_call_id = "c2", content = text("r2") }, + } + end + + it("keeps a positional note between tool rounds in place", function() + local mapped: any = compat_mapper.map_messages(note_rows(), { model = "gpt" }) + local roles = {} + for _, message in ipairs(mapped) do table.insert(roles, message.role) end + assert_order(roles, { "user", "assistant", "tool", "system", "assistant", "tool" }) + test.eq(#mapped[2].tool_calls, 1) + test.eq(mapped[2].tool_calls[1].id, "c1") + test.eq(mapped[4].content[1].text, "NOTE") + test.eq(mapped[5].tool_calls[1].id, "c2") + test.eq(mapped[6].tool_call_id, "c2") + end) + + it("keeps parallel calls of one round on one assistant message", function() + local mapped: any = compat_mapper.map_messages({ + { role = "assistant", content = text("look") }, + { role = "function_call", content = {}, function_call = { id = "c1", name = "t", arguments = "{}" } }, + { role = "function_result", name = "t", function_call_id = "c1", content = text("r1") }, + { role = "function_call", content = {}, function_call = { id = "c2", name = "t", arguments = "{}" } }, + { role = "function_result", name = "t", function_call_id = "c2", content = text("r2") }, + }, { model = "gpt" }) + test.eq(#mapped[1].tool_calls, 2) + end) + + it("reads past a cache marker between rounds", function() + local mapped: any = compat_mapper.map_messages({ + { role = "assistant", content = text("look") }, + { role = "function_call", content = {}, function_call = { id = "c1", name = "t", arguments = "{}" } }, + { role = "function_result", name = "t", function_call_id = "c1", content = text("r1") }, + { role = "cache_marker", marker_id = "m" }, + { role = "function_call", content = {}, function_call = { id = "c2", name = "t", arguments = "{}" } }, + { role = "function_result", name = "t", function_call_id = "c2", content = text("r2") }, + }, { model = "gpt" }) + test.eq(#mapped[1].tool_calls, 2) + end) + end) + + describe("blank assistant turns", function() + local function blank_rows(row) + return { { role = "user", content = text("Q") }, row, { role = "user", content = text("Q2") } } + end + local BLANKS = { + { name = "empty text", row = { role = "assistant", content = text("") } }, + { name = "whitespace text", row = { role = "assistant", content = text(" \n\t") } }, + { name = "no content", row = { role = "assistant", content = {} } }, + { name = "string blank", row = { role = "assistant", content = " " } }, + { name = "thinking only", row = { role = "assistant", content = {}, metadata = { thinking_blocks = { signed("s1") } } } }, + { name = "thinking and whitespace", row = { role = "assistant", content = text(" "), metadata = { thinking_blocks = { signed("s1") } } } }, + } + for _, case in ipairs(BLANKS) do + it("renders nothing on Claude: " .. case.name, function() + local mapped = claude_mapper.map_messages(blank_rows(case.row)) + test.eq(#mapped.messages, 2) + test.eq(mapped.messages[1].role, "user") + test.eq(mapped.messages[2].role, "user") + test.is_false(json.encode(mapped.messages):find("intercepted", 1, true) ~= nil) + end) + it("renders nothing on Bedrock: " .. case.name, function() + local mapped = bedrock_mapper.map_messages(blank_rows(case.row)) + test.eq(#mapped.messages, 1) + test.eq(mapped.messages[1].role, "user") + test.eq(#mapped.messages[1].content, 2) + end) + it("renders nothing on OpenAI: " .. case.name, function() + local items = openai_mapper.map_messages(blank_rows(case.row), {}) + test.eq(#items, 2) + test.eq(items[1].role, "user") + test.eq(items[2].role, "user") + end) + it("renders nothing on OpenAI-compatible: " .. case.name, function() + local mapped = compat_mapper.map_messages(blank_rows(case.row), {}) + test.eq(#mapped, 2) + test.eq(mapped[2].role, "user") + end) + it("renders nothing on Google: " .. case.name, function() + local mapped = google_mapper.map_messages(blank_rows(case.row), {}) + test.eq(#mapped, 2) + test.eq(mapped[2].role, "user") + end) + end + + it("drops only the blank parts of an OpenAI assistant message", function() + local items = openai_mapper.map_messages({ + { role = "user", content = text("Q") }, + { role = "assistant", content = { { type = "text", text = " " }, { type = "text", text = "kept" }, { type = "text", text = "" } } }, + }, {}) + test.eq(#items[2].content, 1) + test.eq(items[2].content[1].text, "kept") + end) + + it("keeps a turn that has a call on every driver", function() + local rows = { + { role = "user", content = text("Q") }, + { role = "assistant", content = text(""), metadata = { thinking_blocks = { signed("s1") } } }, + { role = "function_call", content = {}, function_call = { id = "c1", name = "t", arguments = "{\"a\":1}" } }, + { role = "function_result", name = "t", function_call_id = "c1", content = text("r") }, + } + assert_order(claude_order(claude_mapper.map_messages(rows).messages[2]), { "s1", "c1" }) + assert_order(bedrock_order(bedrock_mapper.map_messages(rows).messages[2]), { "s1", "c1" }) + test.eq(openai_mapper.map_messages(rows, {})[2].type, "function_call") + test.eq((compat_mapper.map_messages(rows, {}) :: any)[2].tool_calls[1].id, "c1") + test.not_nil(google_mapper.map_messages(rows, {})[2].parts[1].functionCall) + end) + + it("keeps whitespace text inside a Claude turn with a call for an exact replay", function() + local rows = { + { role = "user", content = text("Q") }, + { role = "assistant", content = text("\n\n") }, + { role = "function_call", content = {}, function_call = { id = "c1", name = "t", arguments = "{}" } }, + { role = "function_result", name = "t", function_call_id = "c1", content = text("r") }, + } + assert_order(claude_order(claude_mapper.map_messages(rows).messages[2]), { "\n\n", "c1" }) + end) + + it("moves a cache marker off a blank turn", function() + local mapped = claude_mapper.map_messages({ + { role = "user", content = text("Q") }, + { role = "assistant", content = text(" ") }, + { role = "cache_marker", marker_id = "tail" }, + }) + test.eq(#mapped.messages, 1) + test.not_nil(mapped.messages[1].content[1].cache_control) + end) + end) + + describe("effective output", function() + local function setup(handler, method, card, policy) + llm._usage_tracker = false + use_card(card) + llm._providers = policy_providers(handler, method, policy or BUDGET_POLICY) + end + + local function card(fields) + local out = { name = "m", id = "m", capabilities = {}, class = {}, + providers = { { id = "route", provider_model = "wire" } } } + for key, value in pairs(fields) do out[key] = value end + return out + end + + it("clamps the Claude budget raise to the card limit and reports the wire value", function() + local sent = capture_claude(claude_generate) + setup(claude_generate, "generate", card({ output_tokens = 8192 })) + local result: any = assert(llm.generate("hi", { model = "m", max_tokens = 512, thinking_effort = 100 })) + test.eq(sent[1].payload.max_tokens, 8192) + test.eq(sent[1].payload.thinking.budget_tokens, 7168) + test.eq(result.metadata.adjusted.max_tokens.requested, 512) + test.eq(result.metadata.adjusted.max_tokens.sent, 8192) + test.eq(result.metadata.adjusted.thinking_budget.requested, 24000) + test.eq(result.metadata.adjusted.thinking_budget.sent, 7168) + end) + + it("clamps the Bedrock budget raise to the card limit and reports the wire value", function() + local sent = capture_bedrock(bedrock_generate) + setup(bedrock_generate, "generate", card({ output_tokens = 8192 })) + local result: any = assert(llm.generate("hi", { model = "m", max_tokens = 512, thinking_effort = 100 })) + test.eq(sent[1].payload.inferenceConfig.maxTokens, 8192) + test.eq(sent[1].payload.additionalModelRequestFields.thinking.budget_tokens, 7168) + test.eq(result.metadata.adjusted.max_tokens.sent, 8192) + end) + + it("reports a raise with no limit in the way", function() + local sent = capture_claude(claude_generate) + setup(claude_generate, "generate", card({})) + local result: any = assert(llm.generate("hi", { model = "m", max_tokens = 512, thinking_effort = 100 })) + test.eq(sent[1].payload.max_tokens, 25024) + test.eq(sent[1].payload.thinking.budget_tokens, 24000) + test.eq(result.metadata.adjusted.max_tokens.requested, 512) + test.eq(result.metadata.adjusted.max_tokens.sent, 25024) + test.is_nil(result.metadata.adjusted.thinking_budget) + end) + + it("leaves a request above the budget and under the limit alone", function() + local sent = capture_claude(claude_generate) + setup(claude_generate, "generate", card({ output_tokens = 64000 })) + local result: any = assert(llm.generate("hi", { model = "m", max_tokens = 30000, thinking_effort = 100 })) + test.eq(sent[1].payload.max_tokens, 30000) + test.eq(sent[1].payload.thinking.budget_tokens, 24000) + test.is_nil(result.metadata.adjusted) + end) + + it("rejects the raise under strict", function() + capture_claude(claude_generate) + setup(claude_generate, "generate", card({})) + local result, err = llm.generate("hi", { model = "m", max_tokens = 512, thinking_effort = 100, strict = true }) + test.is_nil(result) + test.contains(tostring(err), "max_tokens") + end) + + it("gives up thinking when the limit leaves no room for the minimum budget", function() + local sent = capture_claude(claude_generate) + setup(claude_generate, "generate", card({ output_tokens = 1000 })) + local result: any = assert(llm.generate("hi", { model = "m", max_tokens = 512, thinking_effort = 100 })) + test.is_nil(sent[1].payload.thinking) + test.eq(sent[1].payload.max_tokens, 512) + test.eq(result.metadata.adjusted.thinking_effort.requested, 100) + end) + + it("admits on the raised output", function() + local sent = capture_claude(claude_generate) + setup(claude_generate, "generate", card({ max_tokens = 30000 })) + local result, err = llm.generate(string.rep("x", 80000), { model = "m", max_tokens = 512, thinking_effort = 100 }) + test.is_nil(result) + test.contains(tostring(err), "context window") + test.eq(#sent, 0) + end) + + it("reserves the driver default when max_tokens is unset", function() + local sent = capture_claude(claude_generate) + setup(claude_generate, "generate", card({ max_tokens = 1000 })) + local result, err = llm.generate(string.rep("x", 800), { model = "m" }) + test.is_nil(result) + test.contains(tostring(err), "context window") + test.eq(#sent, 0) + end) + + it("keeps the wire default for an unset max_tokens under the limit without adjustment", function() + local sent = capture_bedrock(bedrock_generate) + setup(bedrock_generate, "generate", card({ output_tokens = 64000 })) + local result: any = assert(llm.generate("hi", { model = "m" })) + test.eq(sent[1].payload.inferenceConfig.maxTokens, 2000) + test.is_nil(result.metadata.adjusted) + end) + + it("clamps the driver default to the card limit", function() + local sent = capture_claude(claude_generate) + setup(claude_generate, "generate", card({ output_tokens = 1000 })) + local result: any = assert(llm.generate("hi", { model = "m" })) + test.eq(sent[1].payload.max_tokens, 1000) + test.is_nil(result.metadata.adjusted) + end) + + it("applies to structured output", function() + local sent = capture_claude(claude_structured) + setup(claude_structured, "structured_output", card({ output_tokens = 8192 })) + local result: any = assert(llm.structured_output(SCHEMA, "hi", { model = "m", max_tokens = 512, thinking_effort = 100 })) + test.eq(sent[1].payload.max_tokens, 8192) + test.eq(result.metadata.adjusted.max_tokens.sent, 8192) + end) + + it("leaves a driver without a policy to its request", function() + local sent = capture_openai(openai_generate) + setup(openai_generate, "generate", card({ max_tokens = 1000 }), false) + llm._providers.driver_output_policy = function() return nil, nil end + assert(llm.generate(string.rep("x", 800), { model = "m" })) + test.is_nil(sent[1].payload.max_output_tokens) + end) + + it("declares the policy on the real driver bindings", function() + for _, driver in ipairs({ + { id = "wippy.llm.claude:driver", mapper = claude_mapper }, + { id = "wippy.llm.bedrock:driver", mapper = bedrock_mapper }, + }) do + local entry = assert(registry.get(driver.id)) + test.eq(entry.meta.output_policy.thinking, driver.mapper.CAPABILITY.defaults.thinking) + test.eq(entry.meta.output_policy.default_max_tokens, driver.mapper.DEFAULT_MAX_TOKENS) + end + test.is_nil(assert(registry.get("wippy.llm.openai:driver")).meta.output_policy) + end) + + it("reads the policy through the provider discovery", function() + local entry = assert(registry.get("wippy.llm.bedrock:provider")) + local policy, err = providers.driver_output_policy(entry.id) + test.is_nil(err) + test.eq(policy.thinking, "budget") + test.eq(policy.default_max_tokens, 2000) + end) + end) + + describe("blank user, system and developer text", function() + local function rows() + return { + { role = "system", content = text("") }, + { role = "system", content = " \n" }, + { role = "system", content = text("S") }, + { role = "user", content = { { type = "text", text = " " }, { type = "text", text = "Q" } } }, + { role = "user", content = text("\t") }, + { role = "developer", content = text(" ") }, + { role = "developer", content = text(""), metadata = { preserve_position = true } }, + { role = "system", content = text(" "), metadata = { preserve_position = true } }, + } + end + + it("never reaches Claude", function() + local mapped = claude_mapper.map_messages(rows()) + test.eq(#mapped.system, 1) + test.eq(mapped.system[1].text, "S") + test.eq(#mapped.messages, 1) + test.eq(#mapped.messages[1].content, 1) + test.eq(mapped.messages[1].content[1].text, "Q") + end) + + it("never reaches Bedrock", function() + local mapped = bedrock_mapper.map_messages(rows()) + test.eq(#mapped.system, 1) + test.eq(#mapped.messages, 1) + test.eq(#mapped.messages[1].content, 1) + end) + + it("never reaches OpenAI", function() + local input = rows() + test.eq(openai_mapper.extract_instructions(input), "S") + local items = openai_mapper.map_messages(input, {}) + test.eq(#items, 1) + test.eq(#items[1].content, 1) + end) + + it("never reaches OpenAI-compatible", function() + local mapped = compat_mapper.map_messages(rows(), {}) + test.eq(#mapped, 2) + test.eq(mapped[1].role, "system") + test.eq(mapped[2].role, "user") + test.eq(#mapped[2].content, 1) + end) + + it("never reaches OpenAI-compatible structured output", function() + local sent = capture_openai(compat_structured) + compat_structured.handler({ model = "m", messages = rows(), schema = SCHEMA, options = {} }) + test.eq(#sent[1].payload.messages, 2) + end) + + it("never reaches Google", function() + local mapped, system = google_mapper.map_messages(rows(), {}) + test.eq(#system, 1) + test.eq(#mapped, 1) + test.eq(#mapped[1].parts, 1) + end) + + it("sends no blank part in place of a part OpenAI cannot render", function() + local document = { type = "document", source = { type = "base64", mime_type = "application/pdf", data = "BBBB" } } + local items = openai_mapper.map_messages({ + { role = "user", content = { { type = "text", text = "See" }, document } }, + { role = "user", content = { document } }, + }, {}) + test.eq(#items, 1) + test.eq(#items[1].content, 1) + end) + + it("keeps an image beside blank text", function() + local image = { type = "image", source = { type = "base64", mime_type = "image/png", data = "AAAA" } } + local mapped = claude_mapper.map_messages({ { role = "user", content = { { type = "text", text = " " }, image } } }) + test.eq(#mapped.messages[1].content, 1) + test.eq(mapped.messages[1].content[1].type, "image") + end) + + it("keeps an empty tool result", function() + local mapped = claude_mapper.map_messages({ + { role = "function_call", content = {}, function_call = { id = "c1", name = "t", arguments = "{}" } }, + { role = "function_result", name = "t", function_call_id = "c1", content = text("") }, + }) + test.eq(mapped.messages[2].content[1].type, "tool_result") + end) + + it("leaves the caller's rows untouched", function() + local input = rows() + claude_mapper.map_messages(input) + test.eq(#input, 8) + test.eq(#input[4].content, 2) + end) + end) + + describe("requests with no turn", function() + local SYSTEM_ONLY = { { role = "system", content = text("S") } } + local BLANK_USER = { { role = "user", content = text(" ") } } + local CASES = { + { name = "Claude generate", handler = claude_generate, capture = capture_claude }, + { name = "Claude structured", handler = claude_structured, capture = capture_claude, schema = true }, + { name = "Claude native structured", handler = claude_structured, capture = capture_claude, schema = true, accepts = { structured_output = "native" } }, + { name = "Bedrock generate", handler = bedrock_generate, capture = capture_bedrock }, + { name = "Bedrock structured", handler = bedrock_structured, capture = capture_bedrock, schema = true }, + { name = "Google generate", handler = google_generate, capture = capture_google }, + { name = "Google structured", handler = google_structured, capture = capture_google, schema = true }, + { name = "OpenAI generate", handler = openai_generate, capture = capture_openai }, + { name = "OpenAI structured", handler = openai_structured, capture = capture_openai, schema = true }, + } + for _, case in ipairs(CASES) do + for label, messages in pairs({ ["system only"] = SYSTEM_ONLY, ["blank user"] = BLANK_USER }) do + it("rejects before HTTP on " .. case.name .. ": " .. label, function() + local sent = case.capture(case.handler) + local response, err = case.handler.handler({ model = "m", messages = messages, options = {}, + accepts = case.accepts, schema = case.schema and SCHEMA or nil }) + test.is_nil(response) + test.eq(#sent, 0) + test.eq(err:kind(), "Invalid") + test.contains(err:message(), "no conversation turn") + end) + end + end + + it("still sends a system-role note alone on OpenAI", function() + local sent = capture_openai(openai_generate) + assert(openai_generate.handler({ model = "m", options = {}, messages = { + { role = "system", content = text("N"), metadata = { preserve_position = true } } } })) + test.eq(#sent, 1) + test.eq(#sent[1].payload.input, 1) + end) + + it("still sends a system message alone on OpenAI-compatible", function() + local sent = capture_openai(compat_generate) + assert(compat_generate.handler({ model = "m", options = {}, messages = SYSTEM_ONLY })) + test.eq(#sent, 1) + test.eq(#sent[1].payload.messages, 1) + end) + end) + + describe("drop_stale_thinking on string content", function() + it("handles an assistant message with string content", function() + local builder: any = prompt.new({ + { role = "user", content = "q" }, + { role = "assistant", content = "hi", metadata = { thinking_blocks = { signed("A") } } }, + { role = "user", content = "q2" }, + } :: any) + local ok, result, err = pcall(function() return builder:drop_stale_thinking(function() return false end) end) + test.is_true(ok) + test.is_nil(err) + test.is_nil(result.messages[2].metadata.thinking_blocks) + end) + end) + + describe("mark_stale_thinking", function() + it("marks every block carrier up to the last stale one on copies", function() + local first = { role = "assistant", content = text("a"), metadata = { thinking_blocks = { signed("A") }, epoch = "old" } } + local second = { role = "assistant", content = text("b"), metadata = { thinking_blocks = { signed("B") }, epoch = "new" } } + local third = { role = "assistant", content = text("c"), metadata = { thinking_blocks = { signed("C") }, epoch = "new" } } + local builder: any = prompt.new({ { role = "user", content = text("q") }, second, first, third } :: any) + builder:mark_stale_thinking(function(msg) return msg.metadata.epoch == "new" end) + test.is_true(builder.messages[2].metadata.stale_thinking) + test.is_true(builder.messages[3].metadata.stale_thinking) + test.is_nil(builder.messages[4].metadata.stale_thinking) + test.is_nil(first.metadata.stale_thinking) + test.is_nil(second.metadata.stale_thinking) + end) + + it("handles rows without blocks and string content", function() + local builder: any = prompt.new({ + { role = "user", content = "q" }, + { role = "assistant", content = "hi" }, + } :: any) + builder:mark_stale_thinking(function() return false end) + test.is_nil(builder.messages[2].metadata) + end) + end) + end) +end + +return test.run_cases(define_tests) diff --git a/src/llm/test/route_flow_test.lua b/src/llm/test/route_flow_test.lua index d2b05d55..1161544b 100644 --- a/src/llm/test/route_flow_test.lua +++ b/src/llm/test/route_flow_test.lua @@ -78,7 +78,7 @@ local function define_tests() } end handler._client = { ENDPOINTS = { MESSAGES = "/messages" }, request = request, converse = request } - llm._providers = { driver_declares_legacy_reasoning_flag = function() return false end, open = function() + llm._providers = { driver_declares_legacy_reasoning_flag = function() return false end, driver_output_policy = function() return nil, nil end, open = function() return { [method] = function(_, args) return handler.handler(args) end } end } return sent @@ -194,7 +194,7 @@ local function define_tests() } end handler._client = { ENDPOINTS = { MESSAGES = "/messages" }, request = request, converse = request } - llm._providers = { driver_declares_legacy_reasoning_flag = function() return false end, open = function() + llm._providers = { driver_declares_legacy_reasoning_flag = function() return false end, driver_output_policy = function() return nil, nil end, open = function() return { generate = function(_, args) return handler.handler(args) end } end } return sent @@ -363,7 +363,7 @@ local function define_tests() for key, value in pairs(driver.accepts) do ref[key] = value end local card = { name = "route-model", providers = { ref } } llm._model_resolver = { resolve = function() return card end } - llm._providers = { driver_declares_legacy_reasoning_flag = function() return false end, open = function() + llm._providers = { driver_declares_legacy_reasoning_flag = function() return false end, driver_output_policy = function() return nil, nil end, open = function() return { [method] = function(_, args) return handler.handler(args) end } end } return sent @@ -460,7 +460,7 @@ local function define_tests() options = { thinking_effort = effort, request = request } } } } llm._model_resolver = { resolve = function() return card end } - llm._providers = { driver_declares_legacy_reasoning_flag = function() return false end, open = function() + llm._providers = { driver_declares_legacy_reasoning_flag = function() return false end, driver_output_policy = function() return nil, nil end, open = function() return { [method] = function(_, args) return handler.handler(args) end } end } local _, err @@ -530,7 +530,7 @@ local function define_tests() for key, value in pairs(driver.accepts) do ref[key] = value end local card = { name = "route-model", providers = { ref } } llm._model_resolver = { resolve = function() return card end } - llm._providers = { driver_declares_legacy_reasoning_flag = function() return false end, open = function() + llm._providers = { driver_declares_legacy_reasoning_flag = function() return false end, driver_output_policy = function() return nil, nil end, open = function() return { structured_output = function(_, args) return handler.handler(args) end } end } local result, err = call("structured_output", { model = "route-model" }) @@ -550,7 +550,7 @@ local function define_tests() local card = { name = "route-model", providers = { { id = "route", provider_model = "wire", thinking = "adaptive", options = { request = { usage = { include = false } } } } } } llm._model_resolver = { resolve = function() return card end } - llm._providers = { driver_declares_legacy_reasoning_flag = function() return false end, open = function() + llm._providers = { driver_declares_legacy_reasoning_flag = function() return false end, driver_output_policy = function() return nil, nil end, open = function() return { generate = function(_, args) return openai_compat_generate.handler(args) end } end } local result, err = call("generate", { model = "route-model" }) @@ -562,7 +562,7 @@ local function define_tests() for _, method in ipairs({ "generate", "structured_output" }) do it("returns an invalid request from Google " .. method .. " for declared thinking", function() local handler = method == "generate" and google_generate or google_structured - llm._providers = { driver_declares_legacy_reasoning_flag = function() return false end, open = function() + llm._providers = { driver_declares_legacy_reasoning_flag = function() return false end, driver_output_policy = function() return nil, nil end, open = function() return { [method] = function(_, args) return handler.handler(args) end } end } local result, err = call(method, { provider_id = "route", model = "wire", accepts = { thinking = "adaptive" } }) From 409ba2ceabecd5563a4a2c3edba326f37029305e Mon Sep 17 00:00:00 2001 From: KickSide Date: Wed, 4 Nov 2026 09:11:19 -0500 Subject: [PATCH 26/37] fix(llm): keep blank text out of recorded layouts A recorded layout renders only visible text, and a row whose text differs from the layout only in outer whitespace keeps its layout. The API compares neither whitespace-only text blocks nor a block's outer whitespace, so neither is an edit to the prefix. The README now limits the order guarantee to calls of one response that consolidate into one assistant turn. --- src/llm/src/README.md | 4 +- src/llm/src/util/_index.yaml | 2 + src/llm/src/util/thinking.lua | 17 ++++- src/llm/test/replay_safety_test.lua | 108 ++++++++++++++++++++++++++-- 4 files changed, 120 insertions(+), 11 deletions(-) diff --git a/src/llm/src/README.md b/src/llm/src/README.md index 8cb27740..803132e7 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -133,7 +133,7 @@ local response, err = llm.generate("Hello", { ### Thinking block binding -Claude and Bedrock store each response's `thinking` blocks as `{ type, thinking, signature }` and `redacted_thinking` blocks as `{ type, data }` in `metadata.thinking_blocks`, in response order (Bedrock: replayed as `reasoningContent.reasoningText` and `reasoningContent.redactedContent`). A response that is thinking, then at most one text block, then tool calls replays in that order with no record. Any other response, such as thinking between text and tool calls, also carries `metadata.content_positions` and a `position` on each thinking block; a host that stores `content_positions` next to `thinking_blocks` on the assistant row (or the first function call row of the response when it has no text) gets the exact response order back, with each text block separate. Rows without `content_positions`, including every row stored before it existed, replay as all the thinking blocks of the row, then its text as one block, then its calls; so do rows whose text no longer equals the recorded text. Positions never reach the wire, and other drivers ignore them. Claude replays each block unchanged except for fields an earlier stored shape carried that its type does not accept (`data` on `thinking`, `thinking` and `signature` on `redacted_thinking`). A Claude thinking block is bound to the conversation that produced it: the top-level system prompt, the tools, and every earlier message. After an agent handoff, a prompt or tool change, or a history edit, models that enforce the check reject a request that replays such a block with a 400. +Claude and Bedrock store each response's `thinking` blocks as `{ type, thinking, signature }` and `redacted_thinking` blocks as `{ type, data }` in `metadata.thinking_blocks`, in response order (Bedrock: replayed as `reasoningContent.reasoningText` and `reasoningContent.redactedContent`). A response that is thinking, then at most one text block, then tool calls replays in that order with no record. Any other response, such as thinking between text and tool calls, also carries `metadata.content_positions` and a `position` on each thinking block; a host that stores `content_positions` next to `thinking_blocks` on the assistant row (or the first function call row of the response when it has no text) gets the response order back, with each visible text block separate, as long as the calls of that response are consecutive function call rows, which consolidate into one assistant turn. A function result between those rows splits the turn, and the order is then not rebuilt across the split. Whitespace-only text blocks of a recorded response are not rendered: the API compares neither those nor the outer whitespace of a text block, so leaving them out changes no prefix. Rows without `content_positions`, including every row stored before it existed, replay as all the thinking blocks of the row, then its text as one block, then its calls; so do rows whose text, apart from outer whitespace, no longer equals the recorded text. Positions never reach the wire, and other drivers ignore them. Claude replays each block unchanged except for fields an earlier stored shape carried that its type does not accept (`data` on `thinking`, `thinking` and `signature` on `redacted_thinking`). A Claude thinking block is bound to the conversation that produced it: the top-level system prompt, the tools, and every earlier message. After an agent handoff, a prompt or tool change, or a history edit, models that enforce the check reject a request that replays such a block with a 400. When a request replays at least one thinking block and sends a thinking object of type `adaptive` or `enabled` (a positive thinking_effort, or `thinking = always` at any effort), the driver sets `thinking.block_binding.prefix_mismatch_behavior = "drop_block"` and adds the `thinking-binding-controls-2026-08-01` beta. The API then drops the first mismatched block and every later thinking block instead of rejecting the request. The drop applies to that request only and the history keeps the blocks, so after one change to the system prompt, tools or an earlier message, every later request again drops every thinking block from the first one bound to the old prefix onward, until the history before the change no longer carries thinking blocks; such conversations continue without that earlier reasoning, and `metadata.input_transformations` shows the drops on every response. Setting `prefix_mismatch_behavior` also opts the request into enforcement: on models that enforce the check only for organizations created on or after 2026-08-31, an older organization's mismatched blocks reached the model before and are now dropped. A `block_binding` the thinking object already carries is kept as set, and any request whose thinking object carries `block_binding` sends the beta. The Claude driver sends one `anthropic-beta` header: the configured `beta_features`, then the betas of a configured `anthropic-beta` context header (any letter case), then request betas, each named once. Bedrock appends the beta to the `anthropic_beta` list of `additionalModelRequestFields` and `/input_transformations` to `additionalModelResponseFieldPaths`, each named once. Plain generation and structured output behave the same. The driver decides the binding after merging `options.request`, from the thinking object as sent: a `request.thinking.type = "between_tools"` gets no binding and no beta, and a `request.thinking.block_binding` is kept as set and sends the beta. @@ -259,7 +259,7 @@ Bedrock sends developer messages as user text instead of `system` blocks. A Bedr One rule decides what is blank: text with no non-whitespace character. - Blank text in user, system and developer messages never reaches a driver. A message left with no content goes entirely, and an image beside blank text stays. Tool results are sent as they are. -- An assistant turn, after consolidating its assistant and function call rows, with no visible text and no call renders as nothing on every driver, its thinking blocks included. Claude invents no text for it. Whitespace text in a turn with a call stays on Claude and Bedrock, because the replay has to match the response. +- An assistant turn, after consolidating its assistant and function call rows, with no visible text and no call renders as nothing on every driver, its thinking blocks included. Claude invents no text for it. Whitespace text in a turn with a call stays on Claude and Bedrock when the row has no recorded layout; a recorded layout never renders it. A stored conversation with such a row renders a different prefix once after upgrading (Claude sent the text `intercepted`, Bedrock the thinking blocks alone, OpenAI-compatible an empty assistant message): its cache restarts from that row and later thinking blocks no longer match. diff --git a/src/llm/src/util/_index.yaml b/src/llm/src/util/_index.yaml index 8b1b1efe..e9b4bd4b 100644 --- a/src/llm/src/util/_index.yaml +++ b/src/llm/src/util/_index.yaml @@ -53,6 +53,8 @@ entries: - name: thinking kind: library.lua source: file://thinking.lua + imports: + prompt: wippy.llm:prompt # wippy.llm.util:wire_request - name: wire_request diff --git a/src/llm/src/util/thinking.lua b/src/llm/src/util/thinking.lua index fd0219f0..93cd6571 100644 --- a/src/llm/src/util/thinking.lua +++ b/src/llm/src/util/thinking.lua @@ -1,3 +1,5 @@ +local prompt = require("prompt") + local thinking = {} -- Beta that admits thinking.block_binding and adds input_transformations to responses. @@ -97,8 +99,15 @@ local function row_text(content: any): string return text end +local function without_outer_whitespace(text: string): string + return text:match("^%s*(.-)%s*$") :: string +end + -- A row's recorded layout, or nil when it has none or its text is no longer the --- text the layout recorded: the row is what the host stands behind. +-- text the layout recorded: the row is what the host stands behind. Providers +-- compare neither whitespace-only text blocks nor the outer whitespace of a block, so +-- the layout holds only visible text, and whitespace a host stripped from the row's +-- ends is not an edit. local function read_layout(msg: any): any? local positions = type(msg.metadata) == "table" and msg.metadata.content_positions or nil if type(positions) ~= "table" then @@ -108,13 +117,15 @@ local function read_layout(msg: any): any? local text = "" for _, entry in ipairs(positions) do if entry.type == "text" then - table.insert(layout.texts, entry) + if not prompt.is_blank(entry.text) then + table.insert(layout.texts, entry) + end text = text .. entry.text elseif entry.type == "tool_use" then layout.calls[entry.id] = entry.position end end - if text ~= row_text(msg.content) then + if without_outer_whitespace(text) ~= without_outer_whitespace(row_text(msg.content)) then return nil end return layout diff --git a/src/llm/test/replay_safety_test.lua b/src/llm/test/replay_safety_test.lua index acea50be..ba956907 100644 --- a/src/llm/test/replay_safety_test.lua +++ b/src/llm/test/replay_safety_test.lua @@ -139,9 +139,9 @@ local function sse_stream(events) end }, metadata = {} } end -local function claude_stream_events() +local function claude_stream_events(response) local events = { { "message_start", { message = { usage = { input_tokens = 1 } } } } } - for index, block in ipairs(CLAUDE_RESPONSE) do + for index, block in ipairs(response) do local at = index - 1 if block.type == "thinking" then table.insert(events, { "content_block_start", { index = at, content_block = { type = "thinking", thinking = "", signature = "" } } }) @@ -171,10 +171,10 @@ local function eventstream_message(event_type, body) return string.pack(">I4I4I4", total, #headers, 0) .. headers .. payload .. string.pack(">I4", 0) end -local function bedrock_stream() +local function bedrock_stream(response) local chunks = { eventstream_message("messageStart", { role = "assistant" }) } local function add(name, body) table.insert(chunks, eventstream_message(name, body)) end - for index, block in ipairs(BEDROCK_RESPONSE) do + for index, block in ipairs(response) do local at = index - 1 if block.reasoningContent then add("contentBlockDelta", { contentBlockIndex = at, delta = { reasoningContent = { @@ -278,6 +278,55 @@ end local BUDGET_POLICY = { thinking = "budget", default_max_tokens = 2000 } +-- A response as the wire blocks of each driver, from { "s1", "text", { "c1" } } shorthand. +local function claude_blocks(shape) + local blocks = {} + for _, item in ipairs(shape) do + if type(item) == "table" then + table.insert(blocks, { type = "tool_use", id = item[1], name = "t", input = { a = 1 } }) + elseif item:match("^s%d$") then + table.insert(blocks, { type = "thinking", thinking = "", signature = item }) + else + table.insert(blocks, { type = "text", text = item }) + end + end + return blocks +end + +local function bedrock_blocks(shape) + local blocks = {} + for _, item in ipairs(shape) do + if type(item) == "table" then + table.insert(blocks, { toolUse = { toolUseId = item[1], name = "t", input = { a = 1 } } }) + elseif item:match("^s%d$") then + table.insert(blocks, reasoning_block(item)) + else + table.insert(blocks, { text = item }) + end + end + return blocks +end + +-- The four extraction paths, each returning a response as llm hands it to a host. +local EXTRACTION_PATHS = { + { name = "Claude JSON", replay = claude_replay, order = claude_order, extract = function(shape) + return claude_mapper.format_success_response({ content = claude_blocks(shape), stop_reason = "tool_use", + usage = { input_tokens = 1, output_tokens = 1 } }, "m", {}) + end }, + { name = "Claude stream", replay = claude_replay, order = claude_order, extract = function(shape) + local _, _, stream = claude_client.process_stream(sse_stream(claude_stream_events(claude_blocks(shape))), {}) + return claude_mapper.format_streaming_response(stream, {}, stream.usage, stream.finish_reason, stream.metadata) + end }, + { name = "Bedrock JSON", replay = bedrock_replay, order = bedrock_order, extract = function(shape) + return bedrock_mapper.format_success_response({ output = { message = { role = "assistant", content = bedrock_blocks(shape) } }, + stopReason = "tool_use", usage = { inputTokens = 1, outputTokens = 1 } }, {}) + end }, + { name = "Bedrock stream", replay = bedrock_replay, order = bedrock_order, extract = function(shape) + local _, _, stream = bedrock_client.process_converse_stream(bedrock_stream(bedrock_blocks(shape)), {}) + return bedrock_mapper.format_streaming_response(stream, {}, stream.usage, stream.finish_reason, stream.metadata) + end }, +} + local function define_tests() describe("Replay safety", function() after_each(function() @@ -392,7 +441,7 @@ local function define_tests() end) it("replays Claude stream exactly", function() - local _, stream_err, stream_result = claude_client.process_stream(sse_stream(claude_stream_events()), {}) + local _, stream_err, stream_result = claude_client.process_stream(sse_stream(claude_stream_events(CLAUDE_RESPONSE)), {}) test.is_nil(stream_err) local result = claude_mapper.format_streaming_response(stream_result, {}, stream_result.usage, stream_result.finish_reason, stream_result.metadata) assert_order(claude_replay_order(result), RESPONSE_ORDER) @@ -405,12 +454,59 @@ local function define_tests() end) it("replays Bedrock stream exactly", function() - local _, stream_err, stream_result = bedrock_client.process_converse_stream(bedrock_stream(), {}) + local _, stream_err, stream_result = bedrock_client.process_converse_stream(bedrock_stream(BEDROCK_RESPONSE), {}) test.is_nil(stream_err) local result = bedrock_mapper.format_streaming_response(stream_result, {}, stream_result.usage, stream_result.finish_reason, stream_result.metadata) assert_order(bedrock_replay_order(result), RESPONSE_ORDER) end) + for _, path in ipairs(EXTRACTION_PATHS) do + it("renders no blank layout text beside visible text: " .. path.name, function() + local result: any = path.extract({ "s1", "A", { "c1" }, "s2", "\n\n", { "c2" } }) + test.eq(result.result.content, "A\n\n") + local payload = path.replay(stored_rows(result)) + assert_order(path.order(payload.messages[2]), { "s1", "A", "c1", "s2", "c2" }) + end) + + for label, stored in pairs({ + ["as received"] = " \n", ["as empty text"] = "", ["as other whitespace"] = "\t", + }) do + it("keeps the order of a layout with only blank text stored " .. label .. ": " .. path.name, function() + local result: any = path.extract({ "s1", " ", { "c1" }, "s2", "\n", { "c2" } }) + local rows = stored_rows(result) + rows[2].content = text(stored) + local payload = path.replay(rows) + assert_order(path.order(payload.messages[2]), { "s1", "c1", "s2", "c2" }) + end) + end + + it("keeps the order of a layout with only blank text stored with no content: " .. path.name, function() + local result: any = path.extract({ "s1", " ", { "c1" }, "s2", "\n", { "c2" } }) + local rows = stored_rows(result) + rows[2].content = {} + local payload = path.replay(rows) + assert_order(path.order(payload.messages[2]), { "s1", "c1", "s2", "c2" }) + end) + + it("keeps the layout when the host trimmed the text: " .. path.name, function() + local result: any = path.extract({ "s1", "A", { "c1" }, "s2", "\n\n", { "c2" } }) + local rows = stored_rows(result) + rows[2].content = text("A") + local payload = path.replay(rows) + assert_order(path.order(payload.messages[2]), { "s1", "A", "c1", "s2", "c2" }) + end) + + it("renders a turn with only blank layout text and no call as nothing: " .. path.name, function() + local result: any = path.extract({ "s1", " ", "s2", "\n" }) + local rows = stored_rows(result) + table.insert(rows, { role = "user", content = text("next") }) + local payload = path.replay(rows) + for _, message in ipairs(payload.messages) do + test.eq(message.role, "user") + end + end) + end + it("sends no position field on the wire", function() local result = claude_mapper.format_success_response({ content = CLAUDE_RESPONSE, stop_reason = "tool_use", usage = { input_tokens = 1, output_tokens = 1 } }, "m", {}) From 15fd56da36a1722a05cf819b7ef86b1df3538ade Mon Sep 17 00:00:00 2001 From: KickSide Date: Wed, 4 Nov 2026 09:14:07 -0500 Subject: [PATCH 27/37] fix(llm): apply the blank rule to assistant text on every row Whitespace-only assistant text is not rendered inside a turn with a call, whether or not the row has a recorded layout. Claude and Bedrock leave the block out, and OpenAI-compatible sends the empty content its calls need. The API does not compare whitespace-only text blocks, so thinking blocks stay valid; a stored row with one restarts its prompt cache once, as the README notes. --- src/llm/src/README.md | 4 +-- src/llm/src/bedrock/mapper.lua | 8 ++--- src/llm/src/claude/mapper.lua | 8 ++--- src/llm/src/openai_compat/mapper.lua | 8 +++-- src/llm/test/replay_safety_test.lua | 46 ++++++++++++++++++++++++++-- 5 files changed, 58 insertions(+), 16 deletions(-) diff --git a/src/llm/src/README.md b/src/llm/src/README.md index 803132e7..a6538780 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -259,9 +259,9 @@ Bedrock sends developer messages as user text instead of `system` blocks. A Bedr One rule decides what is blank: text with no non-whitespace character. - Blank text in user, system and developer messages never reaches a driver. A message left with no content goes entirely, and an image beside blank text stays. Tool results are sent as they are. -- An assistant turn, after consolidating its assistant and function call rows, with no visible text and no call renders as nothing on every driver, its thinking blocks included. Claude invents no text for it. Whitespace text in a turn with a call stays on Claude and Bedrock when the row has no recorded layout; a recorded layout never renders it. +- An assistant turn, after consolidating its assistant and function call rows, with no visible text and no call renders as nothing on every driver, its thinking blocks included. Claude invents no text for it. Inside a turn that stays, whitespace-only assistant text is not rendered either, on any driver and whether or not the row has a recorded layout; text with something visible keeps its outer whitespace. -A stored conversation with such a row renders a different prefix once after upgrading (Claude sent the text `intercepted`, Bedrock the thinking blocks alone, OpenAI-compatible an empty assistant message): its cache restarts from that row and later thinking blocks no longer match. +A stored conversation with such a row renders a different prefix once after upgrading (Claude sent the text `intercepted`, Bedrock the thinking blocks alone, OpenAI-compatible an empty assistant message): its cache restarts from that row and later thinking blocks no longer match. A stored row with whitespace-only text beside a call also renders differently once after upgrading: its prompt cache restarts one time from that row, while thinking blocks stay valid, because the API does not compare whitespace-only text blocks. A request whose mapped conversation is empty fails with `invalid_request` before any HTTP request on Claude, Bedrock, Google and OpenAI. A system-role note as the only content stays a valid request on OpenAI, as does any system message on OpenAI-compatible. diff --git a/src/llm/src/bedrock/mapper.lua b/src/llm/src/bedrock/mapper.lua index 28bd9cc0..47691a99 100644 --- a/src/llm/src/bedrock/mapper.lua +++ b/src/llm/src/bedrock/mapper.lua @@ -266,9 +266,7 @@ local function without_blank_turns(messages) for _, msg in ipairs(messages) do local visible = msg.role ~= "assistant" for _, block in ipairs(msg.role == "assistant" and msg.content or {}) do - if block.text then - visible = visible or not prompt.is_blank(block.text) - elseif not block.reasoningContent and not block.cachePoint then + if not block.reasoningContent and not block.cachePoint then visible = true end end @@ -391,12 +389,12 @@ function mapper.map_messages(contract_messages) local content = msg.content if type(content) == "string" then - if content ~= "" and not layout then + if not prompt.is_blank(content) and not layout then table.insert(content_blocks, { text = content }) end elseif type(content) == "table" then for _, part in ipairs(content) do - if part.type == "text" and part.text and part.text ~= "" and not layout then + if part.type == "text" and not prompt.is_blank(part.text) and not layout then table.insert(content_blocks, { text = part.text }) elseif part.type == "function_call" then local args = normalize_tool_arguments(part.arguments) diff --git a/src/llm/src/claude/mapper.lua b/src/llm/src/claude/mapper.lua index 041d0472..19bb178c 100644 --- a/src/llm/src/claude/mapper.lua +++ b/src/llm/src/claude/mapper.lua @@ -387,9 +387,7 @@ local function without_blank_turns(messages) for _, msg in ipairs(messages) do local visible = msg.role ~= "assistant" for _, block in ipairs(msg.role == "assistant" and msg.content or {}) do - if block.type == "text" then - visible = visible or not prompt.is_blank(block.text) - elseif block.type ~= "thinking" and block.type ~= "redacted_thinking" then + if block.type ~= "thinking" and block.type ~= "redacted_thinking" then visible = true end end @@ -537,7 +535,7 @@ function mapper.map_messages(contract_messages) end end - if type(regular_content) == "string" and regular_content ~= "" and not layout then + if type(regular_content) == "string" and not prompt.is_blank(regular_content) and not layout then table.insert(content_blocks, { type = "text", text = regular_content @@ -554,7 +552,7 @@ function mapper.map_messages(contract_messages) } order.place_call(tool_use, part.id) table.insert(content_blocks, tool_use) - elseif part.type == "text" and part.text and part.text ~= "" and not layout then + elseif part.type == "text" and not prompt.is_blank(part.text) and not layout then table.insert(content_blocks, part) elseif part.type ~= "text" then table.insert(content_blocks, part) diff --git a/src/llm/src/openai_compat/mapper.lua b/src/llm/src/openai_compat/mapper.lua index c9ba8cba..fa7c065d 100644 --- a/src/llm/src/openai_compat/mapper.lua +++ b/src/llm/src/openai_compat/mapper.lua @@ -266,8 +266,12 @@ function openai_mapper.map_messages(contract_messages, options) assistant_msg.tool_calls = nil end - -- A turn with no visible text and no call renders as nothing. - if assistant_msg.tool_calls or not prompt.is_blank(assistant_msg.content) then + -- A turn with no visible text and no call renders as nothing; a turn with + -- calls and no visible text sends the empty content its calls need. + if prompt.is_blank(assistant_msg.content) then + assistant_msg.content = "" + end + if assistant_msg.tool_calls or assistant_msg.content ~= "" then table.insert(processed_messages, assistant_msg) end diff --git a/src/llm/test/replay_safety_test.lua b/src/llm/test/replay_safety_test.lua index ba956907..71c9794e 100644 --- a/src/llm/test/replay_safety_test.lua +++ b/src/llm/test/replay_safety_test.lua @@ -496,6 +496,13 @@ local function define_tests() assert_order(path.order(payload.messages[2]), { "s1", "A", "c1", "s2", "c2" }) end) + it("renders no whitespace text of a plain-order response: " .. path.name, function() + local result: any = path.extract({ "s1", "\n\n", { "c1" } }) + test.is_nil(result.metadata.content_positions) + local payload = path.replay(stored_rows(result)) + assert_order(path.order(payload.messages[2]), { "s1", "c1" }) + end) + it("renders a turn with only blank layout text and no call as nothing: " .. path.name, function() local result: any = path.extract({ "s1", " ", "s2", "\n" }) local rows = stored_rows(result) @@ -703,14 +710,49 @@ local function define_tests() test.not_nil(google_mapper.map_messages(rows, {})[2].parts[1].functionCall) end) - it("keeps whitespace text inside a Claude turn with a call for an exact replay", function() + it("sends no whitespace text inside a turn with a call on any driver", function() local rows = { { role = "user", content = text("Q") }, { role = "assistant", content = text("\n\n") }, { role = "function_call", content = {}, function_call = { id = "c1", name = "t", arguments = "{}" } }, { role = "function_result", name = "t", function_call_id = "c1", content = text("r") }, } - assert_order(claude_order(claude_mapper.map_messages(rows).messages[2]), { "\n\n", "c1" }) + assert_order(claude_order(claude_mapper.map_messages(rows).messages[2]), { "c1" }) + assert_order(bedrock_order(bedrock_mapper.map_messages(rows).messages[2]), { "c1" }) + test.eq(openai_mapper.map_messages(rows, {})[2].type, "function_call") + test.not_nil(google_mapper.map_messages(rows, {})[2].parts[1].functionCall) + test.eq((compat_mapper.map_messages(rows, {}) :: any)[2].content, "") + end) + + it("sends no whitespace string content inside a turn with a call", function() + local rows = { + { role = "user", content = text("Q") }, + { role = "assistant", content = " " }, + { role = "function_call", content = {}, function_call = { id = "c1", name = "t", arguments = "{}" } }, + { role = "function_result", name = "t", function_call_id = "c1", content = text("r") }, + } + assert_order(claude_order(claude_mapper.map_messages(rows).messages[2]), { "c1" }) + assert_order(bedrock_order(bedrock_mapper.map_messages(rows).messages[2]), { "c1" }) + end) + + it("drops a whitespace part beside visible text", function() + local rows = { + { role = "user", content = text("Q") }, + { role = "assistant", content = { { type = "text", text = " " }, { type = "text", text = "kept" } } }, + { role = "user", content = text("Q2") }, + } + assert_order(claude_order(claude_mapper.map_messages(rows).messages[2]), { "kept" }) + assert_order(bedrock_order(bedrock_mapper.map_messages(rows).messages[2]), { "kept" }) + end) + + it("keeps outer whitespace of visible text", function() + local rows = { + { role = "user", content = text("Q") }, + { role = "assistant", content = text(" kept\n") }, + { role = "user", content = text("Q2") }, + } + assert_order(claude_order(claude_mapper.map_messages(rows).messages[2]), { " kept\n" }) + assert_order(bedrock_order(bedrock_mapper.map_messages(rows).messages[2]), { " kept\n" }) end) it("moves a cache marker off a blank turn", function() From 49f40b9238992ad20137d9480e82c06067597302 Mon Sep 17 00:00:00 2001 From: KickSide Date: Wed, 4 Nov 2026 09:32:18 -0500 Subject: [PATCH 28/37] fix(llm,agent): replay no thinking behind a discarded call A truncated step discards its calls but kept their thinking and layout, so every later request sent an assistant turn ending in thinking, and the fingerprint still matched. The agent now takes the calls and the thinking that led to them out of the response metadata. Rows already stored get the same rule at replay: a call the conversation no longer holds leaves the layout, and thinking behind the last remaining text or call is not sent. The Bedrock JSON path no longer reports a length stop with tool calls as a tool_call stop, which hid truncation and let incomplete calls run. --- src/agent/src/README.md | 4 + src/agent/src/_index.yaml | 1 + src/agent/src/agent.lua | 4 + src/agent/test/_index.yaml | 4 + src/agent/test/replay_safety_test.lua | 201 +++++++++++++++++++++++++- src/llm/src/README.md | 2 +- src/llm/src/bedrock/mapper.lua | 9 +- src/llm/src/claude/mapper.lua | 14 +- src/llm/src/util/thinking.lua | 75 +++++++++- src/llm/test/_index.yaml | 1 + src/llm/test/replay_safety_test.lua | 123 ++++++++++++++++ 11 files changed, 421 insertions(+), 17 deletions(-) diff --git a/src/agent/src/README.md b/src/agent/src/README.md index 1d34de8f..8574e670 100644 --- a/src/agent/src/README.md +++ b/src/agent/src/README.md @@ -365,6 +365,10 @@ retry. User content assistant turn with no tool result in between ends that turn; ordinary system messages, which drivers hoist into the top-level system, do not. +A truncated step (`truncation_reason = "tool_calls"`) discards its calls, so its +`metadata` drops them from `content_positions` along with the thinking that led to +them. Rows persisted from it replay as a turn that ends in text, or as nothing. + Stripping works on copies: the host's message tables keep their blocks, and the stored fingerprint never reaches the provider. When a stale turn still has a function call without a result, the step fails with that error and sends no diff --git a/src/agent/src/_index.yaml b/src/agent/src/_index.yaml index 1a10c827..cad77b90 100644 --- a/src/agent/src/_index.yaml +++ b/src/agent/src/_index.yaml @@ -55,6 +55,7 @@ entries: prompt: wippy.llm:prompt llm: wippy.llm:llm output: wippy.llm:output + thinking: wippy.llm.util:thinking canonical: wippy.agent:canonical # wippy.agent:canonical diff --git a/src/agent/src/agent.lua b/src/agent/src/agent.lua index bff21aaa..842609b8 100644 --- a/src/agent/src/agent.lua +++ b/src/agent/src/agent.lua @@ -5,6 +5,7 @@ local funcs = require("funcs") local output = require("output") local hash = require("hash") local canonical = require("canonical") +local thinking = require("thinking") type UnifiedTool = { name: string, @@ -808,6 +809,9 @@ function agent:step(prompt_builder: any, runtime_options: any): (table?, string? -- to distinguish a tool retry from an empty no-tool response. response.truncation_reason = result.tool_calls and #result.tool_calls > 0 and "tool_calls" or "empty_output" + if response.truncation_reason == "tool_calls" and type(response.metadata) == "table" then + thinking.discard_calls(response.metadata, response.result) + end return response end diff --git a/src/agent/test/_index.yaml b/src/agent/test/_index.yaml index 2660dc44..9a5309de 100644 --- a/src/agent/test/_index.yaml +++ b/src/agent/test/_index.yaml @@ -114,6 +114,10 @@ entries: llm: wippy.llm:llm claude_generate: wippy.llm.claude:generate bedrock_generate: wippy.llm.bedrock:generate + claude_mapper: wippy.llm.claude:mapper + claude_client: wippy.llm.claude:client + bedrock_mapper: wippy.llm.bedrock:mapper + bedrock_client: wippy.llm.bedrock:client test: wippy.test:test method: run_tests diff --git a/src/agent/test/replay_safety_test.lua b/src/agent/test/replay_safety_test.lua index 9284e0d2..f3292519 100644 --- a/src/agent/test/replay_safety_test.lua +++ b/src/agent/test/replay_safety_test.lua @@ -5,6 +5,10 @@ local prompt = require("prompt") local agent = require("agent") local claude_generate = require("claude_generate") local bedrock_generate = require("bedrock_generate") +local claude_mapper = require("claude_mapper") +local claude_client = require("claude_client") +local bedrock_mapper = require("bedrock_mapper") +local bedrock_client = require("bedrock_client") local BUDGET_POLICY = { thinking = "budget", default_max_tokens = 2000 } @@ -49,7 +53,7 @@ end local function handoff_history() local answer = { role = "assistant", content = text("handing off"), metadata = { thinking_blocks = blocks("A"), prefix_fingerprint = "fingerprint-of-agent-a" } } - local builder = prompt.new({ { role = "user", content = text("q1") }, answer }) + local builder = prompt.new({ { role = "user", content = text("q1") }, answer } :: any) builder:add_function_call("handoff", '{"to":"b"}', "c1") builder:add_function_result("handoff", "ok", "c1") return builder, answer @@ -87,6 +91,173 @@ local function sent_thinking(route, sent) return fields and fields.thinking end +local function claude_blocks(shape) + local out = {} + for _, item in ipairs(shape) do + if type(item) == "table" then + table.insert(out, { type = "tool_use", id = item[1], name = "lookup", input = { a = 1 } }) + elseif item:match("^s%d$") then + table.insert(out, { type = "thinking", thinking = "", signature = item }) + else + table.insert(out, { type = "text", text = item }) + end + end + return out +end + +local function bedrock_blocks(shape) + local out = {} + for _, item in ipairs(shape) do + if type(item) == "table" then + table.insert(out, { toolUse = { toolUseId = item[1], name = "lookup", input = { a = 1 } } }) + elseif item:match("^s%d$") then + table.insert(out, { reasoningContent = { reasoningText = { text = "", signature = item } } }) + else + table.insert(out, { text = item }) + end + end + return out +end + +local function claude_stream(shape) + local body = "" + local function add(name, data) body = body .. "event: " .. name .. "\ndata: " .. json.encode(data) .. "\n\n" end + add("message_start", { message = { usage = { input_tokens = 1 } } }) + for index, block in ipairs(claude_blocks(shape)) do + local at = index - 1 + if block.type == "thinking" then + add("content_block_start", { index = at, content_block = { type = "thinking", thinking = "", signature = "" } }) + add("content_block_delta", { index = at, delta = { type = "signature_delta", signature = block.signature } }) + elseif block.type == "text" then + add("content_block_start", { index = at, content_block = { type = "text", text = "" } }) + add("content_block_delta", { index = at, delta = { type = "text_delta", text = block.text } }) + else + add("content_block_start", { index = at, content_block = { type = "tool_use", id = block.id, name = block.name } }) + add("content_block_delta", { index = at, delta = { type = "input_json_delta", partial_json = '{"a":' } }) + end + add("content_block_stop", { index = at }) + end + add("message_delta", { delta = { stop_reason = "max_tokens" }, usage = { output_tokens = 2 } }) + add("message_stop", {}) + local done = false + return { stream = { read = function() + if done then return nil end + done = true + return body + end }, metadata = {} } +end + +local function bedrock_stream(shape) + local function header(name, value) + return string.char(#name) .. name .. string.char(7) .. string.pack(">I2", #value) .. value + end + local chunks = {} + local function add(name, data) + local headers = header(":event-type", name) .. header(":content-type", "application/json") + .. header(":message-type", "event") + local payload = json.encode(data) + local total = 12 + #headers + #payload + 4 + table.insert(chunks, string.pack(">I4I4I4", total, #headers, 0) .. headers .. payload .. string.pack(">I4", 0)) + end + add("messageStart", { role = "assistant" }) + for index, block in ipairs(bedrock_blocks(shape)) do + local at = index - 1 + if block.reasoningContent then + add("contentBlockDelta", { contentBlockIndex = at, delta = { reasoningContent = { + signature = block.reasoningContent.reasoningText.signature } } }) + elseif block.text then + add("contentBlockDelta", { contentBlockIndex = at, delta = { text = block.text } }) + else + add("contentBlockStart", { contentBlockIndex = at, start = { toolUse = { toolUseId = block.toolUse.toolUseId, name = "lookup" } } }) + add("contentBlockDelta", { contentBlockIndex = at, delta = { toolUse = { input = '{"a":' } } }) + end + add("contentBlockStop", { contentBlockIndex = at }) + end + add("messageStop", { stopReason = "max_tokens" }) + add("metadata", { usage = { inputTokens = 1, outputTokens = 2 } }) + local at = 0 + return { stream = { read = function() + at = at + 1 + return chunks[at] + end }, metadata = {} } +end + +local function claude_replay(rows) + local sent = install({ id = "claude_route", provider_model = "wire" }) + assert(claude_generate.handler({ model = "m", messages = rows, options = { max_tokens = 100 } })) + return sent[1].payload.messages +end + +local function bedrock_replay(rows) + local sent = install({ id = "bedrock_route", provider_model = "wire" }) + assert(bedrock_generate.handler({ model = "m", messages = rows, options = { max_tokens = 100 } })) + return sent[1].payload.messages +end + +local function claude_order(message) + local order = {} + for _, part in ipairs(message.content) do + table.insert(order, part.signature or part.id or part.text or part.type) + end + return order +end + +local function bedrock_order(message) + local order = {} + for _, part in ipairs(message.content) do + local reasoning = part.reasoningContent and part.reasoningContent.reasoningText + table.insert(order, reasoning and reasoning.signature or (part.toolUse and part.toolUse.toolUseId) or part.text or "other") + end + return order +end + +local TRUNCATED_PATHS: any = { + { name = "Claude JSON", replay = claude_replay, order = claude_order, extract = function(shape) + return claude_mapper.format_success_response({ content = claude_blocks(shape), stop_reason = "max_tokens", + usage = { input_tokens = 1, output_tokens = 1 } }, "m", {}) + end }, + { name = "Claude stream", replay = claude_replay, order = claude_order, extract = function(shape) + local _, _, stream = claude_client.process_stream(claude_stream(shape), {}) + return claude_mapper.format_streaming_response(stream, {}, stream.usage, stream.finish_reason, stream.metadata) + end }, + { name = "Bedrock JSON", replay = bedrock_replay, order = bedrock_order, extract = function(shape) + return bedrock_mapper.format_success_response({ output = { message = { role = "assistant", content = bedrock_blocks(shape) } }, + stopReason = "max_tokens", usage = { inputTokens = 1, outputTokens = 1 } }, {}) + end }, + { name = "Bedrock stream", replay = bedrock_replay, order = bedrock_order, extract = function(shape) + local _, _, stream = bedrock_client.process_converse_stream(bedrock_stream(shape), {}) + return bedrock_mapper.format_streaming_response(stream, {}, stream.usage, stream.finish_reason, stream.metadata) + end }, +} + +-- The step of an agent on a length-limited response, then the request a host sends +-- after persisting what the step returned. +local function replay_after_truncation(path, shape) + local extracted = path.extract(shape) + agent._llm = { generate = function() + return { result = extracted.result.content, tool_calls = extracted.result.tool_calls, + finish_reason = extracted.finish_reason, metadata = extracted.metadata, tokens = {} } + end } + local builder = prompt.new() + builder:add_user("q") + local response = assert(agent.new(spec("agent_a", 0)):step(builder, {})) + agent._llm = nil + test.is_true(response.truncated) + test.is_nil(response.tool_calls) + local messages = path.replay({ + { role = "user", content = text("q") }, + { role = "assistant", content = text(response.result), metadata = { + thinking_blocks = response.metadata.thinking_blocks, + content_positions = response.metadata.content_positions, + prefix_fingerprint = response.prefix_fingerprint } }, + { role = "user", content = text("try again") }, + }) + for _, message in ipairs(messages) do + if message.role == "assistant" then return path.order(message), response end + end + return {}, response +end + local function define_tests() describe("Agent replay safety", function() after_each(function() @@ -95,6 +266,7 @@ local function define_tests() llm._usage_tracker = nil claude_generate._client = nil bedrock_generate._client = nil + agent._llm = nil end) for _, route in ipairs(ROUTES) do @@ -162,6 +334,33 @@ local function define_tests() end) end + for _, path in ipairs(TRUNCATED_PATHS) do + it("never replays thinking behind a discarded call: text, thinking, call: " .. path.name, function() + local order, response = replay_after_truncation(path, { "A", "s1", { "c1" } }) + test.eq(table.concat(order, "|"), "A") + test.eq(#response.metadata.thinking_blocks, 0) + for _, entry in ipairs(response.metadata.content_positions or {}) do + test.eq(entry.type, "text") + end + end) + + it("never replays thinking behind a discarded call: thinking, call: " .. path.name, function() + local order, response = replay_after_truncation(path, { "s1", { "c1" } }) + test.eq(table.concat(order, "|"), "") + test.eq(#response.metadata.thinking_blocks, 0) + end) + + it("keeps thinking in front of the kept text: " .. path.name, function() + local order = replay_after_truncation(path, { "s1", "A", "s2", { "c1" } }) + test.eq(table.concat(order, "|"), "s1|A") + end) + + it("keeps a plain-order response's thinking in front of its text: " .. path.name, function() + local order = replay_after_truncation(path, { "s1", "A", { "c1" } }) + test.eq(table.concat(order, "|"), "s1|A") + end) + end + it("marks only what the host's predicate rejects", function() local sent = install({ id = "claude_route", provider_model = "wire" }) local builder = handoff_history() diff --git a/src/llm/src/README.md b/src/llm/src/README.md index a6538780..08361995 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -133,7 +133,7 @@ local response, err = llm.generate("Hello", { ### Thinking block binding -Claude and Bedrock store each response's `thinking` blocks as `{ type, thinking, signature }` and `redacted_thinking` blocks as `{ type, data }` in `metadata.thinking_blocks`, in response order (Bedrock: replayed as `reasoningContent.reasoningText` and `reasoningContent.redactedContent`). A response that is thinking, then at most one text block, then tool calls replays in that order with no record. Any other response, such as thinking between text and tool calls, also carries `metadata.content_positions` and a `position` on each thinking block; a host that stores `content_positions` next to `thinking_blocks` on the assistant row (or the first function call row of the response when it has no text) gets the response order back, with each visible text block separate, as long as the calls of that response are consecutive function call rows, which consolidate into one assistant turn. A function result between those rows splits the turn, and the order is then not rebuilt across the split. Whitespace-only text blocks of a recorded response are not rendered: the API compares neither those nor the outer whitespace of a text block, so leaving them out changes no prefix. Rows without `content_positions`, including every row stored before it existed, replay as all the thinking blocks of the row, then its text as one block, then its calls; so do rows whose text, apart from outer whitespace, no longer equals the recorded text. Positions never reach the wire, and other drivers ignore them. Claude replays each block unchanged except for fields an earlier stored shape carried that its type does not accept (`data` on `thinking`, `thinking` and `signature` on `redacted_thinking`). A Claude thinking block is bound to the conversation that produced it: the top-level system prompt, the tools, and every earlier message. After an agent handoff, a prompt or tool change, or a history edit, models that enforce the check reject a request that replays such a block with a 400. +Claude and Bedrock store each response's `thinking` blocks as `{ type, thinking, signature }` and `redacted_thinking` blocks as `{ type, data }` in `metadata.thinking_blocks`, in response order (Bedrock: replayed as `reasoningContent.reasoningText` and `reasoningContent.redactedContent`). A response that is thinking, then at most one text block, then tool calls replays in that order with no record. Any other response, such as thinking between text and tool calls, also carries `metadata.content_positions` and a `position` on each thinking block; a host that stores `content_positions` next to `thinking_blocks` on the assistant row (or the first function call row of the response when it has no text) gets the response order back, with each visible text block separate, as long as the calls of that response are consecutive function call rows, which consolidate into one assistant turn. A function result between those rows splits the turn, and the order is then not rebuilt across the split. Whitespace-only text blocks of a recorded response are not rendered: the API compares neither those nor the outer whitespace of a text block, so leaving them out changes no prefix. Rows without `content_positions`, including every row stored before it existed, replay as all the thinking blocks of the row, then its text as one block, then its calls; so do rows whose text, apart from outer whitespace, no longer equals the recorded text. A recorded call the conversation no longer holds leaves the layout, and the thinking behind the last remaining text or call is not replayed: it led to that call, and an assistant turn cannot end in it. `thinking.discard_calls` applies the same rule to a response's metadata for a caller that discards its calls. Positions never reach the wire, and other drivers ignore them. Claude replays each block unchanged except for fields an earlier stored shape carried that its type does not accept (`data` on `thinking`, `thinking` and `signature` on `redacted_thinking`). A Claude thinking block is bound to the conversation that produced it: the top-level system prompt, the tools, and every earlier message. After an agent handoff, a prompt or tool change, or a history edit, models that enforce the check reject a request that replays such a block with a 400. When a request replays at least one thinking block and sends a thinking object of type `adaptive` or `enabled` (a positive thinking_effort, or `thinking = always` at any effort), the driver sets `thinking.block_binding.prefix_mismatch_behavior = "drop_block"` and adds the `thinking-binding-controls-2026-08-01` beta. The API then drops the first mismatched block and every later thinking block instead of rejecting the request. The drop applies to that request only and the history keeps the blocks, so after one change to the system prompt, tools or an earlier message, every later request again drops every thinking block from the first one bound to the old prefix onward, until the history before the change no longer carries thinking blocks; such conversations continue without that earlier reasoning, and `metadata.input_transformations` shows the drops on every response. Setting `prefix_mismatch_behavior` also opts the request into enforcement: on models that enforce the check only for organizations created on or after 2026-08-31, an older organization's mismatched blocks reached the model before and are now dropped. A `block_binding` the thinking object already carries is kept as set, and any request whose thinking object carries `block_binding` sends the beta. The Claude driver sends one `anthropic-beta` header: the configured `beta_features`, then the betas of a configured `anthropic-beta` context header (any letter case), then request betas, each named once. Bedrock appends the beta to the `anthropic_beta` list of `additionalModelRequestFields` and `/input_transformations` to `additionalModelResponseFieldPaths`, each named once. Plain generation and structured output behave the same. The driver decides the binding after merging `options.request`, from the thinking object as sent: a `request.thinking.type = "between_tools"` gets no binding and no beta, and a `request.thinking.block_binding` is kept as set and sends the beta. diff --git a/src/llm/src/bedrock/mapper.lua b/src/llm/src/bedrock/mapper.lua index 47691a99..1ff4bfa3 100644 --- a/src/llm/src/bedrock/mapper.lua +++ b/src/llm/src/bedrock/mapper.lua @@ -284,7 +284,7 @@ function mapper.map_messages(contract_messages) end contract_messages = prompt.without_blank_text(contract_messages) - local order = thinking.new_turn_order() + local order = thinking.new_turn_order(contract_messages) local stale_thinking = {} local function add_reasoning_blocks(content_blocks, msg) @@ -292,7 +292,7 @@ function mapper.map_messages(contract_messages) return end for _, thinking_block in ipairs(msg.metadata.thinking_blocks) do - local reasoning = reasoning_content(thinking_block) + local reasoning = order.keeps_thinking(thinking_block) and reasoning_content(thinking_block) if reasoning then order.place_thinking(reasoning, thinking_block) if msg.metadata.stale_thinking then @@ -812,8 +812,9 @@ function mapper.format_success_response(converse_response, name_to_id_map) tokens.thinking_tokens = approximate_token_count(thinking_content) end - -- If fallback parser recovered tool calls from text, treat it as tool_call stop - if #extracted.tool_calls > 0 and finish_reason ~= "tool_call" then + -- Tool calls make a tool_call stop (the fallback parser may have recovered them + -- from text), except a length stop: its last call can be incomplete. + if #extracted.tool_calls > 0 and finish_reason ~= "tool_call" and finish_reason ~= output.FINISH_REASON.LENGTH then finish_reason = "tool_call" end diff --git a/src/llm/src/claude/mapper.lua b/src/llm/src/claude/mapper.lua index 19bb178c..3ef667dc 100644 --- a/src/llm/src/claude/mapper.lua +++ b/src/llm/src/claude/mapper.lua @@ -408,7 +408,7 @@ function mapper.map_messages(contract_messages) contract_messages = prompt.without_blank_text(contract_messages) local tool_id = tool_id_mapper(contract_messages) - local order = thinking.new_turn_order() + local order = thinking.new_turn_order(contract_messages) local stale_thinking = {} local function add_thinking_blocks(content_blocks, msg) @@ -416,12 +416,14 @@ function mapper.map_messages(contract_messages) return end for _, thinking_block in ipairs(msg.metadata.thinking_blocks) do - local wire_block = thinking_block_shape(thinking_block) or thinking_block - order.place_thinking(wire_block, thinking_block) - if msg.metadata.stale_thinking then - stale_thinking[wire_block] = true + if order.keeps_thinking(thinking_block) then + local wire_block = thinking_block_shape(thinking_block) or thinking_block + order.place_thinking(wire_block, thinking_block) + if msg.metadata.stale_thinking then + stale_thinking[wire_block] = true + end + table.insert(content_blocks, wire_block) end - table.insert(content_blocks, wire_block) end end diff --git a/src/llm/src/util/thinking.lua b/src/llm/src/util/thinking.lua index 93cd6571..7fb926c1 100644 --- a/src/llm/src/util/thinking.lua +++ b/src/llm/src/util/thinking.lua @@ -103,12 +103,58 @@ local function without_outer_whitespace(text: string): string return text:match("^%s*(.-)%s*$") :: string end +-- The position of the last block that still exists when calls outside `calls` are +-- gone, 0 when none does, or nil when the layout misses no call. Thinking behind that +-- block led to a call that no longer exists: the turn would end in thinking the +-- response did not end in, so it cannot be replayed. +local function last_standing(positions: {any}, calls: {[any]: boolean}): number? + local last, missing = 0, false + for _, entry in ipairs(positions) do + if entry.type == "tool_use" and not calls[entry.id] then + missing = true + elseif entry.type == "tool_use" or not prompt.is_blank(entry.text) then + last = math.max(last, entry.position) + end + end + return missing and last or nil +end + +-- Takes the calls out of a response's metadata when the caller discards them, along +-- with the thinking that led to them. `text` is the response's text. +function thinking.discard_calls(metadata: any, text: any) + local blocks = metadata.thinking_blocks + local positions = metadata.content_positions + if type(positions) ~= "table" then + if prompt.is_blank(text) then + metadata.thinking_blocks = {} + end + return + end + + local limit = last_standing(positions, {}) or math.huge + local kept_blocks = {} + for _, block in ipairs(type(blocks) == "table" and blocks or {}) do + if block.position and block.position < limit then + table.insert(kept_blocks, block) + end + end + local kept_positions = {} + for _, entry in ipairs(positions) do + if entry.type == "text" then + table.insert(kept_positions, entry) + end + end + metadata.thinking_blocks = kept_blocks + metadata.content_positions = limit > 0 and kept_positions or nil +end + -- A row's recorded layout, or nil when it has none or its text is no longer the -- text the layout recorded: the row is what the host stands behind. Providers -- compare neither whitespace-only text blocks nor the outer whitespace of a block, so -- the layout holds only visible text, and whitespace a host stripped from the row's --- ends is not an edit. -local function read_layout(msg: any): any? +-- ends is not an edit. A call the conversation no longer holds leaves the layout, and +-- thinking_limit bounds the thinking that can still be replayed. +local function read_layout(msg: any, calls: {[any]: boolean}): any? local positions = type(msg.metadata) == "table" and msg.metadata.content_positions or nil if type(positions) ~= "table" then return nil @@ -122,9 +168,12 @@ local function read_layout(msg: any): any? end text = text .. entry.text elseif entry.type == "tool_use" then - layout.calls[entry.id] = entry.position + if calls[entry.id] then + layout.calls[entry.id] = entry.position + end end end + layout.thinking_limit = last_standing(positions, calls) if without_outer_whitespace(text) ~= without_outer_whitespace(row_text(msg.content)) then return nil end @@ -133,14 +182,25 @@ end -- Tracks the wire blocks of assistant rows against their recorded layouts and -- rebuilds response order once the rows are consolidated into one turn. -function thinking.new_turn_order() +function thinking.new_turn_order(messages: {any}) local placed: {[any]: any} = {} local active: any = nil local order = {} + local calls: {[any]: boolean} = {} + for _, msg in ipairs(messages) do + if msg.function_call then + calls[msg.function_call.id] = true + elseif type(msg.content) == "table" then + for _, part in ipairs(msg.content) do + if part.type == "function_call" then calls[part.id] = true end + end + end + end + -- Returns the row's own layout, whose text blocks replace the row's text. function order.enter_row(msg: any): any? - local layout = read_layout(msg) + local layout = read_layout(msg, calls) if layout then active = layout end return layout end @@ -149,6 +209,11 @@ function thinking.new_turn_order() active = nil end + function order.keeps_thinking(stored_block: any): boolean + local limit = active and active.thinking_limit + return not (limit and stored_block.position and stored_block.position > limit) + end + function order.place_thinking(wire_block: any, stored_block: any) if active and stored_block.position then placed[wire_block] = { layout = active, position = stored_block.position } diff --git a/src/llm/test/_index.yaml b/src/llm/test/_index.yaml index f7ee3519..f456c0cd 100644 --- a/src/llm/test/_index.yaml +++ b/src/llm/test/_index.yaml @@ -269,6 +269,7 @@ entries: llm: wippy.llm:llm prompt: wippy.llm:prompt providers: wippy.llm.discovery:providers + thinking: wippy.llm.util:thinking claude_mapper: wippy.llm.claude:mapper claude_client: wippy.llm.claude:client claude_generate: wippy.llm.claude:generate diff --git a/src/llm/test/replay_safety_test.lua b/src/llm/test/replay_safety_test.lua index 71c9794e..1857e5bd 100644 --- a/src/llm/test/replay_safety_test.lua +++ b/src/llm/test/replay_safety_test.lua @@ -4,6 +4,7 @@ local registry = require("registry") local llm = require("llm") local prompt = require("prompt") local providers = require("providers") +local thinking = require("thinking") local claude_mapper = require("claude_mapper") local claude_client = require("claude_client") local claude_generate = require("claude_generate") @@ -514,6 +515,89 @@ local function define_tests() end) end + -- Rows a host stored from a response whose calls it discarded. + local function rows_without_calls(result, content): any + local rows = stored_rows(result) + local kept = { rows[1], rows[2], { role = "user", content = text("next") } } + if content then kept[2].content = content end + return kept + end + + local function assistant_order(path, payload) + for _, message in ipairs(payload.messages) do + if message.role == "assistant" then return path.order(message) end + end + return {} + end + + for _, path in ipairs(EXTRACTION_PATHS) do + it("drops thinking behind a discarded call in a stored row: text, thinking, call: " .. path.name, function() + local result: any = path.extract({ "A", "s1", { "c1" } }) + assert_order(assistant_order(path, path.replay(rows_without_calls(result))), { "A" }) + end) + + it("drops thinking behind a discarded call in a stored row: thinking, call: " .. path.name, function() + local result: any = path.extract({ "s1", { "c1" } }) + assert_order(assistant_order(path, path.replay(rows_without_calls(result))), {}) + end) + + it("keeps thinking in front of the last visible text of a stored row: " .. path.name, function() + local result: any = path.extract({ "s1", "A", "s2", { "c1" } }) + assert_order(assistant_order(path, path.replay(rows_without_calls(result))), { "s1", "A" }) + end) + + it("drops thinking behind a discarded call when the host dropped the text too: " .. path.name, function() + local result: any = path.extract({ "A", "s1", { "c1" } }) + for _, content in ipairs({ {}, text(""), text(" ") }) do + assert_order(assistant_order(path, path.replay(rows_without_calls(result, content))), {}) + end + end) + + it("keeps the calls that remain and drops thinking behind the last of them: " .. path.name, function() + local result: any = path.extract({ "s1", { "c1" }, "s2", { "c2" } }) + local rows = stored_rows(result) + local kept = {} + for _, row in ipairs(rows) do + local id = row.function_call and row.function_call.id or row.function_call_id + if id ~= "c2" then table.insert(kept, row) end + end + assert_order(assistant_order(path, path.replay(kept)), { "s1", "c1" }) + end) + + it("keeps a response that itself ended in thinking: " .. path.name, function() + local result: any = path.extract({ "A", "s1" }) + assert_order(assistant_order(path, path.replay(rows_without_calls(result))), { "A", "s1" }) + end) + + it("finds a call rendered after a result of the same response: " .. path.name, function() + local result: any = path.extract({ "s1", "A", { "c1" }, "s2", { "c2" } }) + local rows = stored_rows(result) + local paired = { rows[1], rows[2], rows[3], rows[5], rows[4], rows[6] } + local payload = path.replay(paired) + local signatures = table.concat(assistant_order(path, payload), "|") + test.contains(signatures, "s2") + end) + end + + it("treats a streamed call that never closed as discarded", function() + local events = claude_stream_events(claude_blocks({ "A", "s1", { "c1" } })) + local kept = {} + for _, event in ipairs(events) do + if not (event[1] == "content_block_stop" and event[2].index == 2) then + table.insert(kept, event) + end + end + local _, _, stream = claude_client.process_stream(sse_stream(kept), {}) + local result: any = claude_mapper.format_streaming_response(stream, {}, stream.usage, "max_tokens", stream.metadata) + test.eq(#result.result.tool_calls, 0) + local payload = claude_replay({ { role = "user", content = text("q") }, + { role = "assistant", content = text(result.result.content), metadata = { + thinking_blocks = result.metadata.thinking_blocks, + content_positions = result.metadata.content_positions } }, + { role = "user", content = text("next") } }) + assert_order(claude_order(payload.messages[2]), { "A" }) + end) + it("sends no position field on the wire", function() local result = claude_mapper.format_success_response({ content = CLAUDE_RESPONSE, stop_reason = "tool_use", usage = { input_tokens = 1, output_tokens = 1 } }, "m", {}) @@ -1059,6 +1143,45 @@ local function define_tests() end) end) + describe("discard_calls", function() + it("removes calls and the thinking behind the last visible text", function() + local metadata: any = { + thinking_blocks = { { type = "thinking", signature = "s1", position = 1 }, + { type = "thinking", signature = "s2", position = 3 } }, + content_positions = { { type = "text", position = 2, text = "A" }, + { type = "tool_use", position = 4, id = "c1" } }, + } + thinking.discard_calls(metadata, "A") + test.eq(#metadata.thinking_blocks, 1) + test.eq(metadata.thinking_blocks[1].signature, "s1") + test.eq(#metadata.content_positions, 1) + test.eq(metadata.content_positions[1].type, "text") + end) + + it("clears thinking of a plain-order response with no visible text", function() + local metadata: any = { thinking_blocks = { signed("s1") } } + thinking.discard_calls(metadata, " ") + test.eq(#metadata.thinking_blocks, 0) + end) + + it("keeps thinking of a plain-order response with visible text", function() + local metadata: any = { thinking_blocks = { signed("s1") } } + thinking.discard_calls(metadata, "A") + test.eq(#metadata.thinking_blocks, 1) + end) + + it("leaves no layout when no visible text remains", function() + local metadata: any = { + thinking_blocks = { { type = "thinking", signature = "s1", position = 2 } }, + content_positions = { { type = "text", position = 1, text = " " }, + { type = "tool_use", position = 3, id = "c1" } }, + } + thinking.discard_calls(metadata, " ") + test.eq(#metadata.thinking_blocks, 0) + test.is_nil(metadata.content_positions) + end) + end) + describe("mark_stale_thinking", function() it("marks every block carrier up to the last stale one on copies", function() local first = { role = "assistant", content = text("a"), metadata = { thinking_blocks = { signed("A") }, epoch = "old" } } From fd1f7715b25531f3812ab9925c40d0ccc50bb3ef Mon Sep 17 00:00:00 2001 From: KickSide Date: Wed, 4 Nov 2026 09:34:59 -0500 Subject: [PATCH 29/37] fix(llm): translate foreign tool call ids on Bedrock and OpenAI Bedrock sent call ids verbatim, so an id from another provider could break its 64-character limit or its character set, and OpenAI Responses has the same length limit. The id mapper moves from the Claude mapper to a shared util and takes each provider's limits. Calls and results share a translation derived from a digest of the id, stable across requests and collision-checked. Ids within the limits, and all Claude ids, stay byte-identical. --- src/llm/src/README.md | 12 +++ src/llm/src/bedrock/_index.yaml | 1 + src/llm/src/bedrock/mapper.lua | 16 +++- src/llm/src/claude/_index.yaml | 2 +- src/llm/src/claude/mapper.lua | 46 +---------- src/llm/src/openai/_index.yaml | 1 + src/llm/src/openai/mapper.lua | 9 ++- src/llm/src/util/_index.yaml | 9 +++ src/llm/src/util/tool_id.lua | 63 +++++++++++++++ src/llm/test/replay_safety_test.lua | 119 ++++++++++++++++++++++++++++ 10 files changed, 228 insertions(+), 50 deletions(-) create mode 100644 src/llm/src/util/tool_id.lua diff --git a/src/llm/src/README.md b/src/llm/src/README.md index 08361995..aeb8d5ad 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -254,6 +254,18 @@ Developer messages, positional or not, render the same way on Claude and Bedrock Bedrock sends developer messages as user text instead of `system` blocks. A Bedrock conversation stored before this change therefore renders a different prefix once after upgrading: its cache restarts and its earlier thinking blocks no longer match. The provider rejects such blocks unless the request lets it drop them; hosts that keep thinking blocks across the upgrade strip them with `drop_stale_thinking`. +### Tool Call IDs + +History moves between providers, so a call id can be foreign to the provider a request goes to. Each driver sends ids it accepts unchanged and translates the others, on the call and its result alike, into `tool-` plus a sha256 digest of the id. The translation is the same on every request, because the id is part of the prefix, and the stored history keeps the original. + +| Driver | Limits on the wire | +|---|---| +| Claude | `[a-zA-Z0-9_-]+` | +| Bedrock | `[a-zA-Z0-9_-]+`, 64 characters | +| OpenAI (Responses) | 64 characters | + +OpenAI-compatible endpoints differ and send ids unchanged. Google matches results to calls by the provider's own id or by name. + ### Blank Content One rule decides what is blank: text with no non-whitespace character. diff --git a/src/llm/src/bedrock/_index.yaml b/src/llm/src/bedrock/_index.yaml index 45bb410e..38c6d53e 100644 --- a/src/llm/src/bedrock/_index.yaml +++ b/src/llm/src/bedrock/_index.yaml @@ -136,6 +136,7 @@ entries: prompt: wippy.llm:prompt output: wippy.llm:output route: wippy.llm.util:route + tool_id: wippy.llm.util:tool_id transport: wippy.llm.util:transport - name: embed_titan diff --git a/src/llm/src/bedrock/mapper.lua b/src/llm/src/bedrock/mapper.lua index 1ff4bfa3..b57e2555 100644 --- a/src/llm/src/bedrock/mapper.lua +++ b/src/llm/src/bedrock/mapper.lua @@ -4,6 +4,7 @@ local prompt = require("prompt") local json = require("json") local route = require("route") local transport = require("transport") +local tool_ids = require("tool_id") local mapper = {} @@ -30,6 +31,9 @@ mapper.DEFAULT_MAX_TOKENS = 2000 local MAX_CACHE_CHECKPOINTS = 4 +-- ToolUseBlock.toolUseId and ToolResultBlock.toolUseId: at most 64 characters. +local MAX_TOOL_ID_LENGTH = 64 + -- Converse stopReason -> contract finish_reason mapper.FINISH_REASON_MAP = {} :: {[string]: string} mapper.FINISH_REASON_MAP["end_turn"] = output.FINISH_REASON.STOP @@ -285,6 +289,12 @@ function mapper.map_messages(contract_messages) contract_messages = prompt.without_blank_text(contract_messages) local order = thinking.new_turn_order(contract_messages) + local translate_id = tool_ids.mapper(contract_messages, + { pattern = tool_ids.NATIVE_PATTERN, max_length = MAX_TOOL_ID_LENGTH }) + local function tool_use_id(id) + if id == nil or id == "" then return "" end + return translate_id(id) + end local stale_thinking = {} local function add_reasoning_blocks(content_blocks, msg) @@ -345,7 +355,7 @@ function mapper.map_messages(contract_messages) content = { { toolResult = { - toolUseId = msg.function_call_id or "", + toolUseId = tool_use_id(msg.function_call_id), content = { { text = result_text } }, status = msg.is_error == true and "error" or "success" } @@ -361,7 +371,7 @@ function mapper.map_messages(contract_messages) local tool_use = { toolUse = { - toolUseId = msg.function_call.id or "", + toolUseId = tool_use_id(msg.function_call.id), name = msg.function_call.name or "", input = arguments } @@ -400,7 +410,7 @@ function mapper.map_messages(contract_messages) local args = normalize_tool_arguments(part.arguments) local tool_use = { toolUse = { - toolUseId = part.id or "", + toolUseId = tool_use_id(part.id), name = part.name or "", input = args } diff --git a/src/llm/src/claude/_index.yaml b/src/llm/src/claude/_index.yaml index c3724650..67d066fa 100644 --- a/src/llm/src/claude/_index.yaml +++ b/src/llm/src/claude/_index.yaml @@ -116,12 +116,12 @@ entries: source: file://mapper.lua modules: - json - - hash imports: thinking: wippy.llm.util:thinking prompt: wippy.llm:prompt output: wippy.llm:output route: wippy.llm.util:route + tool_id: wippy.llm.util:tool_id # wippy.llm.claude:provider - name: provider diff --git a/src/llm/src/claude/mapper.lua b/src/llm/src/claude/mapper.lua index 3ef667dc..95a1c129 100644 --- a/src/llm/src/claude/mapper.lua +++ b/src/llm/src/claude/mapper.lua @@ -2,7 +2,7 @@ local thinking = require("thinking") local output = require("output") local prompt = require("prompt") local json = require("json") -local hash = require("hash") +local tool_ids = require("tool_id") local route = require("route") local mapper = {} @@ -68,48 +68,6 @@ local function approximate_token_count(text) return math.ceil(string.len(text) / 4) end --- Preserve valid native IDs. Foreign-provider IDs with other characters need a --- stable translation, shared by calls/results and collision-checked against all --- IDs in the prompt. Character replacement alone merges distinct identities. -local function tool_id_mapper(messages) - local used, translated = {}, {} - local reserved = false - local function reserve_ids() - -- Normal native IDs need no extra history scan or hashing. Reserve all - -- IDs only if a foreign-provider ID actually needs translation. - if reserved then return end - reserved = true - local function reserve(id) - if type(id) == "string" then used[id] = id end - end - for _, msg in ipairs(messages) do - reserve(msg.function_call_id) - if msg.function_call then reserve(msg.function_call.id) end - if type(msg.content) == "table" then - for _, part in ipairs(msg.content) do - if part.type == "function_call" or part.type == "tool_use" then reserve(part.id) end - end - end - end - end - return function(id) - if type(id) ~= "string" or id == "" then error("Tool call ID is required") end - if id:match("^[a-zA-Z0-9_-]+$") then return id end - if translated[id] then return translated[id] end - reserve_ids() - local digest, err = hash.sha256(id) - if err then error(err) end - local base = "tool-" .. digest - local candidate, suffix = base, 0 - while used[candidate] and used[candidate] ~= id do - suffix = suffix + 1 - candidate = base .. "-" .. suffix - end - used[candidate], translated[id] = id, candidate - return candidate - end -end - local function last_cacheable_message_position(messages) for message_index = #messages, 1, -1 do local content = messages[message_index].content or {} @@ -407,7 +365,7 @@ function mapper.map_messages(contract_messages) end contract_messages = prompt.without_blank_text(contract_messages) - local tool_id = tool_id_mapper(contract_messages) + local tool_id = tool_ids.mapper(contract_messages, { pattern = tool_ids.NATIVE_PATTERN }) local order = thinking.new_turn_order(contract_messages) local stale_thinking = {} diff --git a/src/llm/src/openai/_index.yaml b/src/llm/src/openai/_index.yaml index f23a4a9b..caaa5257 100644 --- a/src/llm/src/openai/_index.yaml +++ b/src/llm/src/openai/_index.yaml @@ -157,6 +157,7 @@ entries: output: wippy.llm:output route: wippy.llm.util:route prompt: wippy.llm:prompt + tool_id: wippy.llm.util:tool_id # wippy.llm.openai:organization - name: organization diff --git a/src/llm/src/openai/mapper.lua b/src/llm/src/openai/mapper.lua index 39186462..a4942ffc 100644 --- a/src/llm/src/openai/mapper.lua +++ b/src/llm/src/openai/mapper.lua @@ -1,4 +1,5 @@ local json = require("json") +local tool_ids = require("tool_id") local output = require("output") local route = require("route") local prompt = require("prompt") @@ -199,6 +200,9 @@ function openai_mapper.extract_instructions(contract_messages) return table.concat(parts, "\n\n"), consumed end +-- Responses API call_id: longer ids fail with string_above_max_length. +local MAX_CALL_ID_LENGTH = 64 + -- Map contract messages into the Responses API input[] array. -- Non-positional instructions are pulled out via extract_instructions(). function openai_mapper.map_messages(contract_messages, options) @@ -207,6 +211,7 @@ function openai_mapper.map_messages(contract_messages, options) contract_messages = prompt.without_blank_text(contract_messages) local _, consumed = openai_mapper.extract_instructions(contract_messages) + local wire_call_id = tool_ids.mapper(contract_messages, { max_length = MAX_CALL_ID_LENGTH }) local items = {} for i, msg in ipairs(contract_messages) do @@ -233,7 +238,7 @@ function openai_mapper.map_messages(contract_messages, options) if msg.function_call and msg.function_call.id then table.insert(items, { type = "function_call", - call_id = msg.function_call.id, + call_id = wire_call_id(msg.function_call.id), name = msg.function_call.name, arguments = encode_arguments(msg.function_call.arguments) }) @@ -255,7 +260,7 @@ function openai_mapper.map_messages(contract_messages, options) end table.insert(items, { type = "function_call_output", - call_id = call_id, + call_id = wire_call_id(call_id), output = msg.is_error == true and json.encode({ error = out_text }) or out_text }) end diff --git a/src/llm/src/util/_index.yaml b/src/llm/src/util/_index.yaml index e9b4bd4b..29ec6bf9 100644 --- a/src/llm/src/util/_index.yaml +++ b/src/llm/src/util/_index.yaml @@ -50,6 +50,15 @@ entries: - fallback source: file://fallback.lua + - name: tool_id + kind: library.lua + meta: + type: library + comment: Stable translation of foreign tool call ids into ids a provider accepts + source: file://tool_id.lua + modules: + - hash + - name: thinking kind: library.lua source: file://thinking.lua diff --git a/src/llm/src/util/tool_id.lua b/src/llm/src/util/tool_id.lua new file mode 100644 index 00000000..b2529553 --- /dev/null +++ b/src/llm/src/util/tool_id.lua @@ -0,0 +1,63 @@ +local hash = require("hash") + +local tool_id = {} + +-- The tool call ids Claude and Bedrock accept. A provider's id format can leave +-- another provider's ids outside it after a handoff. +tool_id.NATIVE_PATTERN = "^[a-zA-Z0-9_-]+$" + +local function fit(base: string, suffix: string, max_length: number?): string + if max_length and #base + #suffix > max_length then + base = base:sub(1, max_length - #suffix) + end + return base .. suffix +end + +-- A function mapping a conversation's tool call ids to ids a provider accepts, under +-- `limits`: { pattern?, max_length? }. Ids within the limits pass unchanged. Others +-- get a translation derived from a digest of the id, the same on every request because +-- it is part of the prefix, shared by calls and results, and collision-checked against +-- all ids in the conversation: character replacement or truncation alone would merge +-- distinct identities. +function tool_id.mapper(messages: {any}, limits: table): (id: any) -> string + local used, translated = {}, {} + local reserved = false + local function reserve_ids() + -- Normal native ids need no extra history scan or hashing. Reserve all + -- ids only if a foreign-provider id actually needs translation. + if reserved then return end + reserved = true + local function reserve(id) + if type(id) == "string" then used[id] = id end + end + for _, msg in ipairs(messages) do + reserve(msg.function_call_id) + if msg.function_call then reserve(msg.function_call.id) end + if type(msg.content) == "table" then + for _, part in ipairs(msg.content) do + if part.type == "function_call" or part.type == "tool_use" then reserve(part.id) end + end + end + end + end + return function(id) + if type(id) ~= "string" or id == "" then error("Tool call ID is required") end + local fits_pattern = not limits.pattern or id:match(limits.pattern) + local fits_length = not limits.max_length or #id <= limits.max_length + if fits_pattern and fits_length then return id end + if translated[id] then return translated[id] end + reserve_ids() + local digest, err = hash.sha256(id) + if err then error(err) end + local base = "tool-" .. digest + local candidate, suffix = fit(base, "", limits.max_length), 0 + while used[candidate] and used[candidate] ~= id do + suffix = suffix + 1 + candidate = fit(base, "-" .. suffix, limits.max_length) + end + used[candidate], translated[id] = id, candidate + return candidate + end +end + +return tool_id diff --git a/src/llm/test/replay_safety_test.lua b/src/llm/test/replay_safety_test.lua index 1857e5bd..7f30723e 100644 --- a/src/llm/test/replay_safety_test.lua +++ b/src/llm/test/replay_safety_test.lua @@ -1143,6 +1143,125 @@ local function define_tests() end) end) + describe("foreign tool call ids", function() + local LONG = "get_customer_subscription_details_0194c7a2-9c1e-7b3f-8a4d-1234567890ab" + local LONG_SIBLING = "get_customer_subscription_details_0194c7a2-9c1e-7b3f-8a4d-1234567890ac" + + local function round(id): any + return { + { role = "user", content = text("q") }, + { role = "function_call", content = {}, function_call = { id = id, name = "t", arguments = "{}" } }, + { role = "function_result", name = "t", function_call_id = id, content = text("r") }, + } + end + + local function bedrock_ids(rows): any + local messages: any = bedrock_mapper.map_messages(rows).messages + local ids = {} + for _, message in ipairs(messages) do + for _, block in ipairs(message.content) do + if block.toolUse then table.insert(ids, block.toolUse.toolUseId) end + if block.toolResult then table.insert(ids, block.toolResult.toolUseId) end + end + end + return ids + end + + local function openai_ids(rows): any + local ids = {} + for _, item in ipairs(openai_mapper.map_messages(rows, {})) do + if item.call_id then table.insert(ids, item.call_id) end + end + return ids + end + + it("fits the same 70-character id on a Bedrock call and result", function() + test.eq(#LONG, 70) + local ids = bedrock_ids(round(LONG)) + test.eq(#ids, 2) + test.eq(ids[1], ids[2]) + test.is_true(#ids[1] <= 64) + test.not_nil(ids[1]:match("^[a-zA-Z0-9_-]+$")) + end) + + it("translates ids with characters Bedrock rejects", function() + local ids = bedrock_ids(round("provider:call/one")) + test.eq(ids[1], ids[2]) + test.not_nil(ids[1]:match("^[a-zA-Z0-9_-]+$")) + end) + + it("keeps Bedrock ids stable over builds", function() + test.eq(bedrock_ids(round(LONG))[1], bedrock_ids(round(LONG))[1]) + end) + + it("keeps two long Bedrock ids sharing a prefix apart", function() + local rows = round(LONG) + for _, row in ipairs(round(LONG_SIBLING)) do + if row.role ~= "user" then table.insert(rows, row) end + end + local ids = bedrock_ids(rows) + test.eq(#ids, 4) + test.eq(ids[1], ids[2]) + test.eq(ids[3], ids[4]) + test.is_false(ids[1] == ids[3]) + end) + + it("avoids a Bedrock native id equal to a translated one", function() + local translated = bedrock_ids(round(LONG))[1] + local rows = round(LONG) + for _, row in ipairs(round(translated)) do + if row.role ~= "user" then table.insert(rows, row) end + end + local ids = bedrock_ids(rows) + test.is_false(ids[1] == ids[3]) + test.eq(ids[3], translated) + test.is_true(#ids[1] <= 64) + end) + + it("leaves ordinary ids byte-identical on Bedrock and OpenAI", function() + local ordinary = "tooluse_Abc-123_" .. string.rep("x", 48) + test.eq(#ordinary, 64) + test.eq(bedrock_ids(round(ordinary))[1], ordinary) + test.eq(bedrock_ids(round(ordinary))[2], ordinary) + test.eq(openai_ids(round(ordinary))[1], ordinary) + test.eq(openai_ids(round("call:with/other-chars"))[1], "call:with/other-chars") + end) + + it("translates a function_call content part on Bedrock", function() + local messages: any = bedrock_mapper.map_messages({ + { role = "assistant", content = { { type = "function_call", id = LONG, name = "t", arguments = {} } } }, + { role = "function_result", name = "t", function_call_id = LONG, content = text("r") }, + }).messages + local call = messages[1].content[1].toolUse.toolUseId + test.is_true(#call <= 64) + test.eq(messages[2].content[1].toolResult.toolUseId, call) + end) + + it("fits the same long id on an OpenAI call and output", function() + local ids = openai_ids(round(LONG)) + test.eq(#ids, 2) + test.eq(ids[1], ids[2]) + test.is_true(#ids[1] <= 64) + test.eq(openai_ids(round(LONG))[1], ids[1]) + end) + + it("keeps the layout of a Bedrock response with a long id", function() + local result = bedrock_mapper.format_success_response({ output = { message = { role = "assistant", content = { + reasoning_block("s1"), { text = "a" }, reasoning_block("s2"), + { toolUse = { toolUseId = LONG, name = "t", input = { a = 1 } } }, + } } }, stopReason = "tool_use", usage = { inputTokens = 1, outputTokens = 1 } }, {}) + local order = bedrock_order(bedrock_replay(stored_rows(result)).messages[2]) + test.eq(order[3], "s2") + test.is_true(#order[4] <= 64) + end) + + it("keeps Claude's translated ids as they are", function() + local messages: any = claude_mapper.map_messages(round("provider:call/one")).messages + test.eq(#messages[2].content[1].id, 69) + test.eq(messages[2].content[1].id:sub(1, 5), "tool-") + end) + end) + describe("discard_calls", function() it("removes calls and the thinking behind the last visible text", function() local metadata: any = { From 5d824f7e904e35d5207165598402a9646784981a Mon Sep 17 00:00:00 2001 From: KickSide Date: Wed, 4 Nov 2026 11:04:05 -0500 Subject: [PATCH 30/37] fix(agent): keep a false tool result in parallel execution --- src/agent/src/tools/caller.lua | 6 ++++- src/agent/test/tools/caller_test.lua | 33 ++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/agent/src/tools/caller.lua b/src/agent/src/tools/caller.lua index af35922b..14659d3e 100644 --- a/src/agent/src/tools/caller.lua +++ b/src/agent/src/tools/caller.lua @@ -742,7 +742,11 @@ local function command_result(pending: PendingCall, ok: boolean?): any if err then return { result = nil, error = err, tool_call = pending.tool_call } end - return { result = payload and payload:data() or nil, error = nil, tool_call = pending.tool_call } + local result = nil + if payload then + result = payload:data() + end + return { result = result, error = nil, tool_call = pending.tool_call } end local _, err = command:result() return { diff --git a/src/agent/test/tools/caller_test.lua b/src/agent/test/tools/caller_test.lua index 4e46b561..09f200b1 100644 --- a/src/agent/test/tools/caller_test.lua +++ b/src/agent/test/tools/caller_test.lua @@ -1222,6 +1222,39 @@ local function define_tests() return notes end + for _, observed in ipairs({ "returned", "listener" }) do + it("preserves scalar tool values in " .. observed .. " results across strategies", function() + for _, strategy in ipairs(STRATEGIES) do + tool_results["test:false"] = { result = false } + tool_results["test:zero"] = { result = 0 } + tool_results["test:empty"] = { result = "" } + local notes = recording_listener() + local caller = tool_caller.new():set_strategy(strategy):set_result_listener(notes.listener) + local returned, err = caller:execute({}, { + falsy = tool("falsy", "test:false"), + zero = tool("zero", "test:zero"), + empty = tool("empty", "test:empty"), + failed = tool("failed", "test:failing_tool"), + }) + test.is_nil(err) + test.eq(#notes.order, 4, strategy) + local results = returned + if observed == "listener" then results = notes.results end + for call_id in pairs(returned) do + test.eq(notes.counts[call_id], 1, strategy .. " " .. call_id) + end + test.is_nil(results.failed.result, strategy) + test.eq(results.failed.error, "Tool execution failed", strategy) + test.eq(results.zero.result, 0, strategy) + test.is_nil(results.zero.error, strategy) + test.eq(results.empty.result, "", strategy) + test.is_nil(results.empty.error, strategy) + test.is_nil(results.falsy.error, strategy) + test.eq(results.falsy.result, false, strategy .. " " .. observed .. " false result") + end + end) + end + it("accepts a function or nil and rejects other values", function() local caller = tool_caller.new() test.eq(caller:set_result_listener(function() end), caller) From 81f9339f21aaf9d06d4cef5076daccf0e3c4c74b Mon Sep 17 00:00:00 2001 From: KickSide Date: Wed, 4 Nov 2026 11:04:05 -0500 Subject: [PATCH 31/37] fix(llm): scope a layout's call presence to its own turn --- src/llm/src/util/thinking.lua | 26 +++++++++++++---- src/llm/test/replay_safety_test.lua | 45 ++++++++++++++++++++++------- 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/src/llm/src/util/thinking.lua b/src/llm/src/util/thinking.lua index 7fb926c1..b1b8cbd8 100644 --- a/src/llm/src/util/thinking.lua +++ b/src/llm/src/util/thinking.lua @@ -187,20 +187,34 @@ function thinking.new_turn_order(messages: {any}) local active: any = nil local order = {} + local calls_by_row: {[any]: {[any]: boolean}} = {} local calls: {[any]: boolean} = {} + local after_result = false for _, msg in ipairs(messages) do - if msg.function_call then - calls[msg.function_call.id] = true - elseif type(msg.content) == "table" then - for _, part in ipairs(msg.content) do - if part.type == "function_call" then calls[part.id] = true end + if msg.role == prompt.ROLE.ASSISTANT or msg.role == prompt.ROLE.FUNCTION_CALL then + if after_result and msg.role == prompt.ROLE.ASSISTANT then calls = {} end + after_result = false + calls_by_row[msg] = calls + if msg.function_call then + calls[msg.function_call.id] = true + elseif type(msg.content) == "table" then + for _, part in ipairs(msg.content) do + if part.type == "function_call" then calls[part.id] = true end + end end + elseif msg.role == prompt.ROLE.FUNCTION_RESULT then + -- Calls after a result can belong to the same response; a new + -- assistant row starts the next turn. + after_result = true + elseif msg.role ~= prompt.ROLE.CACHE_MARKER then + calls = {} + after_result = false end end -- Returns the row's own layout, whose text blocks replace the row's text. function order.enter_row(msg: any): any? - local layout = read_layout(msg, calls) + local layout = read_layout(msg, calls_by_row[msg] or {}) if layout then active = layout end return layout end diff --git a/src/llm/test/replay_safety_test.lua b/src/llm/test/replay_safety_test.lua index 7f30723e..1d7aa344 100644 --- a/src/llm/test/replay_safety_test.lua +++ b/src/llm/test/replay_safety_test.lua @@ -140,7 +140,7 @@ local function sse_stream(events) end }, metadata = {} } end -local function claude_stream_events(response) +local function claude_stream_events(response, unclosed) local events = { { "message_start", { message = { usage = { input_tokens = 1 } } } } } for index, block in ipairs(response) do local at = index - 1 @@ -154,9 +154,11 @@ local function claude_stream_events(response) table.insert(events, { "content_block_start", { index = at, content_block = { type = "tool_use", id = block.id, name = block.name } } }) table.insert(events, { "content_block_delta", { index = at, delta = { type = "input_json_delta", partial_json = json.encode(block.input) } } }) end - table.insert(events, { "content_block_stop", { index = at } }) + if not (unclosed and index == #response) then + table.insert(events, { "content_block_stop", { index = at } }) + end end - table.insert(events, { "message_delta", { delta = { stop_reason = "tool_use" }, usage = { output_tokens = 2 } } }) + table.insert(events, { "message_delta", { delta = { stop_reason = unclosed and "max_tokens" or "tool_use" }, usage = { output_tokens = 2 } } }) table.insert(events, { "message_stop", {} }) return events end @@ -172,7 +174,7 @@ local function eventstream_message(event_type, body) return string.pack(">I4I4I4", total, #headers, 0) .. headers .. payload .. string.pack(">I4", 0) end -local function bedrock_stream(response) +local function bedrock_stream(response, unclosed) local chunks = { eventstream_message("messageStart", { role = "assistant" }) } local function add(name, body) table.insert(chunks, eventstream_message(name, body)) end for index, block in ipairs(response) do @@ -186,9 +188,11 @@ local function bedrock_stream(response) add("contentBlockStart", { contentBlockIndex = at, start = { toolUse = { toolUseId = block.toolUse.toolUseId, name = "t" } } }) add("contentBlockDelta", { contentBlockIndex = at, delta = { toolUse = { input = json.encode(block.toolUse.input) } } }) end - add("contentBlockStop", { contentBlockIndex = at }) + if not (unclosed and index == #response) then + add("contentBlockStop", { contentBlockIndex = at }) + end end - add("messageStop", { stopReason = "tool_use" }) + add("messageStop", { stopReason = unclosed and "max_tokens" or "tool_use" }) add("metadata", { usage = { inputTokens = 1, outputTokens = 2 } }) local at = 0 return { stream = { read = function() @@ -314,16 +318,16 @@ local EXTRACTION_PATHS = { return claude_mapper.format_success_response({ content = claude_blocks(shape), stop_reason = "tool_use", usage = { input_tokens = 1, output_tokens = 1 } }, "m", {}) end }, - { name = "Claude stream", replay = claude_replay, order = claude_order, extract = function(shape) - local _, _, stream = claude_client.process_stream(sse_stream(claude_stream_events(claude_blocks(shape))), {}) + { name = "Claude stream", replay = claude_replay, order = claude_order, extract = function(shape, unclosed) + local _, _, stream = claude_client.process_stream(sse_stream(claude_stream_events(claude_blocks(shape), unclosed)), {}) return claude_mapper.format_streaming_response(stream, {}, stream.usage, stream.finish_reason, stream.metadata) end }, { name = "Bedrock JSON", replay = bedrock_replay, order = bedrock_order, extract = function(shape) return bedrock_mapper.format_success_response({ output = { message = { role = "assistant", content = bedrock_blocks(shape) } }, stopReason = "tool_use", usage = { inputTokens = 1, outputTokens = 1 } }, {}) end }, - { name = "Bedrock stream", replay = bedrock_replay, order = bedrock_order, extract = function(shape) - local _, _, stream = bedrock_client.process_converse_stream(bedrock_stream(bedrock_blocks(shape)), {}) + { name = "Bedrock stream", replay = bedrock_replay, order = bedrock_order, extract = function(shape, unclosed) + local _, _, stream = bedrock_client.process_converse_stream(bedrock_stream(bedrock_blocks(shape), unclosed), {}) return bedrock_mapper.format_streaming_response(stream, {}, stream.usage, stream.finish_reason, stream.metadata) end }, } @@ -531,6 +535,27 @@ local function define_tests() end for _, path in ipairs(EXTRACTION_PATHS) do + it("drops thinking behind a missing call whose id belongs to an earlier turn: " .. path.name, function() + local earlier = stored_rows(path.extract({ "before", "s1", { "c" } })) + local before = path.replay(earlier) + local result: any = path.extract({ "A", "s2", { "c" } }, true) + local later = rows_without_calls(result) + local rows = {} + for _, row in ipairs(earlier) do table.insert(rows, row) end + table.insert(rows, later[2]) + table.insert(rows, later[3]) + local payload = path.replay(rows) + for index, message in ipairs(before.messages) do + test.eq(json.encode(payload.messages[index]), json.encode(message), "earlier turn " .. index) + end + local last_assistant + for _, message in ipairs(payload.messages) do + if message.role == "assistant" then last_assistant = message end + end + test.not_nil(last_assistant) + assert_order(path.order(last_assistant), { "A" }) + end) + it("drops thinking behind a discarded call in a stored row: text, thinking, call: " .. path.name, function() local result: any = path.extract({ "A", "s1", { "c1" } }) assert_order(assistant_order(path, path.replay(rows_without_calls(result))), { "A" }) From 3971adf93a68be8ee437428b8c56fbb210c65c86 Mon Sep 17 00:00:00 2001 From: KickSide Date: Wed, 4 Nov 2026 11:27:52 -0500 Subject: [PATCH 32/37] fix(llm): never replay an assistant turn ending in thinking --- src/llm/src/util/thinking.lua | 22 ++--- src/llm/test/replay_safety_test.lua | 123 +++++++++++++++++++++++----- 2 files changed, 113 insertions(+), 32 deletions(-) diff --git a/src/llm/src/util/thinking.lua b/src/llm/src/util/thinking.lua index b1b8cbd8..50568050 100644 --- a/src/llm/src/util/thinking.lua +++ b/src/llm/src/util/thinking.lua @@ -103,20 +103,17 @@ local function without_outer_whitespace(text: string): string return text:match("^%s*(.-)%s*$") :: string end --- The position of the last block that still exists when calls outside `calls` are --- gone, 0 when none does, or nil when the layout misses no call. Thinking behind that --- block led to a call that no longer exists: the turn would end in thinking the --- response did not end in, so it cannot be replayed. -local function last_standing(positions: {any}, calls: {[any]: boolean}): number? - local last, missing = 0, false +-- The position of the last visible text or retained call, or 0 when neither remains. +-- Thinking behind it cannot end a replayed assistant turn. +local function last_standing(positions: {any}, calls: {[any]: boolean}): number + local last = 0 for _, entry in ipairs(positions) do - if entry.type == "tool_use" and not calls[entry.id] then - missing = true - elseif entry.type == "tool_use" or not prompt.is_blank(entry.text) then + if (entry.type == "tool_use" and calls[entry.id]) + or (entry.type == "text" and not prompt.is_blank(entry.text)) then last = math.max(last, entry.position) end end - return missing and last or nil + return last end -- Takes the calls out of a response's metadata when the caller discards them, along @@ -131,7 +128,7 @@ function thinking.discard_calls(metadata: any, text: any) return end - local limit = last_standing(positions, {}) or math.huge + local limit = last_standing(positions, {}) local kept_blocks = {} for _, block in ipairs(type(blocks) == "table" and blocks or {}) do if block.position and block.position < limit then @@ -224,6 +221,9 @@ function thinking.new_turn_order(messages: {any}) end function order.keeps_thinking(stored_block: any): boolean + if stored_block.type == "thinking" and (stored_block.signature or "") == "" then + return false + end local limit = active and active.thinking_limit return not (limit and stored_block.position and stored_block.position > limit) end diff --git a/src/llm/test/replay_safety_test.lua b/src/llm/test/replay_safety_test.lua index 1d7aa344..4a101ee3 100644 --- a/src/llm/test/replay_safety_test.lua +++ b/src/llm/test/replay_safety_test.lua @@ -140,13 +140,16 @@ local function sse_stream(events) end }, metadata = {} } end -local function claude_stream_events(response, unclosed) +local function claude_stream_events(response, unclosed, stop_reason) local events = { { "message_start", { message = { usage = { input_tokens = 1 } } } } } for index, block in ipairs(response) do local at = index - 1 if block.type == "thinking" then table.insert(events, { "content_block_start", { index = at, content_block = { type = "thinking", thinking = "", signature = "" } } }) - table.insert(events, { "content_block_delta", { index = at, delta = { type = "signature_delta", signature = block.signature } } }) + table.insert(events, { "content_block_delta", { index = at, delta = { type = "thinking_delta", thinking = block.thinking } } }) + if block.signature ~= "" then + table.insert(events, { "content_block_delta", { index = at, delta = { type = "signature_delta", signature = block.signature } } }) + end elseif block.type == "text" then table.insert(events, { "content_block_start", { index = at, content_block = { type = "text", text = "" } } }) table.insert(events, { "content_block_delta", { index = at, delta = { type = "text_delta", text = block.text } } }) @@ -158,7 +161,7 @@ local function claude_stream_events(response, unclosed) table.insert(events, { "content_block_stop", { index = at } }) end end - table.insert(events, { "message_delta", { delta = { stop_reason = unclosed and "max_tokens" or "tool_use" }, usage = { output_tokens = 2 } } }) + table.insert(events, { "message_delta", { delta = { stop_reason = stop_reason or (unclosed and "max_tokens" or "tool_use") }, usage = { output_tokens = 2 } } }) table.insert(events, { "message_stop", {} }) return events end @@ -174,14 +177,17 @@ local function eventstream_message(event_type, body) return string.pack(">I4I4I4", total, #headers, 0) .. headers .. payload .. string.pack(">I4", 0) end -local function bedrock_stream(response, unclosed) +local function bedrock_stream(response, unclosed, stop_reason) local chunks = { eventstream_message("messageStart", { role = "assistant" }) } local function add(name, body) table.insert(chunks, eventstream_message(name, body)) end for index, block in ipairs(response) do local at = index - 1 if block.reasoningContent then - add("contentBlockDelta", { contentBlockIndex = at, delta = { reasoningContent = { - signature = block.reasoningContent.reasoningText.signature } } }) + local reasoning = block.reasoningContent.reasoningText + add("contentBlockDelta", { contentBlockIndex = at, delta = { reasoningContent = { text = reasoning.text } } }) + if reasoning.signature ~= "" then + add("contentBlockDelta", { contentBlockIndex = at, delta = { reasoningContent = { signature = reasoning.signature } } }) + end elseif block.text then add("contentBlockDelta", { contentBlockIndex = at, delta = { text = block.text } }) else @@ -192,7 +198,7 @@ local function bedrock_stream(response, unclosed) add("contentBlockStop", { contentBlockIndex = at }) end end - add("messageStop", { stopReason = unclosed and "max_tokens" or "tool_use" }) + add("messageStop", { stopReason = stop_reason or (unclosed and "max_tokens" or "tool_use") }) add("metadata", { usage = { inputTokens = 1, outputTokens = 2 } }) local at = 0 return { stream = { read = function() @@ -201,7 +207,7 @@ local function bedrock_stream(response, unclosed) end }, metadata = {} } end --- The rows a host persists from one tool-using response. +-- The rows a host persists from one response. local function stored_rows(result): any local metadata = { thinking_blocks = result.metadata.thinking_blocks, @@ -287,7 +293,9 @@ local BUDGET_POLICY = { thinking = "budget", default_max_tokens = 2000 } local function claude_blocks(shape) local blocks = {} for _, item in ipairs(shape) do - if type(item) == "table" then + if type(item) == "table" and item.type == "thinking" then + table.insert(blocks, item) + elseif type(item) == "table" then table.insert(blocks, { type = "tool_use", id = item[1], name = "t", input = { a = 1 } }) elseif item:match("^s%d$") then table.insert(blocks, { type = "thinking", thinking = "", signature = item }) @@ -301,7 +309,9 @@ end local function bedrock_blocks(shape) local blocks = {} for _, item in ipairs(shape) do - if type(item) == "table" then + if type(item) == "table" and item.type == "thinking" then + table.insert(blocks, { reasoningContent = { reasoningText = { text = item.thinking, signature = item.signature } } }) + elseif type(item) == "table" then table.insert(blocks, { toolUse = { toolUseId = item[1], name = "t", input = { a = 1 } } }) elseif item:match("^s%d$") then table.insert(blocks, reasoning_block(item)) @@ -314,20 +324,22 @@ end -- The four extraction paths, each returning a response as llm hands it to a host. local EXTRACTION_PATHS = { - { name = "Claude JSON", replay = claude_replay, order = claude_order, extract = function(shape) - return claude_mapper.format_success_response({ content = claude_blocks(shape), stop_reason = "tool_use", + { name = "Claude JSON", replay = claude_replay, order = claude_order, extract = function(shape, _, stop_reason) + return claude_mapper.format_success_response({ content = claude_blocks(shape), stop_reason = stop_reason or "tool_use", usage = { input_tokens = 1, output_tokens = 1 } }, "m", {}) end }, - { name = "Claude stream", replay = claude_replay, order = claude_order, extract = function(shape, unclosed) - local _, _, stream = claude_client.process_stream(sse_stream(claude_stream_events(claude_blocks(shape), unclosed)), {}) + { name = "Claude stream", streaming = true, replay = claude_replay, order = claude_order, extract = function(shape, unclosed, stop_reason) + local _, err, stream = claude_client.process_stream(sse_stream(claude_stream_events(claude_blocks(shape), unclosed, stop_reason)), {}) + test.is_nil(err) return claude_mapper.format_streaming_response(stream, {}, stream.usage, stream.finish_reason, stream.metadata) end }, - { name = "Bedrock JSON", replay = bedrock_replay, order = bedrock_order, extract = function(shape) + { name = "Bedrock JSON", replay = bedrock_replay, order = bedrock_order, extract = function(shape, _, stop_reason) return bedrock_mapper.format_success_response({ output = { message = { role = "assistant", content = bedrock_blocks(shape) } }, - stopReason = "tool_use", usage = { inputTokens = 1, outputTokens = 1 } }, {}) + stopReason = stop_reason or "tool_use", usage = { inputTokens = 1, outputTokens = 1 } }, {}) end }, - { name = "Bedrock stream", replay = bedrock_replay, order = bedrock_order, extract = function(shape, unclosed) - local _, _, stream = bedrock_client.process_converse_stream(bedrock_stream(bedrock_blocks(shape), unclosed), {}) + { name = "Bedrock stream", streaming = true, replay = bedrock_replay, order = bedrock_order, extract = function(shape, unclosed, stop_reason) + local _, err, stream = bedrock_client.process_converse_stream(bedrock_stream(bedrock_blocks(shape), unclosed, stop_reason), {}) + test.is_nil(err) return bedrock_mapper.format_streaming_response(stream, {}, stream.usage, stream.finish_reason, stream.metadata) end }, } @@ -589,11 +601,80 @@ local function define_tests() assert_order(assistant_order(path, path.replay(kept)), { "s1", "c1" }) end) - it("keeps a response that itself ended in thinking: " .. path.name, function() - local result: any = path.extract({ "A", "s1" }) - assert_order(assistant_order(path, path.replay(rows_without_calls(result))), { "A", "s1" }) + for _, case in ipairs({ + { name = "A T", shape = { "A", "s1" }, expected = { "A" } }, + { name = "T A T", shape = { "s1", "A", "s2" }, expected = { "s1", "A" } }, + }) do + it("drops trailing thinking after max_tokens: " .. case.name .. ": " .. path.name, function() + local result: any = path.extract(case.shape, false, "max_tokens") + test.eq(result.finish_reason, "length") + test.eq(#result.result.tool_calls, 0) + test.not_nil(result.metadata.content_positions) + local rows = rows_without_calls(result) + for _ = 1, 2 do + local payload = path.replay(rows) + assert_order(assistant_order(path, payload), case.expected) + test.eq(payload.messages[#payload.messages].role, "user") + table.insert(rows, { role = "assistant", content = text("Later answer") }) + table.insert(rows, { role = "user", content = text("Another question") }) + end + end) + end + + it("omits thinking alone after max_tokens before a user message: " .. path.name, function() + local result: any = path.extract({ "s1" }, false, "max_tokens") + test.eq(result.finish_reason, "length") + test.eq(#result.result.tool_calls, 0) + test.eq(#result.metadata.thinking_blocks, 1) + test.is_nil(result.metadata.content_positions) + local payload = path.replay(rows_without_calls(result)) + for _, message in ipairs(payload.messages) do + test.eq(message.role, "user") + end + test.contains(table.concat(path.order(payload.messages[#payload.messages]), "|"), "next") + end) + + it("keeps signed plain-order thinking in front of text: " .. path.name, function() + local result: any = path.extract({ "s1", "s2", "A" }, false, "max_tokens") + test.eq(result.finish_reason, "length") + test.is_nil(result.metadata.content_positions) + for _, block in ipairs(result.metadata.thinking_blocks) do test.is_nil(block.position) end + assert_order(assistant_order(path, path.replay(rows_without_calls(result))), { "s1", "s2", "A" }) + end) + + it("keeps a completed call turn after max_tokens: " .. path.name, function() + local result: any = path.extract({ "s1", "A", "s2", { "c1" } }, false, "max_tokens") + test.eq(result.finish_reason, "length") + test.eq(#result.result.tool_calls, 1) + local payload = path.replay(stored_rows(result)) + assert_order(assistant_order(path, payload), { "s1", "A", "s2", "c1" }) + test.eq(payload.messages[#payload.messages].role, "user") end) + for _, layout in ipairs({ false, true }) do + it("drops an empty thinking signature before text in " .. (layout and "a layout" or "plain order") .. ": " .. path.name, function() + local unsigned = { type = "thinking", thinking = "unfinished", signature = "" } + local shape = layout and { "before ", unsigned, "A" } or { unsigned, "A" } + local result: any = path.extract(shape, false, "max_tokens") + test.eq(result.metadata.thinking_blocks[1].signature, "") + test.eq(result.metadata.content_positions ~= nil, layout) + local payload = path.replay(rows_without_calls(result)) + assert_order(assistant_order(path, payload), layout and { "before ", "A" } or { "A" }) + end) + end + + if path.streaming then + it("drops an unsigned block cut mid-thinking after text: " .. path.name, function() + local result: any = path.extract({ "A", { type = "thinking", thinking = "unfinished", signature = "" } }, true) + test.eq(result.finish_reason, "length") + test.eq(#result.result.tool_calls, 0) + test.eq(result.metadata.thinking_blocks[1].thinking, "unfinished") + test.eq(result.metadata.thinking_blocks[1].signature, "") + local payload = path.replay(rows_without_calls(result)) + assert_order(assistant_order(path, payload), { "A" }) + end) + end + it("finds a call rendered after a result of the same response: " .. path.name, function() local result: any = path.extract({ "s1", "A", { "c1" }, "s2", { "c2" } }) local rows = stored_rows(result) From ec08ea59db2ca3ce6e6f02dd2f08f5ecbcc7562d Mon Sep 17 00:00:00 2001 From: KickSide Date: Wed, 4 Nov 2026 12:36:19 -0500 Subject: [PATCH 33/37] fix(llm,agent): keep emitted turns valid and cancel every call before listeners run A turn the drivers emit never ends in thinking, whatever the row shape. Stale thinking scans treat cache markers and hoisted system messages as part of the assistant turn. The prefix fingerprint separates arrays from objects. Stop cancels all unfinished calls before any result listener runs. --- src/agent/src/canonical.lua | 8 ++- src/agent/src/tools/caller.lua | 27 ++++++---- src/agent/test/replay_safety_test.lua | 27 ++++++++++ src/agent/test/tools/caller_test.lua | 57 +++++++++++++++++++++ src/llm/src/README.md | 4 +- src/llm/src/bedrock/mapper.lua | 2 +- src/llm/src/claude/mapper.lua | 2 +- src/llm/src/prompt.lua | 15 ++++-- src/llm/src/util/thinking.lua | 71 ++++++++++++--------------- src/llm/test/prompt_test.lua | 48 ++++++++++++++++++ src/llm/test/replay_safety_test.lua | 46 +++++++++++++++-- 11 files changed, 244 insertions(+), 63 deletions(-) diff --git a/src/agent/src/canonical.lua b/src/agent/src/canonical.lua index 12074886..76f39c95 100644 --- a/src/agent/src/canonical.lua +++ b/src/agent/src/canonical.lua @@ -1,13 +1,17 @@ local json = require("json") -- Serializes a value with object keys in sorted order, so equal values always --- produce equal text. Array indices are written as keys, which keeps sparse --- tables unambiguous. +-- produce equal text. Tables keep the array/object shape used by the wire encoder. local canonical = {} function canonical.encode(value: any): string if type(value) ~= "table" then return json.encode(value) or "null" end local keys, out = {}, {} + local encoded = json.encode(value) + if encoded and encoded:sub(1, 1) == "[" then + for _, child in ipairs(value) do out[#out+1] = canonical.encode(child) end + return "[" .. table.concat(out, ",") .. "]" + end for key in pairs(value) do keys[#keys+1] = tostring(key) end table.sort(keys) for _, key in ipairs(keys) do diff --git a/src/agent/src/tools/caller.lua b/src/agent/src/tools/caller.lua index 14659d3e..02670ee5 100644 --- a/src/agent/src/tools/caller.lua +++ b/src/agent/src/tools/caller.lua @@ -775,20 +775,21 @@ local function cancel_pending(pending: {[any]: PendingCall}) end -- Records every pending call: responses that are already complete keep their --- real outcome, the rest are cancelled. +-- real outcome, the rest are cancelled before any listener runs. local function settle_after_stop(run: BatchRun, pending: {[any]: PendingCall}) + local settled = {} for call_id, call in pairs(pending) do - if run.error then break end pending[call_id] = nil local completed, ok = poll_response(call) - if completed then - record_result(run, call_id, command_result(call, ok)) - else - call.command:cancel() - record_result(run, call_id, cancelled_result(call.tool_call)) - end + if not completed then call.command:cancel() end + settled[call_id] = { call = call, completed = completed, ok = ok } + end + for call_id, outcome in pairs(settled) do + if run.error then break end + local entry = outcome.completed and command_result(outcome.call, outcome.ok) + or cancelled_result(outcome.call.tool_call) + record_result(run, call_id, entry) end - cancel_pending(pending) end -- Records pending calls in completion order. Returns true when cancellation was observed. @@ -887,7 +888,13 @@ local function execute_parallel(self: any, run: BatchRun, context: table?, valid end end - if entry then record_result(run, call_id, entry) end + if entry then + if cancelled or cancellation_requested(cancel) then + cancelled = true + settle_after_stop(run, pending) + end + if not run.error then record_result(run, call_id, entry) end + end if run.error then cancel_pending(pending) return diff --git a/src/agent/test/replay_safety_test.lua b/src/agent/test/replay_safety_test.lua index f3292519..481a6815 100644 --- a/src/agent/test/replay_safety_test.lua +++ b/src/agent/test/replay_safety_test.lua @@ -388,6 +388,33 @@ local function define_tests() return assert(agent.new(compiled):step(builder, {})).prefix_fingerprint end + it("distinguishes schema arrays from objects while keeping equal schemas stable", function() + local function schema_fingerprint(schema) + local sent = install({ id = "claude_route", provider_model = "wire" }) + local compiled = spec("agent_b", 0) + compiled.tools.lookup.schema = schema + local builder = prompt.new() + builder:add_user("q") + local result = assert(agent.new(compiled):step(builder, {})) + return result.prefix_fingerprint, sent[1].payload.tools[1].input_schema + end + local array = { type = "object", properties = {}, enum = { "x" } } + local object = { type = "object", properties = {}, enum = { ["1"] = "x" } } + local array_fingerprint, array_wire = schema_fingerprint(array) + local object_fingerprint, object_wire = schema_fingerprint(object) + test.eq(json.encode(array_wire.enum), '["x"]') + test.eq(json.encode(object_wire.enum), '{"1":"x"}') + test.is_false(array_fingerprint == object_fingerprint, "different wire schemas need different fingerprints") + local reordered = {} + reordered.enum = { "x" } + reordered.properties = {} + reordered.type = "object" + test.eq(schema_fingerprint(reordered), array_fingerprint, "key insertion order is irrelevant") + test.eq(schema_fingerprint({ type = "object", properties = {}, enum = { "x" } }), array_fingerprint, + "equal schemas have equal fingerprints") + test.eq(schema_fingerprint({ enum = { ["1"] = "x" }, properties = {}, type = "object" }), object_fingerprint) + end) + it("ignores meta.claude without a type, which never reaches the wire", function() test.eq(fingerprint({ claude = { display_width_px = 10 } }), fingerprint({})) end) diff --git a/src/agent/test/tools/caller_test.lua b/src/agent/test/tools/caller_test.lua index 09f200b1..48c7568a 100644 --- a/src/agent/test/tools/caller_test.lua +++ b/src/agent/test/tools/caller_test.lua @@ -1394,6 +1394,63 @@ local function define_tests() test.eq(#cancelled_commands, 1) end) + for _, phase in ipairs({ "collection", "launch" }) do + it("cancels every pending call before a blocking stop listener during " .. phase, function() + local cancel = channel.new() + local listening, release, done = channel.new(1), channel.new(), channel.new(1) + local responses = { channel.new(1), channel.new(1) } + tool_results["test:pending_a"] = { response = responses[1], result = "committed a" } + tool_results["test:pending_b"] = { response = responses[2], execute = function() + cancel:close() + return "committed b" + end } + local calls = { tool("a", "test:pending_a"), tool("b", "test:pending_b") } + if phase == "launch" then table.insert(calls, tool("c", "test:calculator")) end + local notes = recording_listener() + local caller = tool_caller.new():set_strategy("parallel"):set_result_listener(function(call_id, result) + notes.listener(call_id, result) + if #notes.order == 1 then + listening:send(true) + release:receive() + end + end) + coroutine.spawn(function() + local results, err = caller:execute({}, calls, cancel) + done:send({ results = results, error = err }) + end) + local guard = time.timer("2s") + local first = channel.select({ listening:case_receive(), guard:channel():case_receive() }) + local cancelled_before_listener = #cancelled_commands + local effects = 0 + for index, registry_id in ipairs({ "test:pending_a", "test:pending_b" }) do + local cancelled = false + for _, id in ipairs(cancelled_commands) do + if id == registry_id then cancelled = true end + end + if not cancelled then + effects = effects + 1 + local response = assert(responses[index]) + response:send(true) + end + end + release:close() + local finished = channel.select({ done:case_receive(), guard:channel():case_receive() }) + guard:stop() + test.eq(first.channel, listening) + test.eq(finished.channel, done) + test.eq(cancelled_before_listener, 2, "all pending commands are cancelled before the first listener") + test.eq(effects, 0, "no pending tool commits while the listener blocks") + test.is_nil(finished.value.error) + test.eq(#notes.order, #calls) + for index = 1, #calls do + test.eq(notes.counts[index], 1, "listener fires once per call") + test.is_true(finished.value.results[index].cancelled) + test.is_nil(finished.value.results[index].result) + test.is_true(rawequal(notes.results[index], finished.value.results[index])) + end + end) + end + it("fails the batch on a listener error and stops the remaining calls", function() local failures = { returned = function() return nil, "store unavailable" end, diff --git a/src/llm/src/README.md b/src/llm/src/README.md index aeb8d5ad..2271d0f4 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -133,7 +133,7 @@ local response, err = llm.generate("Hello", { ### Thinking block binding -Claude and Bedrock store each response's `thinking` blocks as `{ type, thinking, signature }` and `redacted_thinking` blocks as `{ type, data }` in `metadata.thinking_blocks`, in response order (Bedrock: replayed as `reasoningContent.reasoningText` and `reasoningContent.redactedContent`). A response that is thinking, then at most one text block, then tool calls replays in that order with no record. Any other response, such as thinking between text and tool calls, also carries `metadata.content_positions` and a `position` on each thinking block; a host that stores `content_positions` next to `thinking_blocks` on the assistant row (or the first function call row of the response when it has no text) gets the response order back, with each visible text block separate, as long as the calls of that response are consecutive function call rows, which consolidate into one assistant turn. A function result between those rows splits the turn, and the order is then not rebuilt across the split. Whitespace-only text blocks of a recorded response are not rendered: the API compares neither those nor the outer whitespace of a text block, so leaving them out changes no prefix. Rows without `content_positions`, including every row stored before it existed, replay as all the thinking blocks of the row, then its text as one block, then its calls; so do rows whose text, apart from outer whitespace, no longer equals the recorded text. A recorded call the conversation no longer holds leaves the layout, and the thinking behind the last remaining text or call is not replayed: it led to that call, and an assistant turn cannot end in it. `thinking.discard_calls` applies the same rule to a response's metadata for a caller that discards its calls. Positions never reach the wire, and other drivers ignore them. Claude replays each block unchanged except for fields an earlier stored shape carried that its type does not accept (`data` on `thinking`, `thinking` and `signature` on `redacted_thinking`). A Claude thinking block is bound to the conversation that produced it: the top-level system prompt, the tools, and every earlier message. After an agent handoff, a prompt or tool change, or a history edit, models that enforce the check reject a request that replays such a block with a 400. +Claude and Bedrock store each response's `thinking` blocks as `{ type, thinking, signature }` and `redacted_thinking` blocks as `{ type, data }` in `metadata.thinking_blocks`, in response order (Bedrock: replayed as `reasoningContent.reasoningText` and `reasoningContent.redactedContent`). A response that is thinking, then at most one text block, then tool calls replays in that order with no record. Any other response, such as thinking between text and tool calls, also carries `metadata.content_positions` and a `position` on each thinking block; a host that stores `content_positions` next to `thinking_blocks` on the assistant row (or the first function call row of the response when it has no text) gets the response order back, with each visible text block separate, as long as the calls of that response are consecutive function call rows, which consolidate into one assistant turn. A function result between those rows splits the turn, and the order is then not rebuilt across the split. Each emitted assistant turn drops trailing thinking, including turns assembled from rows without positions. Whitespace-only text blocks of a recorded response are not rendered: the API compares neither those nor the outer whitespace of a text block, so leaving them out changes no prefix. Rows without `content_positions`, including every row stored before it existed, replay as all the thinking blocks of the row, then its text as one block, then its calls; so do rows whose text, apart from outer whitespace, no longer equals the recorded text. A recorded call the conversation no longer holds leaves the layout, and the thinking behind the last remaining text or call is not replayed: it led to that call, and an assistant turn cannot end in it. `thinking.discard_calls` applies the same rule to a response's metadata for a caller that discards its calls. Positions never reach the wire, and other drivers ignore them. Claude replays each block unchanged except for fields an earlier stored shape carried that its type does not accept (`data` on `thinking`, `thinking` and `signature` on `redacted_thinking`). A Claude thinking block is bound to the conversation that produced it: the top-level system prompt, the tools, and every earlier message. After an agent handoff, a prompt or tool change, or a history edit, models that enforce the check reject a request that replays such a block with a 400. When a request replays at least one thinking block and sends a thinking object of type `adaptive` or `enabled` (a positive thinking_effort, or `thinking = always` at any effort), the driver sets `thinking.block_binding.prefix_mismatch_behavior = "drop_block"` and adds the `thinking-binding-controls-2026-08-01` beta. The API then drops the first mismatched block and every later thinking block instead of rejecting the request. The drop applies to that request only and the history keeps the blocks, so after one change to the system prompt, tools or an earlier message, every later request again drops every thinking block from the first one bound to the old prefix onward, until the history before the change no longer carries thinking blocks; such conversations continue without that earlier reasoning, and `metadata.input_transformations` shows the drops on every response. Setting `prefix_mismatch_behavior` also opts the request into enforcement: on models that enforce the check only for organizations created on or after 2026-08-31, an older organization's mismatched blocks reached the model before and are now dropped. A `block_binding` the thinking object already carries is kept as set, and any request whose thinking object carries `block_binding` sends the beta. The Claude driver sends one `anthropic-beta` header: the configured `beta_features`, then the betas of a configured `anthropic-beta` context header (any letter case), then request betas, each named once. Bedrock appends the beta to the `anthropic_beta` list of `additionalModelRequestFields` and `/input_transformations` to `additionalModelResponseFieldPaths`, each named once. Plain generation and structured output behave the same. The driver decides the binding after merging `options.request`, from the thinking object as sent: a `request.thinking.type = "between_tools"` gets no binding and no beta, and a `request.thinking.block_binding` is kept as set and sends the beta. @@ -289,7 +289,7 @@ end) `mark_stale_thinking` takes the same predicate and sets `metadata.stale_thinking` on copies of every message that carries blocks, up to and including the last one the predicate rejects. Call it after `drop_stale_thinking` to mark the current round's stale blocks, which that call keeps; the driver removes marked blocks when the request cannot bind them. -An empty `thinking_blocks` list counts as no blocks; messages without blocks are not consulted. The current round, the last assistant turn while the request continues it, is never stripped and never consulted: the provider requires the final assistant turn of a tool round to keep its thinking. The round continues while only its function results and user content that joins the tool-result turn follow it (a steering message, a developer or positional note, or the image message `get_messages` inserts after a tool result with `_images`). User content after the turn with no function result in between ends it: a user message, a developer message or a positional system note, which drivers render as a new user turn. Ordinary system messages, which drivers hoist into the top-level system, and cache markers do not. Stale blocks in the current round are left to the provider's own drop-on-mismatch handling, or to the driver when they are marked and the request cannot bind them; once a later assistant turn exists, or the turn is answered without tools and a user message follows, it is strippable like any other. An assistant turn is the run of consecutive assistant and function call messages. If a turn whose blocks would be removed still has a function call without a result (a function call message or a `function_call` content part), the call fails and leaves the history unchanged: the provider needs those blocks to continue the tool round. +An empty `thinking_blocks` list counts as no blocks; messages without blocks are not consulted. The current round, the last assistant turn while the request continues it, is never stripped and never consulted: the provider requires the final assistant turn of a tool round to keep its thinking. The round continues while only its function results and user content that joins the tool-result turn follow it (a steering message, a developer or positional note, or the image message `get_messages` inserts after a tool result with `_images`). User content after the turn with no function result in between ends it: a user message, a developer message or a positional system note, which drivers render as a new user turn. Ordinary system messages, which drivers hoist into the top-level system, and cache markers do not. Stale blocks in the current round are left to the provider's own drop-on-mismatch handling, or to the driver when they are marked and the request cannot bind them; once a later assistant turn exists, or the turn is answered without tools and a user message follows, it is strippable like any other. An assistant turn is the run of assistant and function call messages, with cache markers and hoisted system messages transparent to the turn. If a turn whose blocks would be removed still has a function call without a result (a function call message or a `function_call` content part), the call fails and leaves the history unchanged: the provider needs those blocks to continue the tool round. ### Builder Operations diff --git a/src/llm/src/bedrock/mapper.lua b/src/llm/src/bedrock/mapper.lua index b57e2555..3cb237b7 100644 --- a/src/llm/src/bedrock/mapper.lua +++ b/src/llm/src/bedrock/mapper.lua @@ -288,7 +288,7 @@ function mapper.map_messages(contract_messages) end contract_messages = prompt.without_blank_text(contract_messages) - local order = thinking.new_turn_order(contract_messages) + local order = thinking.new_turn_order() local translate_id = tool_ids.mapper(contract_messages, { pattern = tool_ids.NATIVE_PATTERN, max_length = MAX_TOOL_ID_LENGTH }) local function tool_use_id(id) diff --git a/src/llm/src/claude/mapper.lua b/src/llm/src/claude/mapper.lua index 95a1c129..f3641a0d 100644 --- a/src/llm/src/claude/mapper.lua +++ b/src/llm/src/claude/mapper.lua @@ -366,7 +366,7 @@ function mapper.map_messages(contract_messages) contract_messages = prompt.without_blank_text(contract_messages) local tool_id = tool_ids.mapper(contract_messages, { pattern = tool_ids.NATIVE_PATTERN }) - local order = thinking.new_turn_order(contract_messages) + local order = thinking.new_turn_order() local stale_thinking = {} local function add_thinking_blocks(content_blocks, msg) diff --git a/src/llm/src/prompt.lua b/src/llm/src/prompt.lua index 032d96d8..9609a453 100644 --- a/src/llm/src/prompt.lua +++ b/src/llm/src/prompt.lua @@ -202,6 +202,11 @@ local function in_assistant_turn(msg: Message): boolean return msg.role == prompt.ROLE.ASSISTANT or msg.role == prompt.ROLE.FUNCTION_CALL end +local function transparent_to_assistant_turn(msg: Message): boolean + return msg.role == prompt.ROLE.CACHE_MARKER + or (msg.role == prompt.ROLE.SYSTEM and not prompt.is_positional_note(msg)) +end + local function carries_thinking(msg: Message): boolean if not in_assistant_turn(msg) or type(msg.metadata) ~= "table" then return false end local blocks = msg.metadata.thinking_blocks @@ -472,10 +477,10 @@ function prompt.new(messages: {Message}?) for index, msg in ipairs(messages) do if in_assistant_turn(msg) then turn_start = turn_start or index - else + elseif not transparent_to_assistant_turn(msg) then turn_start = nil end - if turn_start then + if turn_start and in_assistant_turn(msg) then local ids: {any} = {} if msg.role == prompt.ROLE.FUNCTION_CALL then table.insert(ids, { @@ -522,8 +527,8 @@ function prompt.new(messages: {Message}?) index = index - 1 end if index >= 1 and not (seen_user and not seen_result) then - while index >= 1 and in_assistant_turn(messages[index]) do - current_round = index + while index >= 1 and (in_assistant_turn(messages[index]) or transparent_to_assistant_turn(messages[index])) do + if in_assistant_turn(messages[index]) then current_round = index end index = index - 1 end end @@ -545,7 +550,7 @@ function prompt.new(messages: {Message}?) local msg = messages[index] if in_assistant_turn(msg) then turn_start = turn_start or index - else + elseif not transparent_to_assistant_turn(msg) then turn_start = nil end if carries_thinking(msg) then diff --git a/src/llm/src/util/thinking.lua b/src/llm/src/util/thinking.lua index 50568050..ae91303a 100644 --- a/src/llm/src/util/thinking.lua +++ b/src/llm/src/util/thinking.lua @@ -149,9 +149,8 @@ end -- text the layout recorded: the row is what the host stands behind. Providers -- compare neither whitespace-only text blocks nor the outer whitespace of a block, so -- the layout holds only visible text, and whitespace a host stripped from the row's --- ends is not an edit. A call the conversation no longer holds leaves the layout, and --- thinking_limit bounds the thinking that can still be replayed. -local function read_layout(msg: any, calls: {[any]: boolean}): any? +-- ends is not an edit. +local function read_layout(msg: any): any? local positions = type(msg.metadata) == "table" and msg.metadata.content_positions or nil if type(positions) ~= "table" then return nil @@ -165,53 +164,30 @@ local function read_layout(msg: any, calls: {[any]: boolean}): any? end text = text .. entry.text elseif entry.type == "tool_use" then - if calls[entry.id] then - layout.calls[entry.id] = entry.position - end + layout.calls[entry.id] = entry.position end end - layout.thinking_limit = last_standing(positions, calls) if without_outer_whitespace(text) ~= without_outer_whitespace(row_text(msg.content)) then return nil end return layout end +local function is_wire_thinking(block: any): boolean + return block.type == "thinking" or block.type == "redacted_thinking" or block.reasoningContent ~= nil +end + -- Tracks the wire blocks of assistant rows against their recorded layouts and -- rebuilds response order once the rows are consolidated into one turn. -function thinking.new_turn_order(messages: {any}) +function thinking.new_turn_order() local placed: {[any]: any} = {} + local attached: {[any]: boolean} = {} local active: any = nil local order = {} - local calls_by_row: {[any]: {[any]: boolean}} = {} - local calls: {[any]: boolean} = {} - local after_result = false - for _, msg in ipairs(messages) do - if msg.role == prompt.ROLE.ASSISTANT or msg.role == prompt.ROLE.FUNCTION_CALL then - if after_result and msg.role == prompt.ROLE.ASSISTANT then calls = {} end - after_result = false - calls_by_row[msg] = calls - if msg.function_call then - calls[msg.function_call.id] = true - elseif type(msg.content) == "table" then - for _, part in ipairs(msg.content) do - if part.type == "function_call" then calls[part.id] = true end - end - end - elseif msg.role == prompt.ROLE.FUNCTION_RESULT then - -- Calls after a result can belong to the same response; a new - -- assistant row starts the next turn. - after_result = true - elseif msg.role ~= prompt.ROLE.CACHE_MARKER then - calls = {} - after_result = false - end - end - -- Returns the row's own layout, whose text blocks replace the row's text. function order.enter_row(msg: any): any? - local layout = read_layout(msg, calls_by_row[msg] or {}) + local layout = read_layout(msg) if layout then active = layout end return layout end @@ -224,8 +200,7 @@ function thinking.new_turn_order(messages: {any}) if stored_block.type == "thinking" and (stored_block.signature or "") == "" then return false end - local limit = active and active.thinking_limit - return not (limit and stored_block.position and stored_block.position > limit) + return true end function order.place_thinking(wire_block: any, stored_block: any) @@ -249,6 +224,7 @@ function thinking.new_turn_order(messages: {any}) -- A block that belongs right behind `anchor`, wherever the anchor ends up. function order.place_behind(wire_block: any, anchor: any) + attached[wire_block] = true local place = placed[anchor] if place then placed[wire_block] = { layout = place.layout, position = place.position + 0.5 } @@ -258,8 +234,12 @@ function thinking.new_turn_order(messages: {any}) -- The content with each run of blocks from one layout in response order. Blocks -- with no recorded place keep theirs, and they end a run. function order.rebuild(content: {any}): {any} - if next(placed) == nil then - return content + local limits: {[any]: number} = {} + for _, block in ipairs(content) do + local place = placed[block] + if place and not is_wire_thinking(block) and not attached[block] then + limits[place.layout] = math.max(limits[place.layout] or 0, place.position) + end end local rebuilt: {any} = {} local index = 1 @@ -275,9 +255,22 @@ function thinking.new_turn_order(messages: {any}) index = index + 1 end table.sort(run, function(a, b) return placed[a].position < placed[b].position end) - for _, block in ipairs(run) do table.insert(rebuilt, block) end + for _, block in ipairs(run) do + if not is_wire_thinking(block) or placed[block].position <= (limits[place.layout] or 0) then + table.insert(rebuilt, block) + end + end end end + -- Each emitted turn ends on visible content, even when persisted rows + -- split a response or carry thinking without recorded positions. + local last = #rebuilt + while last > 0 and (is_wire_thinking(rebuilt[last]) or attached[rebuilt[last]]) do + last = last - 1 + end + for index = #rebuilt, last + 1, -1 do + if is_wire_thinking(rebuilt[index]) then table.remove(rebuilt, index) end + end return rebuilt end diff --git a/src/llm/test/prompt_test.lua b/src/llm/test/prompt_test.lua index 2d8fe6b3..84938aad 100644 --- a/src/llm/test/prompt_test.lua +++ b/src/llm/test/prompt_test.lua @@ -190,6 +190,54 @@ local function define_tests() return builder end + for _, role in ipairs({ "cache_marker", "system" }) do + local function transparent_row() + return { role = role, content = { prompt.text("policy") }, marker_id = "m" } + end + + it("keeps and marks the current round across " .. role, function() + for _, placement in ipairs({ 3, 4 }) do + local builder = prompt.new() + builder:add_user("q") + builder:add_assistant("A", { thinking_blocks = block("round") }) + builder:add_function_call("lookup", "{}", "call_1") + builder:add_function_result("lookup", "found", "call_1") + table.insert(builder.messages, placement, transparent_row()) + local consulted = 0 + local dropped, err = builder:drop_stale_thinking(function() + consulted = consulted + 1 + return false + end) + test.is_nil(err) + test.eq(dropped, builder) + test.eq(thinking_of(builder.messages, 2), "round", "current thinking survives " .. role) + test.eq(consulted, 0) + builder:mark_stale_thinking(function() return false end) + test.is_true(builder.messages[2].metadata.stale_thinking) + test.eq(thinking_of(builder.messages, 2), "round") + end + end) + + for _, carrier in ipairs({ "before", "after" }) do + it("protects an open call with thinking " .. carrier .. " " .. role, function() + local builder = prompt.new() + builder:add_user("q") + builder:add_assistant("A", carrier == "before" and { thinking_blocks = block("round") } or nil) + builder:add_function_call("lookup", "{}", "call_open") + builder:add_function_call("fetch", "{}", "call_done") + if carrier == "after" then builder.messages[4].metadata = { thinking_blocks = block("round") } end + builder:add_function_result("fetch", "found", "call_done") + builder:add_assistant("later") + builder:add_user("next") + table.insert(builder.messages, carrier == "before" and 3 or 4, transparent_row()) + local dropped, err = builder:drop_stale_thinking(function() return false end) + test.is_nil(dropped, "open call across " .. role) + test.contains(tostring(err), "call_open") + test.eq(thinking_of(builder.messages, carrier == "before" and 2 or 5), "round") + end) + end + end + it("keeps the round when a user message joins its tool-result turn", function() local builder = current_round() builder:add_user("Use metric units") diff --git a/src/llm/test/replay_safety_test.lua b/src/llm/test/replay_safety_test.lua index 4a101ee3..e638b9d3 100644 --- a/src/llm/test/replay_safety_test.lua +++ b/src/llm/test/replay_safety_test.lua @@ -581,6 +581,9 @@ local function define_tests() it("keeps thinking in front of the last visible text of a stored row: " .. path.name, function() local result: any = path.extract({ "s1", "A", "s2", { "c1" } }) assert_order(assistant_order(path, path.replay(rows_without_calls(result))), { "s1", "A" }) + local rows = rows_without_calls(result) + table.insert(rows, 3, { role = "assistant", content = text("B") }) + assert_order(assistant_order(path, path.replay(rows)), { "s1", "A", "B" }) end) it("drops thinking behind a discarded call when the host dropped the text too: " .. path.name, function() @@ -675,13 +678,50 @@ local function define_tests() end) end - it("finds a call rendered after a result of the same response: " .. path.name, function() + it("ends each split response turn with text or a call: " .. path.name, function() local result: any = path.extract({ "s1", "A", { "c1" }, "s2", { "c2" } }) local rows = stored_rows(result) local paired = { rows[1], rows[2], rows[3], rows[5], rows[4], rows[6] } local payload = path.replay(paired) - local signatures = table.concat(assistant_order(path, payload), "|") - test.contains(signatures, "s2") + local blocks = path.order == claude_order and claude_blocks or bedrock_blocks + test.eq(#payload.messages, 5) + for index, role in ipairs({ "user", "assistant", "user", "assistant", "user" }) do + test.eq(payload.messages[index].role, role) + end + test.eq(json.encode(payload.messages[2].content), json.encode(blocks({ "s1", "A", { "c1" } })), + "first emitted turn") + test.eq(json.encode(payload.messages[4].content), json.encode(blocks({ { "c2" } })), + "second emitted turn") + end) + + it("ends a turn before thinking-only rows without positions: " .. path.name, function() + for _, redacted in ipairs({ false, true }) do + for _, cached in ipairs({ false, true }) do + local result: any = path.extract({ "s1", "A", { "c1" } }) + local rows = stored_rows(result) + table.insert(rows, 4, { role = "assistant", content = {}, metadata = { + thinking_blocks = { redacted and { type = "redacted_thinking", data = "opaque" } or signed("s2") } } }) + if cached then table.insert(rows, 5, { role = "cache_marker", marker_id = "tail" }) end + local payload = path.replay(rows) + local blocks = path.order == claude_order and claude_blocks or bedrock_blocks + local expected = blocks({ "s1", "A", { "c1" } }) + if cached then + if path.order == claude_order then + expected[3].cache_control = { type = "ephemeral" } + else + table.insert(expected, { cachePoint = { type = "default" } }) + end + end + test.eq(#payload.messages, 3) + test.eq(json.encode(payload.messages[2].content), json.encode(expected), "emitted turn without positions") + if path.order == claude_order then + rows[4].content = rows[4].metadata.thinking_blocks + rows[4].metadata = nil + payload = path.replay(rows) + test.eq(json.encode(payload.messages[2].content), json.encode(expected), "emitted turn with inline thinking") + end + end + end end) end From dd909e50c369f5e383c93baa966969a5b5e66800 Mon Sep 17 00:00:00 2001 From: KickSide Date: Wed, 4 Nov 2026 13:01:18 -0500 Subject: [PATCH 34/37] fix(llm): keep thinking valid after discarded calls and hoisted rows, and keep notes in tool rounds --- src/agent/test/replay_safety_test.lua | 11 ++++++ src/llm/src/bedrock/mapper.lua | 3 +- src/llm/src/claude/mapper.lua | 3 +- src/llm/src/openai_compat/mapper.lua | 55 ++++++++++++++------------- src/llm/src/prompt.lua | 8 ++-- src/llm/src/util/thinking.lua | 42 +++++++++++--------- src/llm/test/replay_safety_test.lua | 54 ++++++++++++++++++++++++++ 7 files changed, 127 insertions(+), 49 deletions(-) diff --git a/src/agent/test/replay_safety_test.lua b/src/agent/test/replay_safety_test.lua index 481a6815..9c1faa0e 100644 --- a/src/agent/test/replay_safety_test.lua +++ b/src/agent/test/replay_safety_test.lua @@ -335,6 +335,17 @@ local function define_tests() end for _, path in ipairs(TRUNCATED_PATHS) do + it("drops thinking after the first discarded call even before visible text: " .. path.name, function() + local order, response = replay_after_truncation(path, { "s1", { "c1" }, "s2", "A", { "c2" } }) + test.eq(table.concat(order, "|"), "s1|A") + test.eq(response.result, "A") + test.eq(#response.metadata.thinking_blocks, 1) + test.eq(response.metadata.thinking_blocks[1].signature, "s1") + test.eq(#response.metadata.content_positions, 1) + test.eq(response.metadata.content_positions[1].type, "text") + test.eq(response.metadata.content_positions[1].text, "A") + end) + it("never replays thinking behind a discarded call: text, thinking, call: " .. path.name, function() local order, response = replay_after_truncation(path, { "A", "s1", { "c1" } }) test.eq(table.concat(order, "|"), "A") diff --git a/src/llm/src/bedrock/mapper.lua b/src/llm/src/bedrock/mapper.lua index 3cb237b7..659bfa8c 100644 --- a/src/llm/src/bedrock/mapper.lua +++ b/src/llm/src/bedrock/mapper.lua @@ -319,7 +319,8 @@ function mapper.map_messages(contract_messages) local message_cache_positions = {} for _, msg in ipairs(contract_messages) do - if msg.role ~= prompt.ROLE.ASSISTANT and msg.role ~= prompt.ROLE.FUNCTION_CALL and msg.role ~= "cache_marker" then + if msg.role ~= prompt.ROLE.ASSISTANT and msg.role ~= prompt.ROLE.FUNCTION_CALL + and not prompt.transparent_to_assistant_turn(msg) then order.leave_turn() end if msg.role == prompt.ROLE.SYSTEM and not prompt.is_positional_note(msg) then diff --git a/src/llm/src/claude/mapper.lua b/src/llm/src/claude/mapper.lua index f3641a0d..64820999 100644 --- a/src/llm/src/claude/mapper.lua +++ b/src/llm/src/claude/mapper.lua @@ -392,7 +392,8 @@ function mapper.map_messages(contract_messages) local in_system_phase = true for i, msg in ipairs(contract_messages) do - if msg.role ~= prompt.ROLE.ASSISTANT and msg.role ~= prompt.ROLE.FUNCTION_CALL and msg.role ~= "cache_marker" then + if msg.role ~= prompt.ROLE.ASSISTANT and msg.role ~= prompt.ROLE.FUNCTION_CALL + and not prompt.transparent_to_assistant_turn(msg) then order.leave_turn() end if prompt.is_positional_note(msg) or msg.role == prompt.ROLE.DEVELOPER then diff --git a/src/llm/src/openai_compat/mapper.lua b/src/llm/src/openai_compat/mapper.lua index fa7c065d..fce276a6 100644 --- a/src/llm/src/openai_compat/mapper.lua +++ b/src/llm/src/openai_compat/mapper.lua @@ -157,10 +157,28 @@ end -- MESSAGE MAPPING FUNCTIONS +local function map_text_message(msg) + local content = msg.content + if msg.role == "developer" then + content = "" + if type(msg.content) == "string" then + content = msg.content + elseif type(msg.content) == "table" then + for _, part in ipairs(msg.content) do + if part.type == "text" then content = content .. part.text end + end + end + end + return { + role = msg.role == "developer" and "system" or msg.role, + content = normalize_content(content) + } +end + function openai_mapper.map_messages(contract_messages, options) options = options or {} contract_messages = prompt.without_blank_text(contract_messages) - local processed_messages = {} + local processed_messages: {any} = {} local i = 1 local is_claude = is_claude_model(options.model) local cache_positions = {} @@ -182,13 +200,8 @@ function openai_mapper.map_messages(contract_messages, options) end -- Skip cache marker message (don't add to processed_messages) i = i + 1 - elseif msg.role == "user" or msg.role == "system" then - -- Standard user/system messages - use structured content - local processed_msg = { - role = msg.role, - content = normalize_content(msg.content) - } - table.insert(processed_messages, processed_msg) + elseif msg.role == "user" or msg.role == "system" or msg.role == "developer" then + table.insert(processed_messages, map_text_message(msg)) i = i + 1 elseif msg.role == "assistant" then -- Assistant messages - ALWAYS use simple string content for OpenRouter compatibility @@ -209,6 +222,7 @@ function openai_mapper.map_messages(contract_messages, options) -- The prompt builder interleaves them: call_A, result_A, call_B, result_B local function_calls_found = {} local function_results_found = {} + local deferred_messages = {} local open_calls = 0 local answered = {} local scan_end = i - 1 @@ -240,7 +254,11 @@ function openai_mapper.map_messages(contract_messages, options) -- With every call answered the round is complete: a later call is -- the next round, and this message stays in place between them. break + elseif current_msg.role == "system" or current_msg.role == "developer" then + table.insert(deferred_messages, map_text_message(current_msg)) + scan_end = j end + if open_calls == 0 and #deferred_messages > 0 then break end j = j + 1 end @@ -302,6 +320,9 @@ function openai_mapper.map_messages(contract_messages, options) end table.insert(processed_messages, tool_msg) end + for _, deferred in ipairs(deferred_messages) do + table.insert(processed_messages, deferred) + end -- Advance past all scanned messages i = scan_end + 1 @@ -339,24 +360,6 @@ function openai_mapper.map_messages(contract_messages, options) table.insert(processed_messages, tool_msg) i = i + 1 - elseif msg.role == "developer" then - -- Convert developer messages to system messages - local content = "" - if type(msg.content) == "string" then - content = msg.content - elseif type(msg.content) == "table" then - for _, part in ipairs(msg.content) do - if part.type == "text" then - content = content .. part.text - end - end - end - - table.insert(processed_messages, { - role = "system", - content = normalize_content(content) - }) - i = i + 1 elseif msg.role == "function_call" then -- Standalone function_calls without preceding assistant message if not msg.function_call or not msg.function_call.id then diff --git a/src/llm/src/prompt.lua b/src/llm/src/prompt.lua index 9609a453..6d01df1e 100644 --- a/src/llm/src/prompt.lua +++ b/src/llm/src/prompt.lua @@ -202,7 +202,7 @@ local function in_assistant_turn(msg: Message): boolean return msg.role == prompt.ROLE.ASSISTANT or msg.role == prompt.ROLE.FUNCTION_CALL end -local function transparent_to_assistant_turn(msg: Message): boolean +function prompt.transparent_to_assistant_turn(msg: Message): boolean return msg.role == prompt.ROLE.CACHE_MARKER or (msg.role == prompt.ROLE.SYSTEM and not prompt.is_positional_note(msg)) end @@ -477,7 +477,7 @@ function prompt.new(messages: {Message}?) for index, msg in ipairs(messages) do if in_assistant_turn(msg) then turn_start = turn_start or index - elseif not transparent_to_assistant_turn(msg) then + elseif not prompt.transparent_to_assistant_turn(msg) then turn_start = nil end if turn_start and in_assistant_turn(msg) then @@ -527,7 +527,7 @@ function prompt.new(messages: {Message}?) index = index - 1 end if index >= 1 and not (seen_user and not seen_result) then - while index >= 1 and (in_assistant_turn(messages[index]) or transparent_to_assistant_turn(messages[index])) do + while index >= 1 and (in_assistant_turn(messages[index]) or prompt.transparent_to_assistant_turn(messages[index])) do if in_assistant_turn(messages[index]) then current_round = index end index = index - 1 end @@ -550,7 +550,7 @@ function prompt.new(messages: {Message}?) local msg = messages[index] if in_assistant_turn(msg) then turn_start = turn_start or index - elseif not transparent_to_assistant_turn(msg) then + elseif not prompt.transparent_to_assistant_turn(msg) then turn_start = nil end if carries_thinking(msg) then diff --git a/src/llm/src/util/thinking.lua b/src/llm/src/util/thinking.lua index ae91303a..6c5d4400 100644 --- a/src/llm/src/util/thinking.lua +++ b/src/llm/src/util/thinking.lua @@ -103,14 +103,11 @@ local function without_outer_whitespace(text: string): string return text:match("^%s*(.-)%s*$") :: string end --- The position of the last visible text or retained call, or 0 when neither remains. --- Thinking behind it cannot end a replayed assistant turn. -local function last_standing(positions: {any}, calls: {[any]: boolean}): number - local last = 0 - for _, entry in ipairs(positions) do - if (entry.type == "tool_use" and calls[entry.id]) - or (entry.type == "text" and not prompt.is_blank(entry.text)) then - last = math.max(last, entry.position) +-- Thinking at or after a missing call no longer has its signed prefix. +local function thinking_limit(last: number, calls: {[any]: number}, retained: {[any]: boolean}): number + for id, position in pairs(calls) do + if not retained[id] then + last = math.min(last, position) end end return last @@ -128,17 +125,22 @@ function thinking.discard_calls(metadata: any, text: any) return end - local limit = last_standing(positions, {}) - local kept_blocks = {} - for _, block in ipairs(type(blocks) == "table" and blocks or {}) do - if block.position and block.position < limit then - table.insert(kept_blocks, block) - end - end + local limit = 0 + local calls = {} local kept_positions = {} for _, entry in ipairs(positions) do if entry.type == "text" then table.insert(kept_positions, entry) + if not prompt.is_blank(entry.text) then limit = math.max(limit, entry.position) end + elseif entry.type == "tool_use" then + calls[entry.id] = entry.position + end + end + limit = thinking_limit(limit, calls, {}) + local kept_blocks = {} + for _, block in ipairs(type(blocks) == "table" and blocks or {}) do + if block.position and block.position < limit then + table.insert(kept_blocks, block) end end metadata.thinking_blocks = kept_blocks @@ -218,7 +220,7 @@ function thinking.new_turn_order() function order.place_call(wire_block: any, id: any) local position = active and active.calls[id] if position then - placed[wire_block] = { layout = active, position = position } + placed[wire_block] = { layout = active, position = position, call_id = id } end end @@ -235,12 +237,18 @@ function thinking.new_turn_order() -- with no recorded place keep theirs, and they end a run. function order.rebuild(content: {any}): {any} local limits: {[any]: number} = {} + local retained: {[any]: {[any]: boolean}} = {} for _, block in ipairs(content) do local place = placed[block] if place and not is_wire_thinking(block) and not attached[block] then limits[place.layout] = math.max(limits[place.layout] or 0, place.position) + retained[place.layout] = retained[place.layout] or {} + if place.call_id then retained[place.layout][place.call_id] = true end end end + for layout, last in pairs(limits) do + limits[layout] = thinking_limit(last, layout.calls, retained[layout] :: {[any]: boolean}) + end local rebuilt: {any} = {} local index = 1 while index <= #content do @@ -256,7 +264,7 @@ function thinking.new_turn_order() end table.sort(run, function(a, b) return placed[a].position < placed[b].position end) for _, block in ipairs(run) do - if not is_wire_thinking(block) or placed[block].position <= (limits[place.layout] or 0) then + if not is_wire_thinking(block) or placed[block].position < (limits[place.layout] or 0) then table.insert(rebuilt, block) end end diff --git a/src/llm/test/replay_safety_test.lua b/src/llm/test/replay_safety_test.lua index e638b9d3..eefd8081 100644 --- a/src/llm/test/replay_safety_test.lua +++ b/src/llm/test/replay_safety_test.lua @@ -547,6 +547,36 @@ local function define_tests() end for _, path in ipairs(EXTRACTION_PATHS) do + for _, keep_second in ipairs({ false, true }) do + it("drops thinking after a missing call before visible text with " .. (keep_second and "one call retained: " or "no calls retained: ") .. path.name, function() + local result: any = path.extract({ "s1", { "c1" }, "s2", "A", { "c2" } }) + local rows = rows_without_calls(result) + if keep_second then + local stored = stored_rows(result) + table.insert(rows, 3, stored[4]) + table.insert(rows, 4, stored[6]) + end + assert_order(assistant_order(path, path.replay(rows)), keep_second and { "s1", "A", "c2" } or { "s1", "A" }) + test.eq(#result.metadata.thinking_blocks, 2) + test.eq(#result.metadata.content_positions, 3) + end) + end + + it("keeps layouts across hoisted systems and ends them at positional notes: " .. path.name, function() + for _, role in ipairs({ "system", "developer" }) do + local rows = stored_rows(path.extract({ "A", "s1", { "c1" } })) + table.insert(rows, 3, { role = role, content = text("NOTE"), metadata = { preserve_position = true } }) + local payload = path.replay(rows) + assert_order(path.order(payload.messages[2]), { "A" }) + assert_order(path.order(payload.messages[4]), { "c1" }) + end + local rows = stored_rows(path.extract({ "A", "s1", { "c1" } })) + table.insert(rows, 3, { role = "system", content = text("SYSTEM") }) + local payload = path.replay(rows) + assert_order(path.order(payload.messages[2]), { "A", "s1", "c1" }) + test.eq(payload.system[1].text, "SYSTEM") + end) + it("drops thinking behind a missing call whose id belongs to an earlier turn: " .. path.name, function() local earlier = stored_rows(path.extract({ "before", "s1", { "c" } })) local before = path.replay(earlier) @@ -825,6 +855,30 @@ local function define_tests() end) describe("OpenAI-compatible tool rounds", function() + for _, role in ipairs({ "system", "developer" }) do + it("emits an intervening " .. role .. " note after all tool results", function() + local mapped: any = compat_mapper.map_messages({ + { role = "assistant", content = text("look") }, + { role = "function_call", content = {}, function_call = { id = "c1", name = "t", arguments = "{}" } }, + { role = "function_call", content = {}, function_call = { id = "c2", name = "t", arguments = "{}" } }, + { role = "function_result", name = "t", function_call_id = "c1", content = text("r1") }, + { role = role, content = text("NOTE"), metadata = { preserve_position = true } }, + { role = "function_result", name = "t", function_call_id = "c2", content = text("r2") }, + }, { model = "gpt" }) + local roles = {} + for _, message in ipairs(mapped) do table.insert(roles, message.role) end + assert_order(roles, { "assistant", "tool", "tool", "system" }) + test.eq(#mapped[1].tool_calls, 2) + test.eq(mapped[1].tool_calls[1].id, "c1") + test.eq(mapped[1].tool_calls[2].id, "c2") + test.eq(mapped[2].tool_call_id, "c1") + test.eq(mapped[2].content, "r1") + test.eq(mapped[3].tool_call_id, "c2") + test.eq(mapped[3].content, "r2") + test.eq(mapped[4].content[1].text, "NOTE") + end) + end + local function note_rows() return { { role = "user", content = text("q") }, From 291ec536b8b261c117f37d3387ce212f8ccad0a1 Mon Sep 17 00:00:00 2001 From: KickSide Date: Wed, 4 Nov 2026 13:39:30 -0500 Subject: [PATCH 35/37] fix(llm): a rejected layout does not inherit the previous response's layout --- src/llm/src/util/thinking.lua | 4 +++- src/llm/test/replay_safety_test.lua | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/llm/src/util/thinking.lua b/src/llm/src/util/thinking.lua index 6c5d4400..bd654b1c 100644 --- a/src/llm/src/util/thinking.lua +++ b/src/llm/src/util/thinking.lua @@ -190,7 +190,9 @@ function thinking.new_turn_order() -- Returns the row's own layout, whose text blocks replace the row's text. function order.enter_row(msg: any): any? local layout = read_layout(msg) - if layout then active = layout end + if type(msg.metadata) == "table" and msg.metadata.content_positions ~= nil then + active = layout + end return layout end diff --git a/src/llm/test/replay_safety_test.lua b/src/llm/test/replay_safety_test.lua index eefd8081..68151a0b 100644 --- a/src/llm/test/replay_safety_test.lua +++ b/src/llm/test/replay_safety_test.lua @@ -478,6 +478,18 @@ local function define_tests() end) for _, path in ipairs(EXTRACTION_PATHS) do + it("keeps an edited response out of the preceding layout: " .. path.name, function() + local rows = stored_rows(path.extract({ "a", "s1", "b" }, false, "end_turn")) + local second = stored_rows(path.extract({ "s2", "c", "d" }, false, "end_turn"))[2] + test.not_nil(rows[2].metadata.content_positions) + test.not_nil(second.metadata.content_positions) + second.content = text("edited") + table.insert(rows, second) + local payload = path.replay(rows) + test.eq(#payload.messages, 2) + assert_order(path.order(payload.messages[2]), { "a", "s1", "b", "s2", "edited" }) + end) + it("renders no blank layout text beside visible text: " .. path.name, function() local result: any = path.extract({ "s1", "A", { "c1" }, "s2", "\n\n", { "c2" } }) test.eq(result.result.content, "A\n\n") From 84004b20b07850bc18d7fcf92648cffd1acdad23 Mon Sep 17 00:00:00 2001 From: KickSide Date: Wed, 4 Nov 2026 14:05:15 -0500 Subject: [PATCH 36/37] fix(llm): keep room for the answer when the output limit caps thinking --- src/llm/src/util/route.lua | 6 ++++-- src/llm/test/replay_safety_test.lua | 24 ++++++++++++++++++++++ src/llm/test/util/route_test.lua | 32 +++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/src/llm/src/util/route.lua b/src/llm/src/util/route.lua index 76c33ecb..62095276 100644 --- a/src/llm/src/util/route.lua +++ b/src/llm/src/util/route.lua @@ -127,8 +127,10 @@ function route.plan_output(max_tokens: number?, effort: number?, mode: string?, if wanted_budget and (not max_tokens or max_tokens <= wanted_budget) then output = wanted_budget + ANSWER_ROOM end + local capped = false if limit and output and output > limit then output = limit + capped = true end if not wanted_budget then return { max_tokens = output } @@ -137,8 +139,8 @@ function route.plan_output(max_tokens: number?, effort: number?, mode: string?, return { max_tokens = math.min(max_tokens or default_max_tokens or output, output), dropped_thinking = true } end local budget = wanted_budget - if budget >= output then - budget = math.max(output - ANSWER_ROOM, MIN_THINKING_BUDGET) + if capped then + budget = math.min(budget, math.max(output - ANSWER_ROOM, MIN_THINKING_BUDGET)) end return { max_tokens = output, budget = budget, wanted_budget = wanted_budget } end diff --git a/src/llm/test/replay_safety_test.lua b/src/llm/test/replay_safety_test.lua index 68151a0b..e490d4d7 100644 --- a/src/llm/test/replay_safety_test.lua +++ b/src/llm/test/replay_safety_test.lua @@ -1097,6 +1097,30 @@ local function define_tests() test.eq(result.metadata.adjusted.max_tokens.sent, 8192) end) + it("reserves Claude answer room when the capped output just exceeds the budget", function() + local sent = capture_claude(claude_generate) + setup(claude_generate, "generate", card({ output_tokens = 8192 })) + local result: any = assert(llm.generate("hi", { model = "m", max_tokens = 512, thinking_effort = 31 })) + test.eq(sent[1].payload.max_tokens, 8192) + test.eq(sent[1].payload.thinking.budget_tokens, 7168) + test.eq(result.metadata.adjusted.max_tokens.requested, 512) + test.eq(result.metadata.adjusted.max_tokens.sent, 8192) + test.eq(result.metadata.adjusted.thinking_budget.requested, 8147) + test.eq(result.metadata.adjusted.thinking_budget.sent, 7168) + end) + + it("reserves Bedrock answer room when the capped output just exceeds the budget", function() + local sent = capture_bedrock(bedrock_generate) + setup(bedrock_generate, "generate", card({ output_tokens = 8192 })) + local result: any = assert(llm.generate("hi", { model = "m", max_tokens = 512, thinking_effort = 31 })) + test.eq(sent[1].payload.inferenceConfig.maxTokens, 8192) + test.eq(sent[1].payload.additionalModelRequestFields.thinking.budget_tokens, 7168) + test.eq(result.metadata.adjusted.max_tokens.requested, 512) + test.eq(result.metadata.adjusted.max_tokens.sent, 8192) + test.eq(result.metadata.adjusted.thinking_budget.requested, 8147) + test.eq(result.metadata.adjusted.thinking_budget.sent, 7168) + end) + it("reports a raise with no limit in the way", function() local sent = capture_claude(claude_generate) setup(claude_generate, "generate", card({})) diff --git a/src/llm/test/util/route_test.lua b/src/llm/test/util/route_test.lua index c1646069..646eaf11 100644 --- a/src/llm/test/util/route_test.lua +++ b/src/llm/test/util/route_test.lua @@ -112,6 +112,38 @@ local function define_tests() end) end) + describe("Route output planning", function() + it("reserves answer room when a cap falls just above the thinking budget", function() + for _, case in ipairs({ + { limit = 8192, output = 8192, budget = 7168 }, + { limit = 8148, output = 8148, budget = 7124 }, + { limit = 9170, output = 9170, budget = 8146 }, + { limit = 9171, output = 9171, budget = 8147 }, + { limit = 10000, output = 9171, budget = 8147 }, + }) do + local plan = route.plan_output(512, 31, "budget", 2000, case.limit) + test.eq(plan.max_tokens, case.output) + test.eq(plan.budget, case.budget) + test.eq(plan.wanted_budget, 8147) + test.is_nil(plan.dropped_thinking) + end + equal(route.plan_output(8192, 31, "budget", 2000, nil), + { max_tokens = 8192, budget = 8147, wanted_budget = 8147 }) + end) + + it("keeps the minimum thinking budget when a cap leaves less than the answer room", function() + for _, limit in ipairs({ 1255, 2047, 2048, 1025 }) do + local plan = route.plan_output(512, 1, "budget", 2000, limit) + test.eq(plan.max_tokens, limit) + test.eq(plan.budget, 1024) + test.eq(plan.wanted_budget, 1254) + test.is_nil(plan.dropped_thinking) + end + equal(route.plan_output(512, 1, "budget", 2000, 1024), + { max_tokens = 512, dropped_thinking = true }) + end) + end) + describe("Route capability helpers", function() local capability = { name = "Test", From 4a1b567771ba46831267226d0562143f4ea38b09 Mon Sep 17 00:00:00 2001 From: KickSide Date: Wed, 4 Nov 2026 15:11:13 -0500 Subject: [PATCH 37/37] fix(llm,agent): ignore time_interval and stop mappers writing to caller rows --- README.md | 5 +++ src/agent/src/README.md | 8 +++-- src/agent/src/traits/init_time_aware.lua | 4 --- src/agent/test/_index.yaml | 1 + src/agent/test/replay_safety_test.lua | 36 +++++++++++++++++++ .../test/traits/init_time_aware_test.lua | 29 ++++++++++++--- src/llm/src/README.md | 2 +- src/llm/src/claude/mapper.lua | 4 ++- src/llm/src/openai_compat/mapper.lua | 3 -- src/llm/test/claude/mapper_test.lua | 18 ++++++++++ src/llm/test/replay_safety_test.lua | 34 ++++++++++++++++++ 11 files changed, 127 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 976216cf..4bb0c678 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,11 @@ Provides the foundational building blocks for developing concurrent AI agent app including actor model primitives, LLM integration, HTTP services, data storage, security, and configuration management — enabling developers to build sophisticated multiagent systems through Wippy's configuration-driven architecture. +Stored conversations with developer messages render a different prefix once after +upgrading on Claude and Bedrock. Their prompt cache restarts, and thinking blocks +bound to the old prefix no longer match. See the [LLM replay guidance](src/llm/src/README.md#positional-notes) +for handling retained thinking blocks across the upgrade. + ## Modules ### Framework Modules diff --git a/src/agent/src/README.md b/src/agent/src/README.md index 8574e670..1ac75ee3 100644 --- a/src/agent/src/README.md +++ b/src/agent/src/README.md @@ -427,8 +427,8 @@ The note ends with "Use this temporal context when relevant to your responses." empty or unknown timezone or a `format` that is not a non-empty string. `format` is an optional Go time layout for the date and time part. The note carries the exact minute and is never rebuilt; the framework does not inject it per step. -The removed `time_interval` setting fails compilation with an error naming -`turn_context`. +The legacy `time_interval` setting is ignored, so existing configurations still +compile. A malformed `time_format` still fails compilation. ### Token Tracking @@ -491,6 +491,8 @@ end start are delivered as soon as the failure is known. - Delivery happens before `after_execute` wrappers run; the wrappers and the return value still receive the complete batch keyed by call ID. +- The listener runs synchronously. Stop is observed only after a listener in + progress returns, so keep listeners short. - A listener fails by raising or by returning a non-nil second value. The batch then stops: no further call starts, calls still running are cancelled, results of calls that finished but were not yet delivered are dropped without @@ -670,7 +672,7 @@ return { execute = execute } ### Built-in Traits -- `wippy.agent.traits:time_aware` -- Declares `agent_options.turn_context.time` through `build_func_id`; the host renders the note with `wippy.agent:turn_context`. Configurable via context: `timezone` (default UTC), `time_format` (optional Go time layout). It contributes no prompt text; a `time_interval` setting is rejected. +- `wippy.agent.traits:time_aware` -- Declares `agent_options.turn_context.time` through `build_func_id`; the host renders the note with `wippy.agent:turn_context`. Configurable via context: `timezone` (default UTC), `time_format` (optional Go time layout). It contributes no prompt text; a legacy `time_interval` setting is ignored. A trait build function that returns an error fails `compiler.compile` with an error naming the trait and the function. diff --git a/src/agent/src/traits/init_time_aware.lua b/src/agent/src/traits/init_time_aware.lua index f979f0db..1fc30913 100644 --- a/src/agent/src/traits/init_time_aware.lua +++ b/src/agent/src/traits/init_time_aware.lua @@ -11,10 +11,6 @@ type TimeAwareResult = { -- Declares the per-turn time context; hosts render it with -- wippy.agent:turn_context and persist it with each user turn. local function execute(base_prompt: string?): (TimeAwareResult?, string?) - if ctx.get("time_interval") ~= nil then - return nil, "time_aware no longer rounds the time: remove time_interval; hosts render the exact minute " - .. "from agent_options.turn_context with wippy.agent:turn_context" - end -- The trait context merges with the agent context, so the layout key is -- namespaced: a generic "format" there belongs to someone else. local format = ctx.get("time_format") diff --git a/src/agent/test/_index.yaml b/src/agent/test/_index.yaml index 9a5309de..4d2ee82a 100644 --- a/src/agent/test/_index.yaml +++ b/src/agent/test/_index.yaml @@ -396,6 +396,7 @@ entries: imports: test: wippy.test:test init_time_aware: wippy.agent.traits:init_time_aware + compiler: wippy.agent.compiler:compiler method: run_tests - name: turn_context_test diff --git a/src/agent/test/replay_safety_test.lua b/src/agent/test/replay_safety_test.lua index 9c1faa0e..06159431 100644 --- a/src/agent/test/replay_safety_test.lua +++ b/src/agent/test/replay_safety_test.lua @@ -269,6 +269,42 @@ local function define_tests() agent._llm = nil end) + it("keeps two cache markers across five steps with one builder on Bedrock and Claude", function() + for _, route in ipairs({ ROUTES[2], ROUTES[1] }) do + local sent = install({ id = route.id, provider_model = "wire" }) + local runner = assert(agent.new(spec("agent_a", 0))) + local builder = prompt.new() + local counts, unchanged = {}, {} + for step = 1, 5 do + if step == 3 then + builder:add_function_call("lookup", "{}", "call-1") + builder:add_function_result("lookup", "Found", "call-1") + else + builder:add_user("Question " .. step) + end + local before = json.encode(builder.messages) + local response = assert(runner:step(builder, {})) + unchanged[step] = json.encode(builder.messages) == before + local count = 0 + local payload = sent[step].payload + for _, block in ipairs(payload.system or {}) do + if block.cache_control or block.cachePoint then count = count + 1 end + end + for _, message in ipairs(payload.messages) do + for _, block in ipairs(message.content) do + if block.cache_control or block.cachePoint then count = count + 1 end + end + end + counts[step] = count + builder:add_assistant(response.result) + end + test.eq(table.concat(counts, ","), "2,2,2,2,2", route.driver .. " per-step cache markers") + for step = 1, 5 do + test.is_true(unchanged[step], route.driver .. " leaves builder unchanged at step " .. step) + end + end + end) + for _, route in ipairs(ROUTES) do local UNBINDABLE = { { name = "default budget", thinking = nil, effort = 0 }, diff --git a/src/agent/test/traits/init_time_aware_test.lua b/src/agent/test/traits/init_time_aware_test.lua index 9ce33e32..b1e59bf0 100644 --- a/src/agent/test/traits/init_time_aware_test.lua +++ b/src/agent/test/traits/init_time_aware_test.lua @@ -1,5 +1,6 @@ local init_time_aware = require("init_time_aware") local funcs = require("funcs") +local compiler = require("compiler") local function define_tests() describe("Init Time Aware Trait", function() @@ -43,14 +44,32 @@ local function define_tests() end end) - it("rejects the removed time_interval setting and names its replacement", function() + it("ignores the legacy time_interval setting", function() local result, err = funcs.new() - :with_context({ timezone = "UTC", time_interval = 15 }) + :with_context({ timezone = "Europe/Berlin", time_format = "2006-01-02 15:04", time_interval = 15 }) :call("wippy.agent.traits:init_time_aware", "base prompt") - test.is_nil(result) - test.contains(tostring(err), "time_interval") - test.contains(tostring(err), "turn_context") + test.is_nil(err, "legacy time_interval is ignored") + test.not_nil(result) + test.is_nil(result.prompt) + test.eq(result.agent_options.turn_context.time.timezone, "Europe/Berlin") + test.eq(result.agent_options.turn_context.time.format, "2006-01-02 15:04") + end) + + it("compiles a time-aware agent with legacy time_interval context", function() + local result, err = compiler.compile({ + id = "test:legacy_time_aware", + model = "test-model", + prompt = "Base prompt", + context = { timezone = "Europe/Berlin", time_interval = 15 }, + traits = { "wippy.agent.traits:time_aware" }, + }) + + test.is_nil(err, "legacy time_interval does not fail compilation") + test.not_nil(result) + test.eq(result.prompt, "Base prompt") + test.eq(result.agent_options.turn_context.time.timezone, "Europe/Berlin") + test.is_nil(result.agent_options.turn_context.time.format) end) end) end diff --git a/src/llm/src/README.md b/src/llm/src/README.md index 2271d0f4..d0443af0 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -252,7 +252,7 @@ builder:add_system("The current date and time is Tuesday, November 3, 2026 at 09 Developer messages, positional or not, render the same way on Claude and Bedrock: each text part becomes its own text block, `` + text + ``, after the content of the user turn it follows, or a new user turn at the start or after an assistant turn. The wrapper tells the model host guidance apart from the user's prose. Positional system notes render as plain text. Existing blocks are never edited, so the rendering does not depend on where cache markers sit. -Bedrock sends developer messages as user text instead of `system` blocks. A Bedrock conversation stored before this change therefore renders a different prefix once after upgrading: its cache restarts and its earlier thinking blocks no longer match. The provider rejects such blocks unless the request lets it drop them; hosts that keep thinking blocks across the upgrade strip them with `drop_stale_thinking`. +Claude developer messages render as separate wrapped user text blocks, and Bedrock sends them as user text instead of `system` blocks. A Claude or Bedrock conversation with developer messages stored before this change therefore renders a different prefix once after upgrading: its cache restarts and thinking blocks bound to the old prefix no longer match. The provider rejects such blocks unless the request lets it drop them; hosts that keep thinking blocks across the upgrade strip them with `drop_stale_thinking`. ### Tool Call IDs diff --git a/src/llm/src/claude/mapper.lua b/src/llm/src/claude/mapper.lua index 64820999..bbb1154d 100644 --- a/src/llm/src/claude/mapper.lua +++ b/src/llm/src/claude/mapper.lua @@ -254,7 +254,9 @@ local function convert_image_content(content_part) } end end - return content_part + local copy = {} + for key, value in pairs(content_part) do copy[key] = value end + return copy end local function process_content_array(content) diff --git a/src/llm/src/openai_compat/mapper.lua b/src/llm/src/openai_compat/mapper.lua index fce276a6..75c745a1 100644 --- a/src/llm/src/openai_compat/mapper.lua +++ b/src/llm/src/openai_compat/mapper.lua @@ -185,9 +185,6 @@ function openai_mapper.map_messages(contract_messages, options) while i <= #contract_messages do local msg = contract_messages[i] - -- Clear metadata - msg.metadata = nil - if msg.role == "cache_marker" then -- A marker after a string tool result cannot be represented as a -- content-part breakpoint without changing the provider payload. diff --git a/src/llm/test/claude/mapper_test.lua b/src/llm/test/claude/mapper_test.lua index be1331eb..969b62c4 100644 --- a/src/llm/test/claude/mapper_test.lua +++ b/src/llm/test/claude/mapper_test.lua @@ -641,6 +641,24 @@ local function define_tests() test.eq(first_system.cache_control.type, "ephemeral") end) + it("marks copies of caller system, user and assistant text blocks", function() + local rows = { + { role = "system", content = { { type = "text", text = "Base" } } }, + { role = "cache_marker" }, + { role = "user", content = { { type = "text", text = "Question" } } }, + { role = "cache_marker" }, + { role = "assistant", content = { { type = "text", text = "Answer" } } }, + { role = "cache_marker" }, + } + local before = json.decode((json.encode(rows))) + local mapped = mapper.map_messages(rows) + + test.eq(mapped.system[1].cache_control.type, "ephemeral") + test.eq(mapped.messages[1].content[1].cache_control.type, "ephemeral") + test.eq(mapped.messages[2].content[1].cache_control.type, "ephemeral") + test.is_true(same(rows, before), "cache marking leaves caller blocks unchanged") + end) + it("should cache a tool-result history tail", function() local result = mapper.map_messages({ { role = "user", content = {{ type = "text", text = "Weather?" }} }, diff --git a/src/llm/test/replay_safety_test.lua b/src/llm/test/replay_safety_test.lua index e490d4d7..94dffd65 100644 --- a/src/llm/test/replay_safety_test.lua +++ b/src/llm/test/replay_safety_test.lua @@ -352,6 +352,40 @@ local function define_tests() llm._usage_tracker = nil end) + it("preserves caller rows when compat maps before a Claude fallback", function() + local rows = { + { role = "system", content = text("Base") }, + { role = "user", content = text("Question") }, + { role = "system", content = text("Turn context"), metadata = { preserve_position = true } }, + { role = "assistant", content = text("Answer"), metadata = { + thinking_blocks = { { type = "thinking", thinking = "Reasoning", signature = "sig" }, + { type = "redacted_thinking", data = "opaque" } }, + } }, + { role = "user", content = text("Follow-up") }, + { role = "developer", content = text("Recall"), metadata = { + preserve_position = true, memory_ids = { "memory-1", "memory-2" }, + } }, + } + local before = json.encode(rows) + + local compat = compat_mapper.map_messages(rows) + local after_compat = json.encode(rows) + local replay = claude_mapper.map_messages(rows) + + test.eq(replay.messages[2].content[1].type, "thinking", "Claude fallback keeps thinking blocks") + test.eq(replay.messages[2].content[1].signature, "sig") + test.eq(replay.messages[2].content[2].type, "redacted_thinking") + test.eq(replay.messages[2].content[2].data, "opaque") + test.eq(replay.messages[2].content[3].text, "Answer") + test.eq(#replay.system, 1) + test.eq(replay.system[1].text, "Base") + test.eq(replay.messages[1].content[2].text, "Turn context") + test.eq(replay.messages[3].content[2].text, "Recall") + test.eq(after_compat, before, "compat leaves all input tables unchanged") + test.eq(json.encode(rows), before, "Claude leaves all input tables unchanged") + for _, row in ipairs(compat) do test.is_nil(row.metadata) end + end) + describe("stale blocks without a bindable thinking object", function() local UNBINDABLE = { { name = "default budget at effort 0", accepts = nil, options = { max_tokens = 100 } },