|
1 | 1 | import { describe, expect, test } from "bun:test"; |
2 | 2 | import { createSizeCapTransform } from "@intx/inference"; |
3 | | -import type { StrategyContext, ToolResult } from "@intx/types/runtime"; |
4 | | -import { MAX_RESULT_CHARS, truncateToolResultContent } from "./result-truncation-plugin.js"; |
| 3 | +import { createBlobReader, type StrategyContext, type ToolResult } from "@intx/types/runtime"; |
| 4 | +import { |
| 5 | + MAX_RESULT_CHARS, |
| 6 | + resultTruncationPlugin, |
| 7 | + spillBlobKey, |
| 8 | + truncateToolResultContent, |
| 9 | +} from "./result-truncation-plugin.js"; |
| 10 | + |
| 11 | +/** In-memory stand-in for ContextStore's writeBlob/readBlob pair, for tests. */ |
| 12 | +function fakeBlobStore() { |
| 13 | + const blobs = new Map<string, Uint8Array>(); |
| 14 | + return { |
| 15 | + blobs, |
| 16 | + writeBlob: async (key: string, bytes: Uint8Array) => { |
| 17 | + blobs.set(key, bytes); |
| 18 | + }, |
| 19 | + readBlob: async (key: string) => { |
| 20 | + const bytes = blobs.get(key); |
| 21 | + if (bytes === undefined) throw new Error(`Blob not found: ${key}`); |
| 22 | + return bytes; |
| 23 | + }, |
| 24 | + }; |
| 25 | +} |
5 | 26 |
|
6 | 27 | describe("truncateToolResultContent", () => { |
7 | | - test("within-cap content passes through unchanged", () => { |
| 28 | + test("within-cap content passes through unchanged", async () => { |
8 | 29 | const content = "x".repeat(100); |
9 | | - expect(truncateToolResultContent(content)).toBe(content); |
| 30 | + expect(await truncateToolResultContent(content)).toBe(content); |
10 | 31 | }); |
11 | 32 |
|
12 | | - test("oversized content gets a marker that never promises retrievable remainder", () => { |
| 33 | + test("oversized content with no blob store gets a marker that never promises retrievable remainder", async () => { |
13 | 34 | const content = "x".repeat(MAX_RESULT_CHARS + 500); |
14 | | - const truncated = truncateToolResultContent(content); |
| 35 | + const truncated = await truncateToolResultContent(content); |
15 | 36 |
|
16 | 37 | expect(truncated).toContain("[output truncated"); |
17 | 38 | expect(truncated).toContain("NOT retrievable"); |
18 | 39 | // The pre-cap discard must never be described as recoverable elsewhere. |
19 | 40 | expect(truncated).not.toContain("see the rest"); |
20 | 41 | expect(truncated).not.toContain("Full output available"); |
| 42 | + // And it must never promise a lifetime it doesn't control either way. |
| 43 | + expect(truncated).not.toContain("removed"); |
| 44 | + expect(truncated).not.toContain("session ends"); |
21 | 45 | }); |
22 | 46 |
|
23 | | - test("truncation marker survives the size-cap blob spill", async () => { |
24 | | - // Reproduce the production pipeline for an output over MAX_RESULT_CHARS: |
25 | | - // truncation runs first (at the tool), size-cap spills the already-cut |
26 | | - // text to a blob and tells the model the blob holds the full output. The |
27 | | - // blob's tail must therefore carry the honest "discarded, NOT retrievable" |
28 | | - // marker so the model does not loop re-running the command. |
29 | | - const original = "x".repeat(MAX_RESULT_CHARS + 500); |
30 | | - const truncated = truncateToolResultContent(original); |
31 | | - |
32 | | - const blobs = new Map<string, string>(); |
33 | | - const transform = createSizeCapTransform({ |
34 | | - maxChars: 10_000, |
35 | | - contextStore: { |
36 | | - writeBlob: async (key: string, bytes: Uint8Array) => { |
37 | | - blobs.set(key, new TextDecoder().decode(bytes)); |
38 | | - }, |
39 | | - }, |
| 47 | + test("the inlined portion stays bounded regardless of blob-store support", async () => { |
| 48 | + const content = "x".repeat(MAX_RESULT_CHARS * 3); |
| 49 | + const truncated = await truncateToolResultContent(content); |
| 50 | + // The marker text itself adds a bounded amount of overhead on top of the cap. |
| 51 | + expect(truncated.length).toBeLessThan(MAX_RESULT_CHARS + 1000); |
| 52 | + }); |
| 53 | + |
| 54 | + describe("with a blob store", () => { |
| 55 | + test("a result over the cap is fully recoverable by following the notice's read_file instructions verbatim", async () => { |
| 56 | + const store = fakeBlobStore(); |
| 57 | + const original = `${"x".repeat(MAX_RESULT_CHARS)}TAIL-MARKER-${"y".repeat(500)}`; |
| 58 | + const truncated = await truncateToolResultContent(original, MAX_RESULT_CHARS, { |
| 59 | + callId: "call-42", |
| 60 | + writeBlob: store.writeBlob, |
| 61 | + }); |
| 62 | + |
| 63 | + // Inline content is bounded and does not itself contain the discarded tail. |
| 64 | + expect(truncated).not.toContain("TAIL-MARKER"); |
| 65 | + expect(truncated.length).toBeLessThan(MAX_RESULT_CHARS + 1000); |
| 66 | + |
| 67 | + const uriMatch = /tool-output:\/\/\/\S+/.exec(truncated); |
| 68 | + expect(uriMatch).not.toBeNull(); |
| 69 | + const uri = uriMatch?.[0].replace(/[.\]]+$/, "") ?? ""; |
| 70 | + expect(uri).toBe(`tool-output:///${spillBlobKey("call-42")}`); |
| 71 | + |
| 72 | + // Follow the notice's instructions literally: read_file with that URI, |
| 73 | + // via the real BlobReader machinery read_file itself uses. |
| 74 | + const blobReader = createBlobReader(store); |
| 75 | + const recoveredBytes = await blobReader.read(uri); |
| 76 | + const recovered = new TextDecoder().decode(recoveredBytes); |
| 77 | + expect(recovered).toBe(original); |
| 78 | + expect(recovered).toContain("TAIL-MARKER"); |
| 79 | + expect(recovered.length).toBe(original.length); |
| 80 | + |
| 81 | + // No false lifetime claim: the blob is part of the committed session |
| 82 | + // history, not something with its own expiry. |
| 83 | + expect(truncated).not.toContain("removed"); |
| 84 | + expect(truncated).not.toContain("session ends"); |
40 | 85 | }); |
41 | 86 |
|
42 | | - const result: ToolResult = { |
43 | | - callId: "call-1", |
44 | | - content: truncated, |
45 | | - isError: false, |
46 | | - }; |
47 | | - const { output } = await transform.apply( |
48 | | - { call: { id: "call-1", name: "run_shell", arguments: {} }, result }, |
49 | | - {} as StrategyContext, |
| 87 | + test("within-cap content never writes a blob", async () => { |
| 88 | + const store = fakeBlobStore(); |
| 89 | + await truncateToolResultContent("x".repeat(100), MAX_RESULT_CHARS, { |
| 90 | + callId: "call-1", |
| 91 | + writeBlob: store.writeBlob, |
| 92 | + }); |
| 93 | + expect(store.blobs.size).toBe(0); |
| 94 | + }); |
| 95 | + |
| 96 | + test( |
| 97 | + "the full spill survives the reactor's own downstream size-cap transform " + |
| 98 | + "(CL-6908 regression: a same-keyed write here would let that second write clobber it)", |
| 99 | + async () => { |
| 100 | + const store = fakeBlobStore(); |
| 101 | + const original = "p".repeat(500_000); |
| 102 | + const truncated = await truncateToolResultContent(original, MAX_RESULT_CHARS, { |
| 103 | + callId: "call-1", |
| 104 | + writeBlob: store.writeBlob, |
| 105 | + }); |
| 106 | + |
| 107 | + // Reproduce the production pipeline: this middleware's ToolResult |
| 108 | + // continues into the reactor, which always runs its own size-cap |
| 109 | + // transform (vendor/intx-inference, default cap 10,000 chars) on |
| 110 | + // every result, keyed by the bare call id. |
| 111 | + const reactorCap = createSizeCapTransform({ |
| 112 | + maxChars: 10_000, |
| 113 | + contextStore: { writeBlob: store.writeBlob }, |
| 114 | + }); |
| 115 | + const result: ToolResult = { callId: "call-1", content: truncated, isError: false }; |
| 116 | + await reactorCap.apply( |
| 117 | + { call: { id: "call-1", name: "run_shell", arguments: {} }, result }, |
| 118 | + {} as StrategyContext, |
| 119 | + ); |
| 120 | + |
| 121 | + // The reactor wrote its own (lossy) blob under the bare "call-1" key. |
| 122 | + expect(store.blobs.has("call-1")).toBe(true); |
| 123 | + // Our full spill lives under a distinct key and is untouched. |
| 124 | + const blobReader = createBlobReader(store); |
| 125 | + const recovered = new TextDecoder().decode( |
| 126 | + await blobReader.read(`tool-output:///${spillBlobKey("call-1")}`), |
| 127 | + ); |
| 128 | + expect(recovered).toBe(original); |
| 129 | + expect(recovered.length).toBe(500_000); |
| 130 | + }, |
| 131 | + ); |
| 132 | + }); |
| 133 | +}); |
| 134 | + |
| 135 | +describe("resultTruncationPlugin", () => { |
| 136 | + test("spills oversized run_shell/grep/search_files/web_fetch results via the live getBlobWriter getter", async () => { |
| 137 | + const store = fakeBlobStore(); |
| 138 | + const original = "q".repeat(MAX_RESULT_CHARS + 200); |
| 139 | + const plugin = resultTruncationPlugin({ getBlobWriter: () => store.writeBlob }); |
| 140 | + if (plugin.middleware === undefined) throw new Error("expected middleware"); |
| 141 | + const middleware = plugin.middleware(async (call) => ({ |
| 142 | + callId: call.id, |
| 143 | + content: original, |
| 144 | + })); |
| 145 | + |
| 146 | + const result = await middleware( |
| 147 | + { id: "call-99", name: "run_shell", arguments: {} }, |
| 148 | + new AbortController().signal, |
50 | 149 | ); |
51 | 150 |
|
52 | | - const spilled = blobs.get("call-1"); |
53 | | - expect(spilled).toBe(truncated); |
54 | | - // The blob's tail tells the truth about the pre-spill discard. |
55 | | - expect(spilled).toContain("NOT retrievable"); |
56 | | - expect(spilled?.endsWith("Use offset/limit or a narrower query.]")).toBe(true); |
57 | | - // The inline marker's blob promise is now genuine: the blob really does |
58 | | - // hold everything that still exists. |
59 | | - expect(output.content).toContain("tool-output:///call-1"); |
| 151 | + const uri = `tool-output:///${spillBlobKey("call-99")}`; |
| 152 | + expect(result.content).toContain(uri); |
| 153 | + const recovered = new TextDecoder().decode(await createBlobReader(store).read(uri)); |
| 154 | + expect(recovered).toBe(original); |
| 155 | + }); |
| 156 | + |
| 157 | + test("falls back to the honest no-store notice when getBlobWriter resolves undefined", async () => { |
| 158 | + const plugin = resultTruncationPlugin({ getBlobWriter: () => undefined }); |
| 159 | + if (plugin.middleware === undefined) throw new Error("expected middleware"); |
| 160 | + const middleware = plugin.middleware(async (call) => ({ |
| 161 | + callId: call.id, |
| 162 | + content: "r".repeat(MAX_RESULT_CHARS + 1), |
| 163 | + })); |
| 164 | + const result = await middleware( |
| 165 | + { id: "call-1", name: "grep", arguments: {} }, |
| 166 | + new AbortController().signal, |
| 167 | + ); |
| 168 | + expect(result.content).toContain("NOT retrievable"); |
60 | 169 | }); |
61 | 170 | }); |
0 commit comments