diff --git a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts index 177fb39c91..d05c8e18b2 100644 --- a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts +++ b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts @@ -100,6 +100,47 @@ describe('createOverloadRetryFetch', () => { assert.strictEqual(bad.calls(), 1) }) + it('retries a fetch that THREW, and gives up with the error it was given', async () => { + // Measured at concurrency four on one key against integrate.api.nvidia.com, + // 2026-09-20: two 200s, one 503, and one socket that never answered. The + // status branches cannot see the fourth - there is no response to branch on. + let calls = 0 + const fetchImpl = (async () => { + calls += 1 + if (calls < 3) throw new Error('read ECONNRESET') + return new Response('{"ok":true}', { + status: 200, + headers: { 'content-type': 'application/json' }, + }) + }) as unknown as typeof fetch + const f = createOverloadRetryFetch({ fetchImpl, sleep: noSleep }) + assert.strictEqual((await f('https://example.test/', {})).status, 200) + assert.strictEqual(calls, 3) + + let always = 0 + const dead = (async () => { + always += 1 + throw new Error('read ECONNRESET') + }) as unknown as typeof fetch + const g = createOverloadRetryFetch({ fetchImpl: dead, sleep: noSleep, maxAttempts: 3 }) + await assert.rejects(() => g('https://example.test/', {}), /ECONNRESET/) + assert.strictEqual(always, 3) + }) + + it('does not retry an abort the caller asked for', async () => { + // Retrying a cancelled request outlives the thing that cancelled it. + let calls = 0 + const aborting = (async () => { + calls += 1 + const error = new Error('aborted') + error.name = 'AbortError' + throw error + }) as unknown as typeof fetch + const f = createOverloadRetryFetch({ fetchImpl: aborting, sleep: noSleep }) + await assert.rejects(() => f('https://example.test/', {}), /aborted/) + assert.strictEqual(calls, 1) + }) + it('replays a stream that arrives in many small chunks', async () => { const encoder = new TextEncoder() const pieces = GOOD.match(/.{1,7}/gs) ?? [] diff --git a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts index 7508b5fd65..e2c07a69f4 100644 --- a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts +++ b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts @@ -130,6 +130,14 @@ function replay( }) } +/** An abort the caller asked for, rather than a network failure. */ +export function isAbort(error: unknown): boolean { + if (typeof error !== 'object' || error === null) return false + const name = (error as { name?: unknown }).name + return name === 'AbortError' || name === 'TimeoutError' +} + + export function createOverloadRetryFetch( options: OverloadRetryOptions = {}, ): typeof fetch { @@ -141,10 +149,33 @@ export function createOverloadRetryFetch( return (async (url: RequestInfo | URL, init?: RequestInit) => { const signal = init?.signal for (let attempt = 1; ; attempt++) { - const response = await fetchImpl(url, init) const last = attempt >= maxAttempts const delayMs = delays[Math.min(attempt - 1, delays.length - 1)] ?? 0 + // A THROW is the same outage wearing different clothes. Measured against + // integrate.api.nvidia.com on 2026-09-20 at concurrency four on one key: + // two answers 200, one 503, and one that never answered at all - the + // socket simply hung until the client gave up. The status branches below + // never saw that fourth one, because there was no response to branch on, + // so it reached the agent loop as a terminal error and ended the turn. + // + // An abort the CALLER asked for is not an outage and is re-thrown at + // once: retrying a cancelled request would outlive the thing that + // cancelled it. + let response: Response + try { + response = await fetchImpl(url, init) + } catch (error) { + if (signal?.aborted || isAbort(error) || last) throw error + options.onRetry?.({ + attempt, + delayMs, + reason: `fetch threw: ${error instanceof Error ? error.message : String(error)}`, + }) + await sleep(delayMs, signal) + continue + } + if (response.status === 429 || response.status >= 500) { if (last) return response await response.body?.cancel().catch(() => {}) diff --git a/trios/agent-server/apps/server/src/tools/filesystem/read.ts b/trios/agent-server/apps/server/src/tools/filesystem/read.ts index 0791560404..f2c4c95270 100644 --- a/trios/agent-server/apps/server/src/tools/filesystem/read.ts +++ b/trios/agent-server/apps/server/src/tools/filesystem/read.ts @@ -53,6 +53,9 @@ function getSelectedLines( return remaining } +/** Room kept for the continuation note, so the answer can always carry one. */ +const CONTINUATION_NOTE_BUDGET = 160 + function formatReadResult(args: { selected: string[] startIdx: number @@ -76,10 +79,50 @@ function formatReadResult(args: { text += `\n\n(Showing lines ${startLineNum}-${endLineNum} of ${args.totalLines})` } + // A REFUSAL COSTS A PROVIDER CALL, AND THE PROVIDER IS THE CEILING. + // + // This used to throw, and the agent's only recovery was to ask again with a + // smaller range. Measured on the running deployment 2026-09-20: 18 of 19 + // filesystem tool failures in one window were this refusal, on files of 159 + // and 500 lines - ordinary specs. Each one spent a round trip on the same + // endpoint that was answering `Service temporarily overloaded` 76 times in + // the same window, and returned no content at all. + // + // So it returns what FITS, and says exactly where to continue. The caller + // gets content on the first call and a correct `offset` for the rest, which + // is what it would have asked for on the second. if (text.length > MAX_READ_CHARS) { - throw new Error( - `Requested lines ${startLineNum}-${endLineNum} produce ${text.length} characters in the response, above the ${MAX_READ_CHARS}-character limit for filesystem_read. Retry with a smaller limit or a later offset.`, - ) + const kept: string[] = [] + let used = 0 + for (let i = 0; i < args.selected.length; i++) { + const rendered = `${String(args.startIdx + i + 1).padStart(width)} | ${args.selected[i]}\n` + // Leave room for the continuation note, which is what makes the answer + // usable rather than merely shorter. + if (used + rendered.length > MAX_READ_CHARS - CONTINUATION_NOTE_BUDGET) + break + used += rendered.length + kept.push(args.selected[i]) + } + if (kept.length === 0) { + // One line longer than the whole budget. Nothing to hand back, and the + // caller needs to hear why rather than get an empty answer. + throw new Error( + `Line ${startLineNum} alone is ${args.selected[0]?.length ?? 0} characters, above the ${MAX_READ_CHARS}-character limit for filesystem_read. Use filesystem_grep to find what you need in it.`, + ) + } + const cutAt = args.startIdx + kept.length + const shortened = kept + .map( + (line, i) => + `${String(args.startIdx + i + 1).padStart(width)} | ${line}`, + ) + .join('\n') + return { + text: + shortened + + `\n\n(${args.totalLines - cutAt} more lines in file; this answer was ` + + `cut at the ${MAX_READ_CHARS}-character limit. Use offset=${cutAt + 1} to continue reading.)`, + } } return { text } diff --git a/trios/agent-server/apps/server/tests/tools/filesystem/read.test.ts b/trios/agent-server/apps/server/tests/tools/filesystem/read.test.ts index c6ca8a7748..db30fdbdb4 100644 --- a/trios/agent-server/apps/server/tests/tools/filesystem/read.test.ts +++ b/trios/agent-server/apps/server/tests/tools/filesystem/read.test.ts @@ -154,6 +154,25 @@ describe('filesystem_read', () => { expect(result.text).toContain(`${MAX_READ_CHARS}-character limit`) }) + it('returns what fits instead of refusing, with the offset to continue', async () => { + // A refusal costs a provider call and returns nothing. Measured on the + // running deployment 2026-09-20: 18 of 19 filesystem failures in one window + // were this refusal, on files of 159 and 500 lines. + const line = 'y'.repeat(200) + const lines = Array.from({ length: 200 }, () => line) + await writeFile(join(tmpDir, 'wide.txt'), lines.join('\n')) + const result = await exec({ path: 'wide.txt', limit: 200 }) + expect(result.isError).toBeFalsy() + expect(result.text.length).toBeLessThanOrEqual(MAX_READ_CHARS) + expect(result.text).toContain('cut at the') + const offset = Number(/offset=(\d+)/.exec(result.text)?.[1]) + expect(Number.isInteger(offset)).toBe(true) + expect(offset).toBeGreaterThan(1) + // The offset it names must actually be the next unread line. + const rest = await exec({ path: 'wide.txt', offset, limit: 1 }) + expect(rest.text).toContain(`${offset} | `) + }) + it('handles files with UTF-8 BOM', async () => { await writeFile(join(tmpDir, 'bom.txt'), '\uFEFFhello bom') const result = await exec({ path: 'bom.txt' }) diff --git a/trios/agent-server/docker-entrypoint.sh b/trios/agent-server/docker-entrypoint.sh index 320ba1fe9b..af7964ea98 100755 --- a/trios/agent-server/docker-entrypoint.sh +++ b/trios/agent-server/docker-entrypoint.sh @@ -206,6 +206,52 @@ if [ -n "$TRIOS_TOOL_SHELL_USER" ] && id "$TRIOS_TOOL_SHELL_USER" >/dev/null 2>& echo "[entrypoint] $WORKSPACE_DIR is not owned by $TRIOS_TOOL_SHELL_USER; settling ownership once" chown -R "$TRIOS_TOOL_SHELL_USER" "$WORKSPACE_DIR" fi + # AND INSIDE THE CHECKOUT, which the test above cannot see. Measured on the + # running deployment 2026-09-20: + # + # error: Your local changes to the following files would be overwritten by checkout: + # error: unable to create file specs/port/tools/gft_deep_demo.t27: Permission denied + # + # $WORKSPACE_DIR was owned by the bee, so the walk above was skipped, while + # files underneath were not - left by a root-run git from an older image. A + # bee that cannot write the checkout produces an EMPTY branch and a turn that + # looks like a model failure: 80 of 101 stuck issues on that day had one. + # + # `find ! -user -print -quit` stops at the FIRST wrong file, so the healthy + # case costs one stat and the 45 GB walk that once outlasted the 300 s + # healthcheck (2026-09-03) cannot come back. The repair walks the checkout + # only - never the worktrees beside it - and changes only what is wrong. + if [ -d "$REPO_DIR" ] && [ -n "$(find "$REPO_DIR" ! -user "$TRIOS_TOOL_SHELL_USER" -print -quit 2>/dev/null)" ]; then + echo "[entrypoint] files inside $REPO_DIR are not owned by $TRIOS_TOOL_SHELL_USER; repairing those" + find "$REPO_DIR" ! -user "$TRIOS_TOOL_SHELL_USER" -exec chown "$TRIOS_TOOL_SHELL_USER" {} + 2>/dev/null || true + fi + # BEE WORKTREES FROM A PREVIOUS LIFE. At entrypoint time no bee is running - + # this process is what starts the server that starts them - so every + # directory under .worktrees/ belongs to a container that is already gone. + # + # They are not free. Measured on the running deployment 2026-09-20: every + # tick for six minutes chose an issue and then refused to start it - + # + # Queen tick chose an issue but the container cannot carry another bee + # issue=4438 resource="disk" + # + # - with twenty lanes open and 684 candidates waiting. The volume had filled + # with worktrees nobody could use; an earlier reading of the same volume + # found 41 of them holding 45 GB and three million inodes. + # + # `git worktree prune` alone does not do it: it drops the ADMIN records for + # directories that are already gone, and these directories are still there. + if [ -d "$REPO_DIR/.worktrees" ]; then + stale=$(ls -1 "$REPO_DIR/.worktrees" 2>/dev/null | wc -l | tr -d ' ') + if [ "$stale" != "0" ]; then + free_before=$(df -Pm "$REPO_DIR" 2>/dev/null | awk 'NR==2 {print $4}') + echo "[entrypoint] removing $stale bee worktree(s) left by a previous container (${free_before:-?} MiB free)" + $AS_USER "rm -rf '$REPO_DIR/.worktrees'/* '$REPO_DIR/.worktrees'/.[!.]*" 2>/dev/null || true + $AS_USER "git -C '$REPO_DIR' worktree prune" >/dev/null 2>&1 || true + free_after=$(df -Pm "$REPO_DIR" 2>/dev/null | awk 'NR==2 {print $4}') + echo "[entrypoint] worktrees cleared; ${free_after:-?} MiB free now" + fi + fi echo "[entrypoint] git runs as $TRIOS_TOOL_SHELL_USER; root does not enter the checkout" else AS_USER="sh -c"