From 20c6eb0b6a29f0e54b40fb13adbde46b6467d3a0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 20 Sep 2026 20:57:40 +0700 Subject: [PATCH 1/3] fix(queen): a bee that cannot write the checkout, and a fetch that threw TWO FAILURES WITH ONE SYMPTOM: an empty branch and a turn that reads as a model failure. 80 of 101 stuck issues on 2026-09-20 had exactly that shape. THE CHECKOUT. Measured in the running deployment's log: error: Your local changes to the following files would be overwritten by checkout: error: unable to create file specs/port/tools/gft_deep_demo.t27: Permission denied $WORKSPACE_DIR was owned by the bee, so the one-time ownership walk was skipped, while files underneath it were not - left by a root-run git from an older image. `find ! -user -print -quit` stops at the FIRST wrong file, so the healthy case costs one stat and the 45 GB walk that once outlasted the 300 s healthcheck cannot come back. The repair walks the checkout only, never the worktrees beside it, and changes only what is wrong. THE FETCH. Measured at concurrency four on one key against integrate.api.nvidia.com: two answers 200, one 503, and one socket that never answered at all. The retry wrapper handled 429 and 5xx and the 200-carrying-an- error case, and could not see the fourth - there is no response to branch on, so it reached the agent loop as a terminal error and ended the turn. A throw is now retried on the same backoff. An abort the CALLER asked for is re-thrown at once: retrying a cancelled request outlives the thing that cancelled it. bun test apps/server/src/lib/overload-retry-fetch.test.ts 10 pass, 0 fail Co-Authored-By: Claude Opus 5 --- .../src/lib/overload-retry-fetch.test.ts | 41 +++++++++++++++++++ .../server/src/lib/overload-retry-fetch.ts | 33 ++++++++++++++- trios/agent-server/docker-entrypoint.sh | 19 +++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts index 177fb39c9..d05c8e18b 100644 --- a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts +++ b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts @@ -100,6 +100,47 @@ describe('createOverloadRetryFetch', () => { assert.strictEqual(bad.calls(), 1) }) + it('retries a fetch that THREW, and gives up with the error it was given', async () => { + // Measured at concurrency four on one key against integrate.api.nvidia.com, + // 2026-09-20: two 200s, one 503, and one socket that never answered. The + // status branches cannot see the fourth - there is no response to branch on. + let calls = 0 + const fetchImpl = (async () => { + calls += 1 + if (calls < 3) throw new Error('read ECONNRESET') + return new Response('{"ok":true}', { + status: 200, + headers: { 'content-type': 'application/json' }, + }) + }) as unknown as typeof fetch + const f = createOverloadRetryFetch({ fetchImpl, sleep: noSleep }) + assert.strictEqual((await f('https://example.test/', {})).status, 200) + assert.strictEqual(calls, 3) + + let always = 0 + const dead = (async () => { + always += 1 + throw new Error('read ECONNRESET') + }) as unknown as typeof fetch + const g = createOverloadRetryFetch({ fetchImpl: dead, sleep: noSleep, maxAttempts: 3 }) + await assert.rejects(() => g('https://example.test/', {}), /ECONNRESET/) + assert.strictEqual(always, 3) + }) + + it('does not retry an abort the caller asked for', async () => { + // Retrying a cancelled request outlives the thing that cancelled it. + let calls = 0 + const aborting = (async () => { + calls += 1 + const error = new Error('aborted') + error.name = 'AbortError' + throw error + }) as unknown as typeof fetch + const f = createOverloadRetryFetch({ fetchImpl: aborting, sleep: noSleep }) + await assert.rejects(() => f('https://example.test/', {}), /aborted/) + assert.strictEqual(calls, 1) + }) + it('replays a stream that arrives in many small chunks', async () => { const encoder = new TextEncoder() const pieces = GOOD.match(/.{1,7}/gs) ?? [] diff --git a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts index 7508b5fd6..e2c07a69f 100644 --- a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts +++ b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts @@ -130,6 +130,14 @@ function replay( }) } +/** An abort the caller asked for, rather than a network failure. */ +export function isAbort(error: unknown): boolean { + if (typeof error !== 'object' || error === null) return false + const name = (error as { name?: unknown }).name + return name === 'AbortError' || name === 'TimeoutError' +} + + export function createOverloadRetryFetch( options: OverloadRetryOptions = {}, ): typeof fetch { @@ -141,10 +149,33 @@ export function createOverloadRetryFetch( return (async (url: RequestInfo | URL, init?: RequestInit) => { const signal = init?.signal for (let attempt = 1; ; attempt++) { - const response = await fetchImpl(url, init) const last = attempt >= maxAttempts const delayMs = delays[Math.min(attempt - 1, delays.length - 1)] ?? 0 + // A THROW is the same outage wearing different clothes. Measured against + // integrate.api.nvidia.com on 2026-09-20 at concurrency four on one key: + // two answers 200, one 503, and one that never answered at all - the + // socket simply hung until the client gave up. The status branches below + // never saw that fourth one, because there was no response to branch on, + // so it reached the agent loop as a terminal error and ended the turn. + // + // An abort the CALLER asked for is not an outage and is re-thrown at + // once: retrying a cancelled request would outlive the thing that + // cancelled it. + let response: Response + try { + response = await fetchImpl(url, init) + } catch (error) { + if (signal?.aborted || isAbort(error) || last) throw error + options.onRetry?.({ + attempt, + delayMs, + reason: `fetch threw: ${error instanceof Error ? error.message : String(error)}`, + }) + await sleep(delayMs, signal) + continue + } + if (response.status === 429 || response.status >= 500) { if (last) return response await response.body?.cancel().catch(() => {}) diff --git a/trios/agent-server/docker-entrypoint.sh b/trios/agent-server/docker-entrypoint.sh index 320ba1fe9..26d6937fd 100755 --- a/trios/agent-server/docker-entrypoint.sh +++ b/trios/agent-server/docker-entrypoint.sh @@ -206,6 +206,25 @@ if [ -n "$TRIOS_TOOL_SHELL_USER" ] && id "$TRIOS_TOOL_SHELL_USER" >/dev/null 2>& echo "[entrypoint] $WORKSPACE_DIR is not owned by $TRIOS_TOOL_SHELL_USER; settling ownership once" chown -R "$TRIOS_TOOL_SHELL_USER" "$WORKSPACE_DIR" fi + # AND INSIDE THE CHECKOUT, which the test above cannot see. Measured on the + # running deployment 2026-09-20: + # + # error: Your local changes to the following files would be overwritten by checkout: + # error: unable to create file specs/port/tools/gft_deep_demo.t27: Permission denied + # + # $WORKSPACE_DIR was owned by the bee, so the walk above was skipped, while + # files underneath were not - left by a root-run git from an older image. A + # bee that cannot write the checkout produces an EMPTY branch and a turn that + # looks like a model failure: 80 of 101 stuck issues on that day had one. + # + # `find ! -user -print -quit` stops at the FIRST wrong file, so the healthy + # case costs one stat and the 45 GB walk that once outlasted the 300 s + # healthcheck (2026-09-03) cannot come back. The repair walks the checkout + # only - never the worktrees beside it - and changes only what is wrong. + if [ -d "$REPO_DIR" ] && [ -n "$(find "$REPO_DIR" ! -user "$TRIOS_TOOL_SHELL_USER" -print -quit 2>/dev/null)" ]; then + echo "[entrypoint] files inside $REPO_DIR are not owned by $TRIOS_TOOL_SHELL_USER; repairing those" + find "$REPO_DIR" ! -user "$TRIOS_TOOL_SHELL_USER" -exec chown "$TRIOS_TOOL_SHELL_USER" {} + 2>/dev/null || true + fi echo "[entrypoint] git runs as $TRIOS_TOOL_SHELL_USER; root does not enter the checkout" else AS_USER="sh -c" From 24bcc909b53c082228e1fab7b13b730f8afaa0f7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 20 Sep 2026 21:23:13 +0700 Subject: [PATCH 2/3] fix(queen): clear the worktrees of containers that are already gone Every tick for six minutes chose an issue and then refused to start it: Queen tick chose an issue but the container cannot carry another bee issue=4438 resource="disk" Twenty lanes open, 684 candidates waiting, and zero bees - because the volume had filled with bee worktrees nobody could use. An earlier reading of the same volume found 41 of them holding 45 GB and three million inodes. At entrypoint time no bee is running - this process is what starts the server that starts them - so every directory under .worktrees/ belongs to a container that is already gone. `git worktree prune` alone does not do it: it drops the admin records for directories already removed, and these are still there. Free space is printed before and after, so the next reader sees what it bought. Co-Authored-By: Claude Opus 5 --- trios/agent-server/docker-entrypoint.sh | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/trios/agent-server/docker-entrypoint.sh b/trios/agent-server/docker-entrypoint.sh index 26d6937fd..af7964ea9 100755 --- a/trios/agent-server/docker-entrypoint.sh +++ b/trios/agent-server/docker-entrypoint.sh @@ -225,6 +225,33 @@ if [ -n "$TRIOS_TOOL_SHELL_USER" ] && id "$TRIOS_TOOL_SHELL_USER" >/dev/null 2>& echo "[entrypoint] files inside $REPO_DIR are not owned by $TRIOS_TOOL_SHELL_USER; repairing those" find "$REPO_DIR" ! -user "$TRIOS_TOOL_SHELL_USER" -exec chown "$TRIOS_TOOL_SHELL_USER" {} + 2>/dev/null || true fi + # BEE WORKTREES FROM A PREVIOUS LIFE. At entrypoint time no bee is running - + # this process is what starts the server that starts them - so every + # directory under .worktrees/ belongs to a container that is already gone. + # + # They are not free. Measured on the running deployment 2026-09-20: every + # tick for six minutes chose an issue and then refused to start it - + # + # Queen tick chose an issue but the container cannot carry another bee + # issue=4438 resource="disk" + # + # - with twenty lanes open and 684 candidates waiting. The volume had filled + # with worktrees nobody could use; an earlier reading of the same volume + # found 41 of them holding 45 GB and three million inodes. + # + # `git worktree prune` alone does not do it: it drops the ADMIN records for + # directories that are already gone, and these directories are still there. + if [ -d "$REPO_DIR/.worktrees" ]; then + stale=$(ls -1 "$REPO_DIR/.worktrees" 2>/dev/null | wc -l | tr -d ' ') + if [ "$stale" != "0" ]; then + free_before=$(df -Pm "$REPO_DIR" 2>/dev/null | awk 'NR==2 {print $4}') + echo "[entrypoint] removing $stale bee worktree(s) left by a previous container (${free_before:-?} MiB free)" + $AS_USER "rm -rf '$REPO_DIR/.worktrees'/* '$REPO_DIR/.worktrees'/.[!.]*" 2>/dev/null || true + $AS_USER "git -C '$REPO_DIR' worktree prune" >/dev/null 2>&1 || true + free_after=$(df -Pm "$REPO_DIR" 2>/dev/null | awk 'NR==2 {print $4}') + echo "[entrypoint] worktrees cleared; ${free_after:-?} MiB free now" + fi + fi echo "[entrypoint] git runs as $TRIOS_TOOL_SHELL_USER; root does not enter the checkout" else AS_USER="sh -c" From 5fa6fd0ea973b16ddbf69c4438ee768abe4a7a1e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 20 Sep 2026 22:02:14 +0700 Subject: [PATCH 3/3] fix(tools): a read that is too large returns what fits, not a refusal A refusal costs a provider call and returns nothing, and the provider is the ceiling. Measured on the running deployment 2026-09-20 in one 400-line window: 18 of 19 filesystem tool failures were this refusal, on files of 159 and 500 lines - ordinary specs - while the same endpoint answered `Service temporarily overloaded` 76 times in that same window. Every one of those refusals spent a round trip to be told to ask again. filesystem_read now returns the lines that fit under the character limit and names the exact offset to continue from, which is what the caller would have asked for on its second call. Room is kept for that note, so the answer can always carry one. A single line longer than the whole budget still throws - there is nothing to hand back - and now says so in those words, pointing at filesystem_grep. bun test apps/server/tests/tools/filesystem/read.test.ts 17 pass, 0 fail Co-Authored-By: Claude Opus 5 --- .../apps/server/src/tools/filesystem/read.ts | 49 +++++++++++++++++-- .../tests/tools/filesystem/read.test.ts | 19 +++++++ 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/trios/agent-server/apps/server/src/tools/filesystem/read.ts b/trios/agent-server/apps/server/src/tools/filesystem/read.ts index 079156040..f2c4c9527 100644 --- a/trios/agent-server/apps/server/src/tools/filesystem/read.ts +++ b/trios/agent-server/apps/server/src/tools/filesystem/read.ts @@ -53,6 +53,9 @@ function getSelectedLines( return remaining } +/** Room kept for the continuation note, so the answer can always carry one. */ +const CONTINUATION_NOTE_BUDGET = 160 + function formatReadResult(args: { selected: string[] startIdx: number @@ -76,10 +79,50 @@ function formatReadResult(args: { text += `\n\n(Showing lines ${startLineNum}-${endLineNum} of ${args.totalLines})` } + // A REFUSAL COSTS A PROVIDER CALL, AND THE PROVIDER IS THE CEILING. + // + // This used to throw, and the agent's only recovery was to ask again with a + // smaller range. Measured on the running deployment 2026-09-20: 18 of 19 + // filesystem tool failures in one window were this refusal, on files of 159 + // and 500 lines - ordinary specs. Each one spent a round trip on the same + // endpoint that was answering `Service temporarily overloaded` 76 times in + // the same window, and returned no content at all. + // + // So it returns what FITS, and says exactly where to continue. The caller + // gets content on the first call and a correct `offset` for the rest, which + // is what it would have asked for on the second. if (text.length > MAX_READ_CHARS) { - throw new Error( - `Requested lines ${startLineNum}-${endLineNum} produce ${text.length} characters in the response, above the ${MAX_READ_CHARS}-character limit for filesystem_read. Retry with a smaller limit or a later offset.`, - ) + const kept: string[] = [] + let used = 0 + for (let i = 0; i < args.selected.length; i++) { + const rendered = `${String(args.startIdx + i + 1).padStart(width)} | ${args.selected[i]}\n` + // Leave room for the continuation note, which is what makes the answer + // usable rather than merely shorter. + if (used + rendered.length > MAX_READ_CHARS - CONTINUATION_NOTE_BUDGET) + break + used += rendered.length + kept.push(args.selected[i]) + } + if (kept.length === 0) { + // One line longer than the whole budget. Nothing to hand back, and the + // caller needs to hear why rather than get an empty answer. + throw new Error( + `Line ${startLineNum} alone is ${args.selected[0]?.length ?? 0} characters, above the ${MAX_READ_CHARS}-character limit for filesystem_read. Use filesystem_grep to find what you need in it.`, + ) + } + const cutAt = args.startIdx + kept.length + const shortened = kept + .map( + (line, i) => + `${String(args.startIdx + i + 1).padStart(width)} | ${line}`, + ) + .join('\n') + return { + text: + shortened + + `\n\n(${args.totalLines - cutAt} more lines in file; this answer was ` + + `cut at the ${MAX_READ_CHARS}-character limit. Use offset=${cutAt + 1} to continue reading.)`, + } } return { text } diff --git a/trios/agent-server/apps/server/tests/tools/filesystem/read.test.ts b/trios/agent-server/apps/server/tests/tools/filesystem/read.test.ts index c6ca8a774..db30fdbdb 100644 --- a/trios/agent-server/apps/server/tests/tools/filesystem/read.test.ts +++ b/trios/agent-server/apps/server/tests/tools/filesystem/read.test.ts @@ -154,6 +154,25 @@ describe('filesystem_read', () => { expect(result.text).toContain(`${MAX_READ_CHARS}-character limit`) }) + it('returns what fits instead of refusing, with the offset to continue', async () => { + // A refusal costs a provider call and returns nothing. Measured on the + // running deployment 2026-09-20: 18 of 19 filesystem failures in one window + // were this refusal, on files of 159 and 500 lines. + const line = 'y'.repeat(200) + const lines = Array.from({ length: 200 }, () => line) + await writeFile(join(tmpDir, 'wide.txt'), lines.join('\n')) + const result = await exec({ path: 'wide.txt', limit: 200 }) + expect(result.isError).toBeFalsy() + expect(result.text.length).toBeLessThanOrEqual(MAX_READ_CHARS) + expect(result.text).toContain('cut at the') + const offset = Number(/offset=(\d+)/.exec(result.text)?.[1]) + expect(Number.isInteger(offset)).toBe(true) + expect(offset).toBeGreaterThan(1) + // The offset it names must actually be the next unread line. + const rest = await exec({ path: 'wide.txt', offset, limit: 1 }) + expect(rest.text).toContain(`${offset} | `) + }) + it('handles files with UTF-8 BOM', async () => { await writeFile(join(tmpDir, 'bom.txt'), '\uFEFFhello bom') const result = await exec({ path: 'bom.txt' })