From 20c6eb0b6a29f0e54b40fb13adbde46b6467d3a0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 20 Sep 2026 20:57:40 +0700 Subject: [PATCH 1/2] fix(queen): a bee that cannot write the checkout, and a fetch that threw TWO FAILURES WITH ONE SYMPTOM: an empty branch and a turn that reads as a model failure. 80 of 101 stuck issues on 2026-09-20 had exactly that shape. THE CHECKOUT. Measured in the running deployment's log: error: Your local changes to the following files would be overwritten by checkout: error: unable to create file specs/port/tools/gft_deep_demo.t27: Permission denied $WORKSPACE_DIR was owned by the bee, so the one-time ownership walk was skipped, while files underneath it were not - left by a root-run git from an older image. `find ! -user -print -quit` stops at the FIRST wrong file, so the healthy case costs one stat and the 45 GB walk that once outlasted the 300 s healthcheck cannot come back. The repair walks the checkout only, never the worktrees beside it, and changes only what is wrong. THE FETCH. Measured at concurrency four on one key against integrate.api.nvidia.com: two answers 200, one 503, and one socket that never answered at all. The retry wrapper handled 429 and 5xx and the 200-carrying-an- error case, and could not see the fourth - there is no response to branch on, so it reached the agent loop as a terminal error and ended the turn. A throw is now retried on the same backoff. An abort the CALLER asked for is re-thrown at once: retrying a cancelled request outlives the thing that cancelled it. bun test apps/server/src/lib/overload-retry-fetch.test.ts 10 pass, 0 fail Co-Authored-By: Claude Opus 5 --- .../src/lib/overload-retry-fetch.test.ts | 41 +++++++++++++++++++ .../server/src/lib/overload-retry-fetch.ts | 33 ++++++++++++++- trios/agent-server/docker-entrypoint.sh | 19 +++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts index 177fb39c9..d05c8e18b 100644 --- a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts +++ b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts @@ -100,6 +100,47 @@ describe('createOverloadRetryFetch', () => { assert.strictEqual(bad.calls(), 1) }) + it('retries a fetch that THREW, and gives up with the error it was given', async () => { + // Measured at concurrency four on one key against integrate.api.nvidia.com, + // 2026-09-20: two 200s, one 503, and one socket that never answered. The + // status branches cannot see the fourth - there is no response to branch on. + let calls = 0 + const fetchImpl = (async () => { + calls += 1 + if (calls < 3) throw new Error('read ECONNRESET') + return new Response('{"ok":true}', { + status: 200, + headers: { 'content-type': 'application/json' }, + }) + }) as unknown as typeof fetch + const f = createOverloadRetryFetch({ fetchImpl, sleep: noSleep }) + assert.strictEqual((await f('https://example.test/', {})).status, 200) + assert.strictEqual(calls, 3) + + let always = 0 + const dead = (async () => { + always += 1 + throw new Error('read ECONNRESET') + }) as unknown as typeof fetch + const g = createOverloadRetryFetch({ fetchImpl: dead, sleep: noSleep, maxAttempts: 3 }) + await assert.rejects(() => g('https://example.test/', {}), /ECONNRESET/) + assert.strictEqual(always, 3) + }) + + it('does not retry an abort the caller asked for', async () => { + // Retrying a cancelled request outlives the thing that cancelled it. + let calls = 0 + const aborting = (async () => { + calls += 1 + const error = new Error('aborted') + error.name = 'AbortError' + throw error + }) as unknown as typeof fetch + const f = createOverloadRetryFetch({ fetchImpl: aborting, sleep: noSleep }) + await assert.rejects(() => f('https://example.test/', {}), /aborted/) + assert.strictEqual(calls, 1) + }) + it('replays a stream that arrives in many small chunks', async () => { const encoder = new TextEncoder() const pieces = GOOD.match(/.{1,7}/gs) ?? [] diff --git a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts index 7508b5fd6..e2c07a69f 100644 --- a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts +++ b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts @@ -130,6 +130,14 @@ function replay( }) } +/** An abort the caller asked for, rather than a network failure. */ +export function isAbort(error: unknown): boolean { + if (typeof error !== 'object' || error === null) return false + const name = (error as { name?: unknown }).name + return name === 'AbortError' || name === 'TimeoutError' +} + + export function createOverloadRetryFetch( options: OverloadRetryOptions = {}, ): typeof fetch { @@ -141,10 +149,33 @@ export function createOverloadRetryFetch( return (async (url: RequestInfo | URL, init?: RequestInit) => { const signal = init?.signal for (let attempt = 1; ; attempt++) { - const response = await fetchImpl(url, init) const last = attempt >= maxAttempts const delayMs = delays[Math.min(attempt - 1, delays.length - 1)] ?? 0 + // A THROW is the same outage wearing different clothes. Measured against + // integrate.api.nvidia.com on 2026-09-20 at concurrency four on one key: + // two answers 200, one 503, and one that never answered at all - the + // socket simply hung until the client gave up. The status branches below + // never saw that fourth one, because there was no response to branch on, + // so it reached the agent loop as a terminal error and ended the turn. + // + // An abort the CALLER asked for is not an outage and is re-thrown at + // once: retrying a cancelled request would outlive the thing that + // cancelled it. + let response: Response + try { + response = await fetchImpl(url, init) + } catch (error) { + if (signal?.aborted || isAbort(error) || last) throw error + options.onRetry?.({ + attempt, + delayMs, + reason: `fetch threw: ${error instanceof Error ? error.message : String(error)}`, + }) + await sleep(delayMs, signal) + continue + } + if (response.status === 429 || response.status >= 500) { if (last) return response await response.body?.cancel().catch(() => {}) diff --git a/trios/agent-server/docker-entrypoint.sh b/trios/agent-server/docker-entrypoint.sh index 320ba1fe9..26d6937fd 100755 --- a/trios/agent-server/docker-entrypoint.sh +++ b/trios/agent-server/docker-entrypoint.sh @@ -206,6 +206,25 @@ if [ -n "$TRIOS_TOOL_SHELL_USER" ] && id "$TRIOS_TOOL_SHELL_USER" >/dev/null 2>& echo "[entrypoint] $WORKSPACE_DIR is not owned by $TRIOS_TOOL_SHELL_USER; settling ownership once" chown -R "$TRIOS_TOOL_SHELL_USER" "$WORKSPACE_DIR" fi + # AND INSIDE THE CHECKOUT, which the test above cannot see. Measured on the + # running deployment 2026-09-20: + # + # error: Your local changes to the following files would be overwritten by checkout: + # error: unable to create file specs/port/tools/gft_deep_demo.t27: Permission denied + # + # $WORKSPACE_DIR was owned by the bee, so the walk above was skipped, while + # files underneath were not - left by a root-run git from an older image. A + # bee that cannot write the checkout produces an EMPTY branch and a turn that + # looks like a model failure: 80 of 101 stuck issues on that day had one. + # + # `find ! -user -print -quit` stops at the FIRST wrong file, so the healthy + # case costs one stat and the 45 GB walk that once outlasted the 300 s + # healthcheck (2026-09-03) cannot come back. The repair walks the checkout + # only - never the worktrees beside it - and changes only what is wrong. + if [ -d "$REPO_DIR" ] && [ -n "$(find "$REPO_DIR" ! -user "$TRIOS_TOOL_SHELL_USER" -print -quit 2>/dev/null)" ]; then + echo "[entrypoint] files inside $REPO_DIR are not owned by $TRIOS_TOOL_SHELL_USER; repairing those" + find "$REPO_DIR" ! -user "$TRIOS_TOOL_SHELL_USER" -exec chown "$TRIOS_TOOL_SHELL_USER" {} + 2>/dev/null || true + fi echo "[entrypoint] git runs as $TRIOS_TOOL_SHELL_USER; root does not enter the checkout" else AS_USER="sh -c" From 24bcc909b53c082228e1fab7b13b730f8afaa0f7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 20 Sep 2026 21:23:13 +0700 Subject: [PATCH 2/2] fix(queen): clear the worktrees of containers that are already gone Every tick for six minutes chose an issue and then refused to start it: Queen tick chose an issue but the container cannot carry another bee issue=4438 resource="disk" Twenty lanes open, 684 candidates waiting, and zero bees - because the volume had filled with bee worktrees nobody could use. An earlier reading of the same volume found 41 of them holding 45 GB and three million inodes. At entrypoint time no bee is running - this process is what starts the server that starts them - so every directory under .worktrees/ belongs to a container that is already gone. `git worktree prune` alone does not do it: it drops the admin records for directories already removed, and these are still there. Free space is printed before and after, so the next reader sees what it bought. Co-Authored-By: Claude Opus 5 --- trios/agent-server/docker-entrypoint.sh | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/trios/agent-server/docker-entrypoint.sh b/trios/agent-server/docker-entrypoint.sh index 26d6937fd..af7964ea9 100755 --- a/trios/agent-server/docker-entrypoint.sh +++ b/trios/agent-server/docker-entrypoint.sh @@ -225,6 +225,33 @@ if [ -n "$TRIOS_TOOL_SHELL_USER" ] && id "$TRIOS_TOOL_SHELL_USER" >/dev/null 2>& echo "[entrypoint] files inside $REPO_DIR are not owned by $TRIOS_TOOL_SHELL_USER; repairing those" find "$REPO_DIR" ! -user "$TRIOS_TOOL_SHELL_USER" -exec chown "$TRIOS_TOOL_SHELL_USER" {} + 2>/dev/null || true fi + # BEE WORKTREES FROM A PREVIOUS LIFE. At entrypoint time no bee is running - + # this process is what starts the server that starts them - so every + # directory under .worktrees/ belongs to a container that is already gone. + # + # They are not free. Measured on the running deployment 2026-09-20: every + # tick for six minutes chose an issue and then refused to start it - + # + # Queen tick chose an issue but the container cannot carry another bee + # issue=4438 resource="disk" + # + # - with twenty lanes open and 684 candidates waiting. The volume had filled + # with worktrees nobody could use; an earlier reading of the same volume + # found 41 of them holding 45 GB and three million inodes. + # + # `git worktree prune` alone does not do it: it drops the ADMIN records for + # directories that are already gone, and these directories are still there. + if [ -d "$REPO_DIR/.worktrees" ]; then + stale=$(ls -1 "$REPO_DIR/.worktrees" 2>/dev/null | wc -l | tr -d ' ') + if [ "$stale" != "0" ]; then + free_before=$(df -Pm "$REPO_DIR" 2>/dev/null | awk 'NR==2 {print $4}') + echo "[entrypoint] removing $stale bee worktree(s) left by a previous container (${free_before:-?} MiB free)" + $AS_USER "rm -rf '$REPO_DIR/.worktrees'/* '$REPO_DIR/.worktrees'/.[!.]*" 2>/dev/null || true + $AS_USER "git -C '$REPO_DIR' worktree prune" >/dev/null 2>&1 || true + free_after=$(df -Pm "$REPO_DIR" 2>/dev/null | awk 'NR==2 {print $4}') + echo "[entrypoint] worktrees cleared; ${free_after:-?} MiB free now" + fi + fi echo "[entrypoint] git runs as $TRIOS_TOOL_SHELL_USER; root does not enter the checkout" else AS_USER="sh -c"