diff --git a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts index 177fb39c9..d05c8e18b 100644 --- a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts +++ b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts @@ -100,6 +100,47 @@ describe('createOverloadRetryFetch', () => { assert.strictEqual(bad.calls(), 1) }) + it('retries a fetch that THREW, and gives up with the error it was given', async () => { + // Measured at concurrency four on one key against integrate.api.nvidia.com, + // 2026-09-20: two 200s, one 503, and one socket that never answered. The + // status branches cannot see the fourth - there is no response to branch on. + let calls = 0 + const fetchImpl = (async () => { + calls += 1 + if (calls < 3) throw new Error('read ECONNRESET') + return new Response('{"ok":true}', { + status: 200, + headers: { 'content-type': 'application/json' }, + }) + }) as unknown as typeof fetch + const f = createOverloadRetryFetch({ fetchImpl, sleep: noSleep }) + assert.strictEqual((await f('https://example.test/', {})).status, 200) + assert.strictEqual(calls, 3) + + let always = 0 + const dead = (async () => { + always += 1 + throw new Error('read ECONNRESET') + }) as unknown as typeof fetch + const g = createOverloadRetryFetch({ fetchImpl: dead, sleep: noSleep, maxAttempts: 3 }) + await assert.rejects(() => g('https://example.test/', {}), /ECONNRESET/) + assert.strictEqual(always, 3) + }) + + it('does not retry an abort the caller asked for', async () => { + // Retrying a cancelled request outlives the thing that cancelled it. + let calls = 0 + const aborting = (async () => { + calls += 1 + const error = new Error('aborted') + error.name = 'AbortError' + throw error + }) as unknown as typeof fetch + const f = createOverloadRetryFetch({ fetchImpl: aborting, sleep: noSleep }) + await assert.rejects(() => f('https://example.test/', {}), /aborted/) + assert.strictEqual(calls, 1) + }) + it('replays a stream that arrives in many small chunks', async () => { const encoder = new TextEncoder() const pieces = GOOD.match(/.{1,7}/gs) ?? [] diff --git a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts index 7508b5fd6..e2c07a69f 100644 --- a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts +++ b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts @@ -130,6 +130,14 @@ function replay( }) } +/** An abort the caller asked for, rather than a network failure. */ +export function isAbort(error: unknown): boolean { + if (typeof error !== 'object' || error === null) return false + const name = (error as { name?: unknown }).name + return name === 'AbortError' || name === 'TimeoutError' +} + + export function createOverloadRetryFetch( options: OverloadRetryOptions = {}, ): typeof fetch { @@ -141,10 +149,33 @@ export function createOverloadRetryFetch( return (async (url: RequestInfo | URL, init?: RequestInit) => { const signal = init?.signal for (let attempt = 1; ; attempt++) { - const response = await fetchImpl(url, init) const last = attempt >= maxAttempts const delayMs = delays[Math.min(attempt - 1, delays.length - 1)] ?? 0 + // A THROW is the same outage wearing different clothes. Measured against + // integrate.api.nvidia.com on 2026-09-20 at concurrency four on one key: + // two answers 200, one 503, and one that never answered at all - the + // socket simply hung until the client gave up. The status branches below + // never saw that fourth one, because there was no response to branch on, + // so it reached the agent loop as a terminal error and ended the turn. + // + // An abort the CALLER asked for is not an outage and is re-thrown at + // once: retrying a cancelled request would outlive the thing that + // cancelled it. + let response: Response + try { + response = await fetchImpl(url, init) + } catch (error) { + if (signal?.aborted || isAbort(error) || last) throw error + options.onRetry?.({ + attempt, + delayMs, + reason: `fetch threw: ${error instanceof Error ? error.message : String(error)}`, + }) + await sleep(delayMs, signal) + continue + } + if (response.status === 429 || response.status >= 500) { if (last) return response await response.body?.cancel().catch(() => {}) diff --git a/trios/agent-server/docker-entrypoint.sh b/trios/agent-server/docker-entrypoint.sh index 320ba1fe9..af7964ea9 100755 --- a/trios/agent-server/docker-entrypoint.sh +++ b/trios/agent-server/docker-entrypoint.sh @@ -206,6 +206,52 @@ if [ -n "$TRIOS_TOOL_SHELL_USER" ] && id "$TRIOS_TOOL_SHELL_USER" >/dev/null 2>& echo "[entrypoint] $WORKSPACE_DIR is not owned by $TRIOS_TOOL_SHELL_USER; settling ownership once" chown -R "$TRIOS_TOOL_SHELL_USER" "$WORKSPACE_DIR" fi + # AND INSIDE THE CHECKOUT, which the test above cannot see. Measured on the + # running deployment 2026-09-20: + # + # error: Your local changes to the following files would be overwritten by checkout: + # error: unable to create file specs/port/tools/gft_deep_demo.t27: Permission denied + # + # $WORKSPACE_DIR was owned by the bee, so the walk above was skipped, while + # files underneath were not - left by a root-run git from an older image. A + # bee that cannot write the checkout produces an EMPTY branch and a turn that + # looks like a model failure: 80 of 101 stuck issues on that day had one. + # + # `find ! -user -print -quit` stops at the FIRST wrong file, so the healthy + # case costs one stat and the 45 GB walk that once outlasted the 300 s + # healthcheck (2026-09-03) cannot come back. The repair walks the checkout + # only - never the worktrees beside it - and changes only what is wrong. + if [ -d "$REPO_DIR" ] && [ -n "$(find "$REPO_DIR" ! -user "$TRIOS_TOOL_SHELL_USER" -print -quit 2>/dev/null)" ]; then + echo "[entrypoint] files inside $REPO_DIR are not owned by $TRIOS_TOOL_SHELL_USER; repairing those" + find "$REPO_DIR" ! -user "$TRIOS_TOOL_SHELL_USER" -exec chown "$TRIOS_TOOL_SHELL_USER" {} + 2>/dev/null || true + fi + # BEE WORKTREES FROM A PREVIOUS LIFE. At entrypoint time no bee is running - + # this process is what starts the server that starts them - so every + # directory under .worktrees/ belongs to a container that is already gone. + # + # They are not free. Measured on the running deployment 2026-09-20: every + # tick for six minutes chose an issue and then refused to start it - + # + # Queen tick chose an issue but the container cannot carry another bee + # issue=4438 resource="disk" + # + # - with twenty lanes open and 684 candidates waiting. The volume had filled + # with worktrees nobody could use; an earlier reading of the same volume + # found 41 of them holding 45 GB and three million inodes. + # + # `git worktree prune` alone does not do it: it drops the ADMIN records for + # directories that are already gone, and these directories are still there. + if [ -d "$REPO_DIR/.worktrees" ]; then + stale=$(ls -1 "$REPO_DIR/.worktrees" 2>/dev/null | wc -l | tr -d ' ') + if [ "$stale" != "0" ]; then + free_before=$(df -Pm "$REPO_DIR" 2>/dev/null | awk 'NR==2 {print $4}') + echo "[entrypoint] removing $stale bee worktree(s) left by a previous container (${free_before:-?} MiB free)" + $AS_USER "rm -rf '$REPO_DIR/.worktrees'/* '$REPO_DIR/.worktrees'/.[!.]*" 2>/dev/null || true + $AS_USER "git -C '$REPO_DIR' worktree prune" >/dev/null 2>&1 || true + free_after=$(df -Pm "$REPO_DIR" 2>/dev/null | awk 'NR==2 {print $4}') + echo "[entrypoint] worktrees cleared; ${free_after:-?} MiB free now" + fi + fi echo "[entrypoint] git runs as $TRIOS_TOOL_SHELL_USER; root does not enter the checkout" else AS_USER="sh -c"