From 20c6eb0b6a29f0e54b40fb13adbde46b6467d3a0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 20 Sep 2026 20:57:40 +0700 Subject: [PATCH 1/6] fix(queen): a bee that cannot write the checkout, and a fetch that threw TWO FAILURES WITH ONE SYMPTOM: an empty branch and a turn that reads as a model failure. 80 of 101 stuck issues on 2026-09-20 had exactly that shape. THE CHECKOUT. Measured in the running deployment's log: error: Your local changes to the following files would be overwritten by checkout: error: unable to create file specs/port/tools/gft_deep_demo.t27: Permission denied $WORKSPACE_DIR was owned by the bee, so the one-time ownership walk was skipped, while files underneath it were not - left by a root-run git from an older image. `find ! -user -print -quit` stops at the FIRST wrong file, so the healthy case costs one stat and the 45 GB walk that once outlasted the 300 s healthcheck cannot come back. The repair walks the checkout only, never the worktrees beside it, and changes only what is wrong. THE FETCH. Measured at concurrency four on one key against integrate.api.nvidia.com: two answers 200, one 503, and one socket that never answered at all. The retry wrapper handled 429 and 5xx and the 200-carrying-an- error case, and could not see the fourth - there is no response to branch on, so it reached the agent loop as a terminal error and ended the turn. A throw is now retried on the same backoff. An abort the CALLER asked for is re-thrown at once: retrying a cancelled request outlives the thing that cancelled it. bun test apps/server/src/lib/overload-retry-fetch.test.ts 10 pass, 0 fail Co-Authored-By: Claude Opus 5 --- .../src/lib/overload-retry-fetch.test.ts | 41 +++++++++++++++++++ .../server/src/lib/overload-retry-fetch.ts | 33 ++++++++++++++- trios/agent-server/docker-entrypoint.sh | 19 +++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts index 177fb39c9..d05c8e18b 100644 --- a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts +++ b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts @@ -100,6 +100,47 @@ describe('createOverloadRetryFetch', () => { assert.strictEqual(bad.calls(), 1) }) + it('retries a fetch that THREW, and gives up with the error it was given', async () => { + // Measured at concurrency four on one key against integrate.api.nvidia.com, + // 2026-09-20: two 200s, one 503, and one socket that never answered. The + // status branches cannot see the fourth - there is no response to branch on. + let calls = 0 + const fetchImpl = (async () => { + calls += 1 + if (calls < 3) throw new Error('read ECONNRESET') + return new Response('{"ok":true}', { + status: 200, + headers: { 'content-type': 'application/json' }, + }) + }) as unknown as typeof fetch + const f = createOverloadRetryFetch({ fetchImpl, sleep: noSleep }) + assert.strictEqual((await f('https://example.test/', {})).status, 200) + assert.strictEqual(calls, 3) + + let always = 0 + const dead = (async () => { + always += 1 + throw new Error('read ECONNRESET') + }) as unknown as typeof fetch + const g = createOverloadRetryFetch({ fetchImpl: dead, sleep: noSleep, maxAttempts: 3 }) + await assert.rejects(() => g('https://example.test/', {}), /ECONNRESET/) + assert.strictEqual(always, 3) + }) + + it('does not retry an abort the caller asked for', async () => { + // Retrying a cancelled request outlives the thing that cancelled it. + let calls = 0 + const aborting = (async () => { + calls += 1 + const error = new Error('aborted') + error.name = 'AbortError' + throw error + }) as unknown as typeof fetch + const f = createOverloadRetryFetch({ fetchImpl: aborting, sleep: noSleep }) + await assert.rejects(() => f('https://example.test/', {}), /aborted/) + assert.strictEqual(calls, 1) + }) + it('replays a stream that arrives in many small chunks', async () => { const encoder = new TextEncoder() const pieces = GOOD.match(/.{1,7}/gs) ?? [] diff --git a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts index 7508b5fd6..e2c07a69f 100644 --- a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts +++ b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts @@ -130,6 +130,14 @@ function replay( }) } +/** An abort the caller asked for, rather than a network failure. */ +export function isAbort(error: unknown): boolean { + if (typeof error !== 'object' || error === null) return false + const name = (error as { name?: unknown }).name + return name === 'AbortError' || name === 'TimeoutError' +} + + export function createOverloadRetryFetch( options: OverloadRetryOptions = {}, ): typeof fetch { @@ -141,10 +149,33 @@ export function createOverloadRetryFetch( return (async (url: RequestInfo | URL, init?: RequestInit) => { const signal = init?.signal for (let attempt = 1; ; attempt++) { - const response = await fetchImpl(url, init) const last = attempt >= maxAttempts const delayMs = delays[Math.min(attempt - 1, delays.length - 1)] ?? 0 + // A THROW is the same outage wearing different clothes. Measured against + // integrate.api.nvidia.com on 2026-09-20 at concurrency four on one key: + // two answers 200, one 503, and one that never answered at all - the + // socket simply hung until the client gave up. The status branches below + // never saw that fourth one, because there was no response to branch on, + // so it reached the agent loop as a terminal error and ended the turn. + // + // An abort the CALLER asked for is not an outage and is re-thrown at + // once: retrying a cancelled request would outlive the thing that + // cancelled it. + let response: Response + try { + response = await fetchImpl(url, init) + } catch (error) { + if (signal?.aborted || isAbort(error) || last) throw error + options.onRetry?.({ + attempt, + delayMs, + reason: `fetch threw: ${error instanceof Error ? error.message : String(error)}`, + }) + await sleep(delayMs, signal) + continue + } + if (response.status === 429 || response.status >= 500) { if (last) return response await response.body?.cancel().catch(() => {}) diff --git a/trios/agent-server/docker-entrypoint.sh b/trios/agent-server/docker-entrypoint.sh index 320ba1fe9..26d6937fd 100755 --- a/trios/agent-server/docker-entrypoint.sh +++ b/trios/agent-server/docker-entrypoint.sh @@ -206,6 +206,25 @@ if [ -n "$TRIOS_TOOL_SHELL_USER" ] && id "$TRIOS_TOOL_SHELL_USER" >/dev/null 2>& echo "[entrypoint] $WORKSPACE_DIR is not owned by $TRIOS_TOOL_SHELL_USER; settling ownership once" chown -R "$TRIOS_TOOL_SHELL_USER" "$WORKSPACE_DIR" fi + # AND INSIDE THE CHECKOUT, which the test above cannot see. Measured on the + # running deployment 2026-09-20: + # + # error: Your local changes to the following files would be overwritten by checkout: + # error: unable to create file specs/port/tools/gft_deep_demo.t27: Permission denied + # + # $WORKSPACE_DIR was owned by the bee, so the walk above was skipped, while + # files underneath were not - left by a root-run git from an older image. A + # bee that cannot write the checkout produces an EMPTY branch and a turn that + # looks like a model failure: 80 of 101 stuck issues on that day had one. + # + # `find ! -user -print -quit` stops at the FIRST wrong file, so the healthy + # case costs one stat and the 45 GB walk that once outlasted the 300 s + # healthcheck (2026-09-03) cannot come back. The repair walks the checkout + # only - never the worktrees beside it - and changes only what is wrong. + if [ -d "$REPO_DIR" ] && [ -n "$(find "$REPO_DIR" ! -user "$TRIOS_TOOL_SHELL_USER" -print -quit 2>/dev/null)" ]; then + echo "[entrypoint] files inside $REPO_DIR are not owned by $TRIOS_TOOL_SHELL_USER; repairing those" + find "$REPO_DIR" ! -user "$TRIOS_TOOL_SHELL_USER" -exec chown "$TRIOS_TOOL_SHELL_USER" {} + 2>/dev/null || true + fi echo "[entrypoint] git runs as $TRIOS_TOOL_SHELL_USER; root does not enter the checkout" else AS_USER="sh -c" From 24bcc909b53c082228e1fab7b13b730f8afaa0f7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 20 Sep 2026 21:23:13 +0700 Subject: [PATCH 2/6] fix(queen): clear the worktrees of containers that are already gone Every tick for six minutes chose an issue and then refused to start it: Queen tick chose an issue but the container cannot carry another bee issue=4438 resource="disk" Twenty lanes open, 684 candidates waiting, and zero bees - because the volume had filled with bee worktrees nobody could use. An earlier reading of the same volume found 41 of them holding 45 GB and three million inodes. At entrypoint time no bee is running - this process is what starts the server that starts them - so every directory under .worktrees/ belongs to a container that is already gone. `git worktree prune` alone does not do it: it drops the admin records for directories already removed, and these are still there. Free space is printed before and after, so the next reader sees what it bought. Co-Authored-By: Claude Opus 5 --- trios/agent-server/docker-entrypoint.sh | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/trios/agent-server/docker-entrypoint.sh b/trios/agent-server/docker-entrypoint.sh index 26d6937fd..af7964ea9 100755 --- a/trios/agent-server/docker-entrypoint.sh +++ b/trios/agent-server/docker-entrypoint.sh @@ -225,6 +225,33 @@ if [ -n "$TRIOS_TOOL_SHELL_USER" ] && id "$TRIOS_TOOL_SHELL_USER" >/dev/null 2>& echo "[entrypoint] files inside $REPO_DIR are not owned by $TRIOS_TOOL_SHELL_USER; repairing those" find "$REPO_DIR" ! -user "$TRIOS_TOOL_SHELL_USER" -exec chown "$TRIOS_TOOL_SHELL_USER" {} + 2>/dev/null || true fi + # BEE WORKTREES FROM A PREVIOUS LIFE. At entrypoint time no bee is running - + # this process is what starts the server that starts them - so every + # directory under .worktrees/ belongs to a container that is already gone. + # + # They are not free. Measured on the running deployment 2026-09-20: every + # tick for six minutes chose an issue and then refused to start it - + # + # Queen tick chose an issue but the container cannot carry another bee + # issue=4438 resource="disk" + # + # - with twenty lanes open and 684 candidates waiting. The volume had filled + # with worktrees nobody could use; an earlier reading of the same volume + # found 41 of them holding 45 GB and three million inodes. + # + # `git worktree prune` alone does not do it: it drops the ADMIN records for + # directories that are already gone, and these directories are still there. + if [ -d "$REPO_DIR/.worktrees" ]; then + stale=$(ls -1 "$REPO_DIR/.worktrees" 2>/dev/null | wc -l | tr -d ' ') + if [ "$stale" != "0" ]; then + free_before=$(df -Pm "$REPO_DIR" 2>/dev/null | awk 'NR==2 {print $4}') + echo "[entrypoint] removing $stale bee worktree(s) left by a previous container (${free_before:-?} MiB free)" + $AS_USER "rm -rf '$REPO_DIR/.worktrees'/* '$REPO_DIR/.worktrees'/.[!.]*" 2>/dev/null || true + $AS_USER "git -C '$REPO_DIR' worktree prune" >/dev/null 2>&1 || true + free_after=$(df -Pm "$REPO_DIR" 2>/dev/null | awk 'NR==2 {print $4}') + echo "[entrypoint] worktrees cleared; ${free_after:-?} MiB free now" + fi + fi echo "[entrypoint] git runs as $TRIOS_TOOL_SHELL_USER; root does not enter the checkout" else AS_USER="sh -c" From 5fa6fd0ea973b16ddbf69c4438ee768abe4a7a1e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 20 Sep 2026 22:02:14 +0700 Subject: [PATCH 3/6] fix(tools): a read that is too large returns what fits, not a refusal A refusal costs a provider call and returns nothing, and the provider is the ceiling. Measured on the running deployment 2026-09-20 in one 400-line window: 18 of 19 filesystem tool failures were this refusal, on files of 159 and 500 lines - ordinary specs - while the same endpoint answered `Service temporarily overloaded` 76 times in that same window. Every one of those refusals spent a round trip to be told to ask again. filesystem_read now returns the lines that fit under the character limit and names the exact offset to continue from, which is what the caller would have asked for on its second call. Room is kept for that note, so the answer can always carry one. A single line longer than the whole budget still throws - there is nothing to hand back - and now says so in those words, pointing at filesystem_grep. bun test apps/server/tests/tools/filesystem/read.test.ts 17 pass, 0 fail Co-Authored-By: Claude Opus 5 --- .../apps/server/src/tools/filesystem/read.ts | 49 +++++++++++++++++-- .../tests/tools/filesystem/read.test.ts | 19 +++++++ 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/trios/agent-server/apps/server/src/tools/filesystem/read.ts b/trios/agent-server/apps/server/src/tools/filesystem/read.ts index 079156040..f2c4c9527 100644 --- a/trios/agent-server/apps/server/src/tools/filesystem/read.ts +++ b/trios/agent-server/apps/server/src/tools/filesystem/read.ts @@ -53,6 +53,9 @@ function getSelectedLines( return remaining } +/** Room kept for the continuation note, so the answer can always carry one. */ +const CONTINUATION_NOTE_BUDGET = 160 + function formatReadResult(args: { selected: string[] startIdx: number @@ -76,10 +79,50 @@ function formatReadResult(args: { text += `\n\n(Showing lines ${startLineNum}-${endLineNum} of ${args.totalLines})` } + // A REFUSAL COSTS A PROVIDER CALL, AND THE PROVIDER IS THE CEILING. + // + // This used to throw, and the agent's only recovery was to ask again with a + // smaller range. Measured on the running deployment 2026-09-20: 18 of 19 + // filesystem tool failures in one window were this refusal, on files of 159 + // and 500 lines - ordinary specs. Each one spent a round trip on the same + // endpoint that was answering `Service temporarily overloaded` 76 times in + // the same window, and returned no content at all. + // + // So it returns what FITS, and says exactly where to continue. The caller + // gets content on the first call and a correct `offset` for the rest, which + // is what it would have asked for on the second. if (text.length > MAX_READ_CHARS) { - throw new Error( - `Requested lines ${startLineNum}-${endLineNum} produce ${text.length} characters in the response, above the ${MAX_READ_CHARS}-character limit for filesystem_read. Retry with a smaller limit or a later offset.`, - ) + const kept: string[] = [] + let used = 0 + for (let i = 0; i < args.selected.length; i++) { + const rendered = `${String(args.startIdx + i + 1).padStart(width)} | ${args.selected[i]}\n` + // Leave room for the continuation note, which is what makes the answer + // usable rather than merely shorter. + if (used + rendered.length > MAX_READ_CHARS - CONTINUATION_NOTE_BUDGET) + break + used += rendered.length + kept.push(args.selected[i]) + } + if (kept.length === 0) { + // One line longer than the whole budget. Nothing to hand back, and the + // caller needs to hear why rather than get an empty answer. + throw new Error( + `Line ${startLineNum} alone is ${args.selected[0]?.length ?? 0} characters, above the ${MAX_READ_CHARS}-character limit for filesystem_read. Use filesystem_grep to find what you need in it.`, + ) + } + const cutAt = args.startIdx + kept.length + const shortened = kept + .map( + (line, i) => + `${String(args.startIdx + i + 1).padStart(width)} | ${line}`, + ) + .join('\n') + return { + text: + shortened + + `\n\n(${args.totalLines - cutAt} more lines in file; this answer was ` + + `cut at the ${MAX_READ_CHARS}-character limit. Use offset=${cutAt + 1} to continue reading.)`, + } } return { text } diff --git a/trios/agent-server/apps/server/tests/tools/filesystem/read.test.ts b/trios/agent-server/apps/server/tests/tools/filesystem/read.test.ts index c6ca8a774..db30fdbdb 100644 --- a/trios/agent-server/apps/server/tests/tools/filesystem/read.test.ts +++ b/trios/agent-server/apps/server/tests/tools/filesystem/read.test.ts @@ -154,6 +154,25 @@ describe('filesystem_read', () => { expect(result.text).toContain(`${MAX_READ_CHARS}-character limit`) }) + it('returns what fits instead of refusing, with the offset to continue', async () => { + // A refusal costs a provider call and returns nothing. Measured on the + // running deployment 2026-09-20: 18 of 19 filesystem failures in one window + // were this refusal, on files of 159 and 500 lines. + const line = 'y'.repeat(200) + const lines = Array.from({ length: 200 }, () => line) + await writeFile(join(tmpDir, 'wide.txt'), lines.join('\n')) + const result = await exec({ path: 'wide.txt', limit: 200 }) + expect(result.isError).toBeFalsy() + expect(result.text.length).toBeLessThanOrEqual(MAX_READ_CHARS) + expect(result.text).toContain('cut at the') + const offset = Number(/offset=(\d+)/.exec(result.text)?.[1]) + expect(Number.isInteger(offset)).toBe(true) + expect(offset).toBeGreaterThan(1) + // The offset it names must actually be the next unread line. + const rest = await exec({ path: 'wide.txt', offset, limit: 1 }) + expect(rest.text).toContain(`${offset} | `) + }) + it('handles files with UTF-8 BOM', async () => { await writeFile(join(tmpDir, 'bom.txt'), '\uFEFFhello bom') const result = await exec({ path: 'bom.txt' }) From cd9aebf2e5d642d64fdcb517ce3df49364155ffd Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 20 Sep 2026 23:38:57 +0700 Subject: [PATCH 4/6] fix(queen): the sweep is bounded by a share of the tick, not four minutes `runRound` awaits the review sweep before it reaps and before it dispatches, so the sweep's deadline is how long the swarm is willing to hand out no work at all. It was an absolute four minutes against a sixty-second tick - four rounds of silence, with the lease heartbeat reporting health throughout. Measured 2026-09-20: the swarm went from 8.5 to 80 dispatches an hour when the worker model changed, and the review sweep - three a round, unchanged since it was written - fell behind, sixteen to twenty unreviewed all afternoon. Raising the COUNT to eight reviews and six measurements killed the container five minutes later, because a count bounds worktrees and not time: each measurement cuts a temporary worktree and runs up to twenty commands. So the bound is time, and a fraction of the tick: 45 s of a 60 s round, floored at 20 s so a very short tick cannot make review impossible, and ceilinged at the old four minutes for a deployment whose tick is minutes long. The count can now be generous because the clock is what protects the dispatcher. The deadline applies to REVIEWS as well as measurements. A review is a provider call and can sit on its own timeout; bounding only the measurements left the dispatcher waiting on the half that was never counted. bun test queen-tick-sweep queen-review-unjudged queen-adversarial-review queen-criteria-run queend-choose 92 pass, 0 fail Co-Authored-By: Claude Opus 5 --- .../server/src/api/services/queen-tick.ts | 35 ++++++++++++++++-- .../server/tests/api/queen-tick-sweep.test.ts | 36 +++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 trios/agent-server/apps/server/tests/api/queen-tick-sweep.test.ts diff --git a/trios/agent-server/apps/server/src/api/services/queen-tick.ts b/trios/agent-server/apps/server/src/api/services/queen-tick.ts index 865fe648f..1c7cd5415 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-tick.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-tick.ts @@ -2666,7 +2666,10 @@ export async function reviewFinishedDispatches( // board read that hands out work, so an unbounded sweep is a swarm that // dispatches nothing while its lease looks healthy. let measurementsLeft = deps.measurementsPerRound?.() ?? measurementsPerRound() - const measurementDeadline = Date.now() + MEASUREMENT_SWEEP_MS + // The whole sweep, not only its measurements: a review is a provider call and + // can sit on its own timeout, so a budget counted in reviews bounds the + // number and not the wall clock the dispatcher is waiting on. + const sweepDeadline = Date.now() + sweepDeadlineMs(tickIntervalSeconds()) let takenKeys: number[] | null = null const repo = process.env.TRIOS_GITHUB_REPO || 'gHashTag/trios' @@ -2985,7 +2988,7 @@ export async function reviewFinishedDispatches( measurementSkipped = 'the review budget for this round is spent' } else if (measurementsLeft <= 0) { measurementSkipped = 'the measurement budget for this round is spent' - } else if (Date.now() >= measurementDeadline) { + } else if (Date.now() >= sweepDeadline) { measurementSkipped = 'the round has measured for as long as it may' } else { measurementsLeft -= 1 @@ -3134,6 +3137,9 @@ export async function reviewFinishedDispatches( // as one past the review budget does: nothing spent, nothing charged, // measured next round. reviewerSkipped = `the criteria were not measured this round (${measurementSkipped})` + } else if (Date.now() >= sweepDeadline) { + reviewerSkipped = + 'the round has reviewed for as long as it may before dispatching' } else if (reviewsLeft <= 0) { reviewerSkipped = 'the review budget for this round is spent' } else { @@ -3711,6 +3717,31 @@ export const REVIEWER_LANE_TRIES = 3 */ export const MEASUREMENT_SWEEP_MS = 4 * 60 * 1000 +/** + * The share of one tick the sweep may spend before the round hands out work. + * + * Four minutes was an absolute number against a sixty-second tick, so a sweep + * could hold the dispatcher for four rounds while the lease heartbeat reported + * health. Measured 2026-09-20: the swarm went from 8.5 to 80 dispatches an + * hour when the worker model changed, and the review sweep - three a round, + * unchanged - fell behind; raising its COUNT to eight reviews and six + * measurements killed the container five minutes later, because the count + * bounds worktrees and not time. + * + * So the bound is a fraction of the tick, and the count can be generous: at a + * sixty-second tick the sweep reviews for at most forty-five seconds and the + * round always dispatches. `MEASUREMENT_SWEEP_MS` stays as the ceiling for a + * deployment whose tick is minutes long, and the floor keeps a very short tick + * from making review impossible. + */ +export const SWEEP_SHARE_OF_TICK = 0.75 +export const SWEEP_FLOOR_MS = 20 * 1000 + +export function sweepDeadlineMs(intervalSeconds: number): number { + const share = Math.round(intervalSeconds * 1000 * SWEEP_SHARE_OF_TICK) + return Math.max(SWEEP_FLOOR_MS, Math.min(MEASUREMENT_SWEEP_MS, share)) +} + /** * Rounds a bought review may fail to arrive for one commit before a person is * asked. Three, the same order as `FREE_ATTEMPT_CEILING`: enough for a lane to diff --git a/trios/agent-server/apps/server/tests/api/queen-tick-sweep.test.ts b/trios/agent-server/apps/server/tests/api/queen-tick-sweep.test.ts new file mode 100644 index 000000000..632230661 --- /dev/null +++ b/trios/agent-server/apps/server/tests/api/queen-tick-sweep.test.ts @@ -0,0 +1,36 @@ +/** + * @license + * Copyright 2025 BrowserOS + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +import { describe, it } from 'bun:test' +import assert from 'node:assert' +import { + MEASUREMENT_SWEEP_MS, + SWEEP_FLOOR_MS, + sweepDeadlineMs, +} from '../../src/api/services/queen-tick' + +describe('the sweep is bounded by a share of the tick, not by an absolute four minutes', () => { + it('gives a sixty-second tick forty-five seconds', () => { + // The round awaits the sweep before it dispatches, so this is how long the + // swarm is willing to hand out no work at all. + assert.strictEqual(sweepDeadlineMs(60), 45_000) + }) + + it('never exceeds the old ceiling, however long the tick', () => { + assert.strictEqual(sweepDeadlineMs(3600), MEASUREMENT_SWEEP_MS) + assert.ok(sweepDeadlineMs(600) <= MEASUREMENT_SWEEP_MS) + }) + + it('never falls below the floor, however short the tick', () => { + // A five-second tick must not make review impossible. + assert.strictEqual(sweepDeadlineMs(5), SWEEP_FLOOR_MS) + assert.strictEqual(sweepDeadlineMs(0), SWEEP_FLOOR_MS) + }) + + it('grows with the tick between the floor and the ceiling', () => { + assert.ok(sweepDeadlineMs(120) > sweepDeadlineMs(60)) + }) +}) From 745410758d3f08d77cfd4ec75f54c4555b0855d9 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 01:52:41 +0700 Subject: [PATCH 5/6] fix(queen): a worktree the remote already has is not the only copy The emergency sweep keeps any worktree whose branch carries commits the base does not, because the container holds no push credential and such a commit would live in exactly one place. That is right, and it stops being true the moment the branch is on origin at the same commit. Measured 2026-09-20: the entrypoint found TWENTY-ONE worktrees at boot, every one of them kept by that branch of the check. The volume filled, the process was killed, the entrypoint cleared them, the swarm ran for a few minutes and it happened again - six restarts in one afternoon, at four lanes as readily as at twenty, because the leak is per FINISHED bee and not per running one. So the sweep now reads the remote. A branch on origin at the same SHA is published, and its worktree is disk. Reading the remote needs no push credential; if it cannot be read the tree stays, because unreachable is not published for the same reason unreadable is not clean. bun test apps/server/tests/api/queen-dispatch.test.ts 96 pass, 0 fail Co-Authored-By: Claude Opus 5 --- .../server/src/api/services/queen-dispatch.ts | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts index 110b96fa5..f5c17aae7 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts @@ -2850,8 +2850,38 @@ export async function reapWorktrees( 60_000, ) if (ahead.code !== 0 || Number(ahead.out.trim() || '1') > 0) { - result.keptUnpushed.push(c.path) - continue + // UNLESS THE REMOTE ALREADY HAS IT, byte for byte. + // + // "The only copy" is the whole of the argument above, and it stops being + // true the moment the branch is on origin at the same commit. Measured + // 2026-09-20: the container had twenty-one worktrees at boot, every one + // of them kept by this branch of the check, and the volume filled until + // the process was killed - then the entrypoint cleared them, the swarm + // ran for a few minutes, and it happened again. Six such restarts in one + // afternoon, at four lanes as readily as at twenty, because the leak is + // per FINISHED bee and not per running one. + // + // A read of the remote needs no push credential. If it cannot be read, + // the tree stays: unreachable is not published, for the same reason + // unreadable is not clean. + const branchName = c.path.slice(c.path.lastIndexOf('/') + 1) + const localHead = await run('git', ['rev-parse', 'HEAD'], c.path, 60_000) + const remote = await run( + 'git', + ['ls-remote', 'origin', `refs/heads/${branchName}`], + c.path, + 120_000, + ) + const remoteSha = remote.out.trim().split(/\s+/)[0] ?? '' + const published = + localHead.code === 0 && + remote.code === 0 && + remoteSha.length > 0 && + remoteSha === localHead.out.trim() + if (!published) { + result.keptUnpushed.push(c.path) + continue + } } // No `--force`, here or anywhere else in this project. A tree that refuses // to go is a tree a person should look at. From a061f8d20ba31949d39d5bd291205da95d484a39 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 12:51:37 +0700 Subject: [PATCH 6/6] fix(queen): a server that stops answering is ended, so the platform restarts it Measured 2026-09-20/21: the agent server twice stopped answering HTTP WITHOUT EXITING. The edge said `Application failed to respond`, and Railway's restart policy never fired, because a restart policy fires on an exit and there was none. The first time it stayed down for nine hours. Raising restartPolicyMaxRetries did not help the second time, for the same reason. An outside watchdog can redeploy, but only with a platform token. A process inside the container needs nothing but loopback. So the entrypoint no longer `exec`s the server. It starts it, and a second process asks `/health` on 127.0.0.1 every LIVENESS_INTERVAL (30 s) after a LIVENESS_GRACE (240 s) boot allowance. LIVENESS_FAILS (4) misses in a row and the server gets SIGTERM, then SIGKILL; the entrypoint exits non-zero and ON_FAILURE brings the container back - through the boot clean-up that frees the disk. One healthy answer resets the count. python3 does the asking because the image declares it; there is no curl here. Proven before shipping on a real HTTP server frozen with SIGSTOP - alive, not answering, the production failure exactly: [liveness] /health did not answer (1 of 3) [liveness] /health did not answer (2 of 3) [liveness] /health did not answer (3 of 3) [liveness] the server stopped answering; ending it so the platform restarts the container [liveness] the server exited (143); exiting so the platform restarts the container Also restores restartPolicyMaxRetries 10000 in railway.json: it was deployed from an uncommitted working tree and lost on the next branch switch. Co-Authored-By: Claude Opus 5 --- trios/agent-server/docker-entrypoint.sh | 64 +++++++++++++++++++++++-- trios/agent-server/railway.json | 2 +- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/trios/agent-server/docker-entrypoint.sh b/trios/agent-server/docker-entrypoint.sh index af7964ea9..a16807ada 100755 --- a/trios/agent-server/docker-entrypoint.sh +++ b/trios/agent-server/docker-entrypoint.sh @@ -21,6 +21,64 @@ set -e +# --------------------------------------------------------------------------- +# LIVENESS: a server that stops answering is ended, so the platform restarts it. +# +# Measured 2026-09-20/21: the agent server twice stopped answering HTTP WITHOUT +# EXITING - the edge said `Application failed to respond` - and Railway's +# restart policy never fired, because a restart policy fires on an exit and +# there was none. The first time it stayed down for nine hours. Raising +# restartPolicyMaxRetries to 10000 did not help the second time, for the same +# reason. An outside watchdog can redeploy, but only with a platform token; a +# process inside the container needs nothing but the loopback interface. +# +# So the entrypoint no longer `exec`s the server. It starts it, and a second +# process asks `/health` on loopback every LIVENESS_INTERVAL seconds after a +# LIVENESS_GRACE boot allowance. LIVENESS_FAILS answers in a row missing and the +# server is sent SIGTERM, then SIGKILL; the entrypoint then exits non-zero and +# `restartPolicyType: ON_FAILURE` brings the container back - with the boot +# clean-up that frees the disk. python3 does the asking because the image +# declares it; curl is not installed here. +# +# One healthy answer resets the count: a slow minute is not a dead server. +run_supervised() { + "$@" & + server=$! + trap 'kill -TERM "$server" 2>/dev/null' TERM INT + ( + port="${PORT:-8080}" + interval="${LIVENESS_INTERVAL:-30}" + fails_allowed="${LIVENESS_FAILS:-4}" + sleep "${LIVENESS_GRACE:-240}" + fails=0 + while kill -0 "$server" 2>/dev/null; do + if python3 -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:$port/health', timeout=10)" >/dev/null 2>&1; then + fails=0 + else + fails=$((fails + 1)) + echo "[liveness] /health did not answer ($fails of $fails_allowed)" + fi + if [ "$fails" -ge "$fails_allowed" ]; then + echo "[liveness] the server stopped answering; ending it so the platform restarts the container" + kill -TERM "$server" 2>/dev/null || true + sleep 15 + kill -KILL "$server" 2>/dev/null || true + break + fi + sleep "$interval" + done + ) & + set +e + wait "$server" + code=$? + # A server that was ended for not answering exits by signal, which `wait` + # reports as 128+N. Anything but a clean 0 must read as a failure to the + # platform, or ON_FAILURE would not restart it. + [ "$code" -eq 0 ] && code=1 + echo "[liveness] the server exited ($code); exiting so the platform restarts the container" + exit "$code" +} + # --------------------------------------------------------------------------- # HOW MANY BEES: derived from what is connected, not typed by hand. # @@ -176,7 +234,7 @@ echo "[entrypoint] TRIOS_QUEEN_MAX_WORKERS=$TRIOS_QUEEN_MAX_WORKERS (derived)" if [ -z "$TRIOS_REPO_URL" ]; then echo "[entrypoint] TRIOS_REPO_URL unset; starting without a checkout" - exec "$@" + run_supervised "$@" fi WORKSPACE_DIR="${WORKSPACE_DIR:-/workspace}" @@ -326,7 +384,7 @@ else # still arrives. $AS_USER "git clone --filter=blob:none --branch '$TRIOS_REPO_REF' \ '$TRIOS_REPO_URL' '$REPO_DIR'" \ - || { echo "[entrypoint] clone FAILED; starting without a checkout"; exec "$@"; } + || { echo "[entrypoint] clone FAILED; starting without a checkout"; run_supervised "$@"; } fi $AS_USER "git -C '$REPO_DIR' config user.name '${GIT_AUTHOR_NAME:-Trinity Bee}' \ @@ -334,4 +392,4 @@ $AS_USER "git -C '$REPO_DIR' config user.name '${GIT_AUTHOR_NAME:-Trinity Bee}' echo "[entrypoint] checkout ready: $($AS_USER "git -C '$REPO_DIR' rev-parse --short HEAD") on $TRIOS_REPO_REF" echo "[entrypoint] this checkout can read and commit; it cannot push, by design" -exec "$@" +run_supervised "$@" diff --git a/trios/agent-server/railway.json b/trios/agent-server/railway.json index a3f822294..41c7bf3f7 100644 --- a/trios/agent-server/railway.json +++ b/trios/agent-server/railway.json @@ -8,7 +8,7 @@ "healthcheckPath": "/health", "healthcheckTimeout": 300, "restartPolicyType": "ON_FAILURE", - "restartPolicyMaxRetries": 10, + "restartPolicyMaxRetries": 10000, "numReplicas": 1 } }