diff --git a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts index 110b96fa56..f5c17aae7b 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts @@ -2850,8 +2850,38 @@ export async function reapWorktrees( 60_000, ) if (ahead.code !== 0 || Number(ahead.out.trim() || '1') > 0) { - result.keptUnpushed.push(c.path) - continue + // UNLESS THE REMOTE ALREADY HAS IT, byte for byte. + // + // "The only copy" is the whole of the argument above, and it stops being + // true the moment the branch is on origin at the same commit. Measured + // 2026-09-20: the container had twenty-one worktrees at boot, every one + // of them kept by this branch of the check, and the volume filled until + // the process was killed - then the entrypoint cleared them, the swarm + // ran for a few minutes, and it happened again. Six such restarts in one + // afternoon, at four lanes as readily as at twenty, because the leak is + // per FINISHED bee and not per running one. + // + // A read of the remote needs no push credential. If it cannot be read, + // the tree stays: unreachable is not published, for the same reason + // unreadable is not clean. + const branchName = c.path.slice(c.path.lastIndexOf('/') + 1) + const localHead = await run('git', ['rev-parse', 'HEAD'], c.path, 60_000) + const remote = await run( + 'git', + ['ls-remote', 'origin', `refs/heads/${branchName}`], + c.path, + 120_000, + ) + const remoteSha = remote.out.trim().split(/\s+/)[0] ?? '' + const published = + localHead.code === 0 && + remote.code === 0 && + remoteSha.length > 0 && + remoteSha === localHead.out.trim() + if (!published) { + result.keptUnpushed.push(c.path) + continue + } } // No `--force`, here or anywhere else in this project. A tree that refuses // to go is a tree a person should look at. diff --git a/trios/agent-server/apps/server/src/api/services/queen-tick.ts b/trios/agent-server/apps/server/src/api/services/queen-tick.ts index 865fe648f2..1c7cd54152 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-tick.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-tick.ts @@ -2666,7 +2666,10 @@ export async function reviewFinishedDispatches( // board read that hands out work, so an unbounded sweep is a swarm that // dispatches nothing while its lease looks healthy. let measurementsLeft = deps.measurementsPerRound?.() ?? measurementsPerRound() - const measurementDeadline = Date.now() + MEASUREMENT_SWEEP_MS + // The whole sweep, not only its measurements: a review is a provider call and + // can sit on its own timeout, so a budget counted in reviews bounds the + // number and not the wall clock the dispatcher is waiting on. + const sweepDeadline = Date.now() + sweepDeadlineMs(tickIntervalSeconds()) let takenKeys: number[] | null = null const repo = process.env.TRIOS_GITHUB_REPO || 'gHashTag/trios' @@ -2985,7 +2988,7 @@ export async function reviewFinishedDispatches( measurementSkipped = 'the review budget for this round is spent' } else if (measurementsLeft <= 0) { measurementSkipped = 'the measurement budget for this round is spent' - } else if (Date.now() >= measurementDeadline) { + } else if (Date.now() >= sweepDeadline) { measurementSkipped = 'the round has measured for as long as it may' } else { measurementsLeft -= 1 @@ -3134,6 +3137,9 @@ export async function reviewFinishedDispatches( // as one past the review budget does: nothing spent, nothing charged, // measured next round. reviewerSkipped = `the criteria were not measured this round (${measurementSkipped})` + } else if (Date.now() >= sweepDeadline) { + reviewerSkipped = + 'the round has reviewed for as long as it may before dispatching' } else if (reviewsLeft <= 0) { reviewerSkipped = 'the review budget for this round is spent' } else { @@ -3711,6 +3717,31 @@ export const REVIEWER_LANE_TRIES = 3 */ export const MEASUREMENT_SWEEP_MS = 4 * 60 * 1000 +/** + * The share of one tick the sweep may spend before the round hands out work. + * + * Four minutes was an absolute number against a sixty-second tick, so a sweep + * could hold the dispatcher for four rounds while the lease heartbeat reported + * health. Measured 2026-09-20: the swarm went from 8.5 to 80 dispatches an + * hour when the worker model changed, and the review sweep - three a round, + * unchanged - fell behind; raising its COUNT to eight reviews and six + * measurements killed the container five minutes later, because the count + * bounds worktrees and not time. + * + * So the bound is a fraction of the tick, and the count can be generous: at a + * sixty-second tick the sweep reviews for at most forty-five seconds and the + * round always dispatches. `MEASUREMENT_SWEEP_MS` stays as the ceiling for a + * deployment whose tick is minutes long, and the floor keeps a very short tick + * from making review impossible. + */ +export const SWEEP_SHARE_OF_TICK = 0.75 +export const SWEEP_FLOOR_MS = 20 * 1000 + +export function sweepDeadlineMs(intervalSeconds: number): number { + const share = Math.round(intervalSeconds * 1000 * SWEEP_SHARE_OF_TICK) + return Math.max(SWEEP_FLOOR_MS, Math.min(MEASUREMENT_SWEEP_MS, share)) +} + /** * Rounds a bought review may fail to arrive for one commit before a person is * asked. Three, the same order as `FREE_ATTEMPT_CEILING`: enough for a lane to diff --git a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts index 177fb39c91..d05c8e18b2 100644 --- a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts +++ b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.test.ts @@ -100,6 +100,47 @@ describe('createOverloadRetryFetch', () => { assert.strictEqual(bad.calls(), 1) }) + it('retries a fetch that THREW, and gives up with the error it was given', async () => { + // Measured at concurrency four on one key against integrate.api.nvidia.com, + // 2026-09-20: two 200s, one 503, and one socket that never answered. The + // status branches cannot see the fourth - there is no response to branch on. + let calls = 0 + const fetchImpl = (async () => { + calls += 1 + if (calls < 3) throw new Error('read ECONNRESET') + return new Response('{"ok":true}', { + status: 200, + headers: { 'content-type': 'application/json' }, + }) + }) as unknown as typeof fetch + const f = createOverloadRetryFetch({ fetchImpl, sleep: noSleep }) + assert.strictEqual((await f('https://example.test/', {})).status, 200) + assert.strictEqual(calls, 3) + + let always = 0 + const dead = (async () => { + always += 1 + throw new Error('read ECONNRESET') + }) as unknown as typeof fetch + const g = createOverloadRetryFetch({ fetchImpl: dead, sleep: noSleep, maxAttempts: 3 }) + await assert.rejects(() => g('https://example.test/', {}), /ECONNRESET/) + assert.strictEqual(always, 3) + }) + + it('does not retry an abort the caller asked for', async () => { + // Retrying a cancelled request outlives the thing that cancelled it. + let calls = 0 + const aborting = (async () => { + calls += 1 + const error = new Error('aborted') + error.name = 'AbortError' + throw error + }) as unknown as typeof fetch + const f = createOverloadRetryFetch({ fetchImpl: aborting, sleep: noSleep }) + await assert.rejects(() => f('https://example.test/', {}), /aborted/) + assert.strictEqual(calls, 1) + }) + it('replays a stream that arrives in many small chunks', async () => { const encoder = new TextEncoder() const pieces = GOOD.match(/.{1,7}/gs) ?? [] diff --git a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts index 7508b5fd65..e2c07a69f4 100644 --- a/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts +++ b/trios/agent-server/apps/server/src/lib/overload-retry-fetch.ts @@ -130,6 +130,14 @@ function replay( }) } +/** An abort the caller asked for, rather than a network failure. */ +export function isAbort(error: unknown): boolean { + if (typeof error !== 'object' || error === null) return false + const name = (error as { name?: unknown }).name + return name === 'AbortError' || name === 'TimeoutError' +} + + export function createOverloadRetryFetch( options: OverloadRetryOptions = {}, ): typeof fetch { @@ -141,10 +149,33 @@ export function createOverloadRetryFetch( return (async (url: RequestInfo | URL, init?: RequestInit) => { const signal = init?.signal for (let attempt = 1; ; attempt++) { - const response = await fetchImpl(url, init) const last = attempt >= maxAttempts const delayMs = delays[Math.min(attempt - 1, delays.length - 1)] ?? 0 + // A THROW is the same outage wearing different clothes. Measured against + // integrate.api.nvidia.com on 2026-09-20 at concurrency four on one key: + // two answers 200, one 503, and one that never answered at all - the + // socket simply hung until the client gave up. The status branches below + // never saw that fourth one, because there was no response to branch on, + // so it reached the agent loop as a terminal error and ended the turn. + // + // An abort the CALLER asked for is not an outage and is re-thrown at + // once: retrying a cancelled request would outlive the thing that + // cancelled it. + let response: Response + try { + response = await fetchImpl(url, init) + } catch (error) { + if (signal?.aborted || isAbort(error) || last) throw error + options.onRetry?.({ + attempt, + delayMs, + reason: `fetch threw: ${error instanceof Error ? error.message : String(error)}`, + }) + await sleep(delayMs, signal) + continue + } + if (response.status === 429 || response.status >= 500) { if (last) return response await response.body?.cancel().catch(() => {}) diff --git a/trios/agent-server/apps/server/src/tools/filesystem/read.ts b/trios/agent-server/apps/server/src/tools/filesystem/read.ts index 0791560404..f2c4c95270 100644 --- a/trios/agent-server/apps/server/src/tools/filesystem/read.ts +++ b/trios/agent-server/apps/server/src/tools/filesystem/read.ts @@ -53,6 +53,9 @@ function getSelectedLines( return remaining } +/** Room kept for the continuation note, so the answer can always carry one. */ +const CONTINUATION_NOTE_BUDGET = 160 + function formatReadResult(args: { selected: string[] startIdx: number @@ -76,10 +79,50 @@ function formatReadResult(args: { text += `\n\n(Showing lines ${startLineNum}-${endLineNum} of ${args.totalLines})` } + // A REFUSAL COSTS A PROVIDER CALL, AND THE PROVIDER IS THE CEILING. + // + // This used to throw, and the agent's only recovery was to ask again with a + // smaller range. Measured on the running deployment 2026-09-20: 18 of 19 + // filesystem tool failures in one window were this refusal, on files of 159 + // and 500 lines - ordinary specs. Each one spent a round trip on the same + // endpoint that was answering `Service temporarily overloaded` 76 times in + // the same window, and returned no content at all. + // + // So it returns what FITS, and says exactly where to continue. The caller + // gets content on the first call and a correct `offset` for the rest, which + // is what it would have asked for on the second. if (text.length > MAX_READ_CHARS) { - throw new Error( - `Requested lines ${startLineNum}-${endLineNum} produce ${text.length} characters in the response, above the ${MAX_READ_CHARS}-character limit for filesystem_read. Retry with a smaller limit or a later offset.`, - ) + const kept: string[] = [] + let used = 0 + for (let i = 0; i < args.selected.length; i++) { + const rendered = `${String(args.startIdx + i + 1).padStart(width)} | ${args.selected[i]}\n` + // Leave room for the continuation note, which is what makes the answer + // usable rather than merely shorter. + if (used + rendered.length > MAX_READ_CHARS - CONTINUATION_NOTE_BUDGET) + break + used += rendered.length + kept.push(args.selected[i]) + } + if (kept.length === 0) { + // One line longer than the whole budget. Nothing to hand back, and the + // caller needs to hear why rather than get an empty answer. + throw new Error( + `Line ${startLineNum} alone is ${args.selected[0]?.length ?? 0} characters, above the ${MAX_READ_CHARS}-character limit for filesystem_read. Use filesystem_grep to find what you need in it.`, + ) + } + const cutAt = args.startIdx + kept.length + const shortened = kept + .map( + (line, i) => + `${String(args.startIdx + i + 1).padStart(width)} | ${line}`, + ) + .join('\n') + return { + text: + shortened + + `\n\n(${args.totalLines - cutAt} more lines in file; this answer was ` + + `cut at the ${MAX_READ_CHARS}-character limit. Use offset=${cutAt + 1} to continue reading.)`, + } } return { text } diff --git a/trios/agent-server/apps/server/tests/api/queen-tick-sweep.test.ts b/trios/agent-server/apps/server/tests/api/queen-tick-sweep.test.ts new file mode 100644 index 0000000000..6322306614 --- /dev/null +++ b/trios/agent-server/apps/server/tests/api/queen-tick-sweep.test.ts @@ -0,0 +1,36 @@ +/** + * @license + * Copyright 2025 BrowserOS + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +import { describe, it } from 'bun:test' +import assert from 'node:assert' +import { + MEASUREMENT_SWEEP_MS, + SWEEP_FLOOR_MS, + sweepDeadlineMs, +} from '../../src/api/services/queen-tick' + +describe('the sweep is bounded by a share of the tick, not by an absolute four minutes', () => { + it('gives a sixty-second tick forty-five seconds', () => { + // The round awaits the sweep before it dispatches, so this is how long the + // swarm is willing to hand out no work at all. + assert.strictEqual(sweepDeadlineMs(60), 45_000) + }) + + it('never exceeds the old ceiling, however long the tick', () => { + assert.strictEqual(sweepDeadlineMs(3600), MEASUREMENT_SWEEP_MS) + assert.ok(sweepDeadlineMs(600) <= MEASUREMENT_SWEEP_MS) + }) + + it('never falls below the floor, however short the tick', () => { + // A five-second tick must not make review impossible. + assert.strictEqual(sweepDeadlineMs(5), SWEEP_FLOOR_MS) + assert.strictEqual(sweepDeadlineMs(0), SWEEP_FLOOR_MS) + }) + + it('grows with the tick between the floor and the ceiling', () => { + assert.ok(sweepDeadlineMs(120) > sweepDeadlineMs(60)) + }) +}) diff --git a/trios/agent-server/apps/server/tests/tools/filesystem/read.test.ts b/trios/agent-server/apps/server/tests/tools/filesystem/read.test.ts index c6ca8a7748..db30fdbdb4 100644 --- a/trios/agent-server/apps/server/tests/tools/filesystem/read.test.ts +++ b/trios/agent-server/apps/server/tests/tools/filesystem/read.test.ts @@ -154,6 +154,25 @@ describe('filesystem_read', () => { expect(result.text).toContain(`${MAX_READ_CHARS}-character limit`) }) + it('returns what fits instead of refusing, with the offset to continue', async () => { + // A refusal costs a provider call and returns nothing. Measured on the + // running deployment 2026-09-20: 18 of 19 filesystem failures in one window + // were this refusal, on files of 159 and 500 lines. + const line = 'y'.repeat(200) + const lines = Array.from({ length: 200 }, () => line) + await writeFile(join(tmpDir, 'wide.txt'), lines.join('\n')) + const result = await exec({ path: 'wide.txt', limit: 200 }) + expect(result.isError).toBeFalsy() + expect(result.text.length).toBeLessThanOrEqual(MAX_READ_CHARS) + expect(result.text).toContain('cut at the') + const offset = Number(/offset=(\d+)/.exec(result.text)?.[1]) + expect(Number.isInteger(offset)).toBe(true) + expect(offset).toBeGreaterThan(1) + // The offset it names must actually be the next unread line. + const rest = await exec({ path: 'wide.txt', offset, limit: 1 }) + expect(rest.text).toContain(`${offset} | `) + }) + it('handles files with UTF-8 BOM', async () => { await writeFile(join(tmpDir, 'bom.txt'), '\uFEFFhello bom') const result = await exec({ path: 'bom.txt' }) diff --git a/trios/agent-server/docker-entrypoint.sh b/trios/agent-server/docker-entrypoint.sh index 320ba1fe9b..a16807adab 100755 --- a/trios/agent-server/docker-entrypoint.sh +++ b/trios/agent-server/docker-entrypoint.sh @@ -21,6 +21,64 @@ set -e +# --------------------------------------------------------------------------- +# LIVENESS: a server that stops answering is ended, so the platform restarts it. +# +# Measured 2026-09-20/21: the agent server twice stopped answering HTTP WITHOUT +# EXITING - the edge said `Application failed to respond` - and Railway's +# restart policy never fired, because a restart policy fires on an exit and +# there was none. The first time it stayed down for nine hours. Raising +# restartPolicyMaxRetries to 10000 did not help the second time, for the same +# reason. An outside watchdog can redeploy, but only with a platform token; a +# process inside the container needs nothing but the loopback interface. +# +# So the entrypoint no longer `exec`s the server. It starts it, and a second +# process asks `/health` on loopback every LIVENESS_INTERVAL seconds after a +# LIVENESS_GRACE boot allowance. LIVENESS_FAILS answers in a row missing and the +# server is sent SIGTERM, then SIGKILL; the entrypoint then exits non-zero and +# `restartPolicyType: ON_FAILURE` brings the container back - with the boot +# clean-up that frees the disk. python3 does the asking because the image +# declares it; curl is not installed here. +# +# One healthy answer resets the count: a slow minute is not a dead server. +run_supervised() { + "$@" & + server=$! + trap 'kill -TERM "$server" 2>/dev/null' TERM INT + ( + port="${PORT:-8080}" + interval="${LIVENESS_INTERVAL:-30}" + fails_allowed="${LIVENESS_FAILS:-4}" + sleep "${LIVENESS_GRACE:-240}" + fails=0 + while kill -0 "$server" 2>/dev/null; do + if python3 -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:$port/health', timeout=10)" >/dev/null 2>&1; then + fails=0 + else + fails=$((fails + 1)) + echo "[liveness] /health did not answer ($fails of $fails_allowed)" + fi + if [ "$fails" -ge "$fails_allowed" ]; then + echo "[liveness] the server stopped answering; ending it so the platform restarts the container" + kill -TERM "$server" 2>/dev/null || true + sleep 15 + kill -KILL "$server" 2>/dev/null || true + break + fi + sleep "$interval" + done + ) & + set +e + wait "$server" + code=$? + # A server that was ended for not answering exits by signal, which `wait` + # reports as 128+N. Anything but a clean 0 must read as a failure to the + # platform, or ON_FAILURE would not restart it. + [ "$code" -eq 0 ] && code=1 + echo "[liveness] the server exited ($code); exiting so the platform restarts the container" + exit "$code" +} + # --------------------------------------------------------------------------- # HOW MANY BEES: derived from what is connected, not typed by hand. # @@ -176,7 +234,7 @@ echo "[entrypoint] TRIOS_QUEEN_MAX_WORKERS=$TRIOS_QUEEN_MAX_WORKERS (derived)" if [ -z "$TRIOS_REPO_URL" ]; then echo "[entrypoint] TRIOS_REPO_URL unset; starting without a checkout" - exec "$@" + run_supervised "$@" fi WORKSPACE_DIR="${WORKSPACE_DIR:-/workspace}" @@ -206,6 +264,52 @@ if [ -n "$TRIOS_TOOL_SHELL_USER" ] && id "$TRIOS_TOOL_SHELL_USER" >/dev/null 2>& echo "[entrypoint] $WORKSPACE_DIR is not owned by $TRIOS_TOOL_SHELL_USER; settling ownership once" chown -R "$TRIOS_TOOL_SHELL_USER" "$WORKSPACE_DIR" fi + # AND INSIDE THE CHECKOUT, which the test above cannot see. Measured on the + # running deployment 2026-09-20: + # + # error: Your local changes to the following files would be overwritten by checkout: + # error: unable to create file specs/port/tools/gft_deep_demo.t27: Permission denied + # + # $WORKSPACE_DIR was owned by the bee, so the walk above was skipped, while + # files underneath were not - left by a root-run git from an older image. A + # bee that cannot write the checkout produces an EMPTY branch and a turn that + # looks like a model failure: 80 of 101 stuck issues on that day had one. + # + # `find ! -user -print -quit` stops at the FIRST wrong file, so the healthy + # case costs one stat and the 45 GB walk that once outlasted the 300 s + # healthcheck (2026-09-03) cannot come back. The repair walks the checkout + # only - never the worktrees beside it - and changes only what is wrong. + if [ -d "$REPO_DIR" ] && [ -n "$(find "$REPO_DIR" ! -user "$TRIOS_TOOL_SHELL_USER" -print -quit 2>/dev/null)" ]; then + echo "[entrypoint] files inside $REPO_DIR are not owned by $TRIOS_TOOL_SHELL_USER; repairing those" + find "$REPO_DIR" ! -user "$TRIOS_TOOL_SHELL_USER" -exec chown "$TRIOS_TOOL_SHELL_USER" {} + 2>/dev/null || true + fi + # BEE WORKTREES FROM A PREVIOUS LIFE. At entrypoint time no bee is running - + # this process is what starts the server that starts them - so every + # directory under .worktrees/ belongs to a container that is already gone. + # + # They are not free. Measured on the running deployment 2026-09-20: every + # tick for six minutes chose an issue and then refused to start it - + # + # Queen tick chose an issue but the container cannot carry another bee + # issue=4438 resource="disk" + # + # - with twenty lanes open and 684 candidates waiting. The volume had filled + # with worktrees nobody could use; an earlier reading of the same volume + # found 41 of them holding 45 GB and three million inodes. + # + # `git worktree prune` alone does not do it: it drops the ADMIN records for + # directories that are already gone, and these directories are still there. + if [ -d "$REPO_DIR/.worktrees" ]; then + stale=$(ls -1 "$REPO_DIR/.worktrees" 2>/dev/null | wc -l | tr -d ' ') + if [ "$stale" != "0" ]; then + free_before=$(df -Pm "$REPO_DIR" 2>/dev/null | awk 'NR==2 {print $4}') + echo "[entrypoint] removing $stale bee worktree(s) left by a previous container (${free_before:-?} MiB free)" + $AS_USER "rm -rf '$REPO_DIR/.worktrees'/* '$REPO_DIR/.worktrees'/.[!.]*" 2>/dev/null || true + $AS_USER "git -C '$REPO_DIR' worktree prune" >/dev/null 2>&1 || true + free_after=$(df -Pm "$REPO_DIR" 2>/dev/null | awk 'NR==2 {print $4}') + echo "[entrypoint] worktrees cleared; ${free_after:-?} MiB free now" + fi + fi echo "[entrypoint] git runs as $TRIOS_TOOL_SHELL_USER; root does not enter the checkout" else AS_USER="sh -c" @@ -280,7 +384,7 @@ else # still arrives. $AS_USER "git clone --filter=blob:none --branch '$TRIOS_REPO_REF' \ '$TRIOS_REPO_URL' '$REPO_DIR'" \ - || { echo "[entrypoint] clone FAILED; starting without a checkout"; exec "$@"; } + || { echo "[entrypoint] clone FAILED; starting without a checkout"; run_supervised "$@"; } fi $AS_USER "git -C '$REPO_DIR' config user.name '${GIT_AUTHOR_NAME:-Trinity Bee}' \ @@ -288,4 +392,4 @@ $AS_USER "git -C '$REPO_DIR' config user.name '${GIT_AUTHOR_NAME:-Trinity Bee}' echo "[entrypoint] checkout ready: $($AS_USER "git -C '$REPO_DIR' rev-parse --short HEAD") on $TRIOS_REPO_REF" echo "[entrypoint] this checkout can read and commit; it cannot push, by design" -exec "$@" +run_supervised "$@" diff --git a/trios/agent-server/railway.json b/trios/agent-server/railway.json index a3f8222940..41c7bf3f79 100644 --- a/trios/agent-server/railway.json +++ b/trios/agent-server/railway.json @@ -8,7 +8,7 @@ "healthcheckPath": "/health", "healthcheckTimeout": 300, "restartPolicyType": "ON_FAILURE", - "restartPolicyMaxRetries": 10, + "restartPolicyMaxRetries": 10000, "numReplicas": 1 } }