fix(queen): the swarm read GitHub anonymously and died on 403 half the day

trios/agent-server/apps/server/src/api/services/queen-tick.ts

@@ -153,11 +153,61 @@ const ISSUE_PAGE_SIZE = 100
 const ISSUE_PAGE_CAP = 5
 
 /**
- * Open issues, read without a credential.
+ * The headers for a GitHub read, carrying the token when there is one.
+ *
+ * ANONYMOUS IS SIXTY REQUESTS AN HOUR, and this file has said so in a comment
+ * since it was written - "a second round trip per candidate against an
+ * anonymous rate limit that is 60 an hour" - while every call it makes went out
+ * unauthenticated anyway. The limit was designed around instead of lifted.
+ *
+ * WHAT IT COST. Measured 2026-09-06 over twelve hours: 144 rounds ended in
+ * `GitHub returned 403`, the swarm had ZERO bees running for 50% of the wall
+ * clock, and the median gap between one burst of work and the next was 22
+ * minutes against a five-minute tick. The pattern is bimodal - 36% of the time
+ * all four bees ran, 50% of the time none did - because a round either got its
+ * issue list or died whole. `openIssues` throws on a bad status, so one 403
+ * takes the entire round with it: no review, no choice, no dispatch.
+ *
+ * The token was in the environment the whole time. `GH_TOKEN` is set on this
+ * service and `/rate_limit` answers 15000 of 15000 remaining, which is the
+ * measurement that turns "we are being throttled" into "we are throttled at
+ * the anonymous tier while holding a key to the other one".
+ *
+ * THIS HANDS NOTHING TO A BEE. It is the supervisor's own outbound read. The
+ * worker environment is built from a ten-entry allowlist that deliberately
+ * excludes the GitHub token, and that stays exactly as it is - a bee still gets
+ * no credential from here.
+ *
+ * Falls back to anonymous when no token is set, so a local run without secrets
+ * behaves as it always has rather than failing to start.
+ */
+export function githubReadHeaders(): Record<string, string> {
+  const token = process.env.GH_TOKEN || process.env.GITHUB_TOKEN
+  const headers: Record<string, string> = {
+    Accept: 'application/vnd.github+json',
+  }
+  if (token) headers.Authorization = `Bearer ${token}`
+  return headers
+}
+
+/**
+ * Open issues.
  *
- * Anonymous on purpose: the repository is public, this is a read, and a token
- * here would be a credential in a container for no gain. GitHub's anonymous
- * rate limit is 60/hour against a loop that ticks at most a few times an hour.
+ * WAS ANONYMOUS ON PURPOSE, and the reasoning is kept here because it was
+ * sound and it is instructive that it expired rather than that it was wrong:
+ * "the repository is public, this is a read, and a token here would be a
+ * credential in a container for no gain. GitHub's anonymous rate limit is
+ * 60/hour against a loop that ticks at MOST A FEW TIMES AN HOUR."
+ *
+ * The loop ticks twelve times an hour now. Measured 2026-09-06 from inside the
+ * container, the service burns 77 anonymous requests an hour against that limit
+ * of 60 - and once it is spent every call returns 403 until the hourly reset,
+ * which is why the longest observed idle stretch was 41.7 minutes. The premise
+ * expired quietly; the sentence did not.
+ *
+ * The stated cost is also already paid: `GH_TOKEN` is on this service. It is
+ * the SUPERVISOR's container, not a bee's, and the worker environment is still
+ * built from an allowlist that excludes the token.
  *
  * PAGINATED, and it says whether it got everything. One page of 50 was the
  * whole list for as long as the repository stayed under the horizon - 44 open
@@ -178,9 +228,22 @@ export async function openIssues(repo: string): Promise<{
     const response = await fetch(
       `https://api.github.com/repos/${repo}/issues` +
         `?state=open&per_page=${ISSUE_PAGE_SIZE}&page=${page}`,
-      { headers: { Accept: 'application/vnd.github+json' } },
+      { headers: githubReadHeaders() },
     )
-    if (!response.ok) throw new Error(`GitHub returned ${response.status}`)
+    if (!response.ok) {
+      // A REFUSAL ON A LATER PAGE IS A TRUNCATION, AND THIS FUNCTION ALREADY
+      // HAS A WORD FOR THAT.
+      //
+      // Throwing here took the whole round with it - no review, no choice, no
+      // dispatch - and 135 of 136 round failures measured on 2026-09-06 were
+      // exactly this, leaving the swarm with zero bees for half the day. But
+      // the contract below already covers a list that is not the whole truth:
+      // `complete` stays false and `rememberIssues` is told not to treat it as
+      // the full set. A first page that fails leaves nothing to decide against,
+      // so that one still throws.
+      if (page > 1) break
+      throw new Error(`GitHub returned ${response.status}`)
+    }
     const batch = (await response.json()) as Array<{
       number: number
       title?: string
@@ -657,7 +720,7 @@ async function bodiesFor(
   for (const number of numbers) {
     const response = await fetch(
       `https://api.github.com/repos/${repo}/issues/${number}`,
-      { headers: { Accept: 'application/vnd.github+json' } },
+      { headers: githubReadHeaders() },
     )
     if (!response.ok) continue
     const issue = (await response.json()) as { body?: string | null }
@@ -932,12 +995,38 @@ export async function runRound(
   // own policy, and only an ESCALATION reaches a person. Without this the hold
   // added to stop the six-times loop would have become a different starvation:
   // every issue she finished would be locked out of the pool for ever.
-  const reviewed = await reviewFinishedDispatches(pool)
+  // BOOKKEEPING MUST NOT BE ABLE TO IDLE THE SWARM.
+  //
+  // The same shape as the 403 above, one layer in. Review and reaping are
+  // housekeeping; dispatch is the thing the hive exists to do. An exception in
+  // either used to take the whole round with it, so a transient database error
+  // cost five minutes of every bee - and one round measured on 2026-09-06 died
+  // exactly that way, on `deadlock detected`.
+  //
+  // Neither is lost by continuing: the review re-reads every unjudged dispatch
+  // next round by construction, and the reaper re-finds a stalled one. What IS
+  // lost by throwing is the dispatch that would have happened, and that is the
+  // one thing a later round cannot give back - the idle minutes are spent.
+  //
+  // Logged at warn with the reason, never swallowed: `tri idle` reads these
+  // lines out of the service log and reports what stopped the rounds, so a
+  // review that fails EVERY round is loud rather than merely survivable.
+  const reviewed = await reviewFinishedDispatches(pool).catch((error) => {
+    logger.warn('Queen review failed; dispatching anyway', {
+      error: error instanceof Error ? error.message : String(error),
+    })
+    return { acted: [], strays: [], tally: [] } as ReviewRound
+  })
   if (reviewed.acted.length > 0) {
     logger.info('Queen reviewed her own work', { verdicts: reviewed.acted })
   }
 
-  const reaped = await reapStalledDispatches(pool)
+  const reaped = await reapStalledDispatches(pool).catch((error) => {
+    logger.warn('Queen reap failed; dispatching anyway', {
+      error: error instanceof Error ? error.message : String(error),
+    })
+    return [] as Awaited<ReturnType<typeof reapStalledDispatches>>
+  })
   if (reaped.length > 0) {
     logger.info('Queen tick reaped stalled dispatches', { issues: reaped })
   }
@@ -1811,7 +1900,11 @@ export function parseVerdictBlock(
   // Trying each and keeping the longest parse is stable under either
   // convention, so a worker running an older brief is not punished for it.
   const starts: number[] = []
-  for (let i = text.indexOf('## VERDICT'); i >= 0; i = text.indexOf('## VERDICT', i + 1)) {
+  for (
+    let i = text.indexOf('## VERDICT');
+    i >= 0;
+    i = text.indexOf('## VERDICT', i + 1)
+  ) {
     starts.push(i)
   }
   if (!starts.length) return []

trios/agent-server/apps/server/tests/api/queen-github-auth.test.ts

@@ -0,0 +1,116 @@
+import { afterEach, describe, expect, it } from 'bun:test'
+
+import {
+  githubReadHeaders,
+  openIssues,
+} from '../../src/api/services/queen-tick'
+
+/**
+ * The round died 144 times in twelve hours on `GitHub returned 403`, and the
+ * swarm sat with zero bees for half of the wall clock, because every GitHub
+ * read went out anonymous - sixty requests an hour - while `GH_TOKEN` sat in
+ * the environment unused.
+ *
+ * These cases pin the header, not the fetch: a test that mocked `fetch` would
+ * pass against a call that still forgot to ask for these headers.
+ */
+describe('githubReadHeaders', () => {
+  const saved = { gh: process.env.GH_TOKEN, github: process.env.GITHUB_TOKEN }
+
+  afterEach(() => {
+    if (saved.gh === undefined) delete process.env.GH_TOKEN
+    else process.env.GH_TOKEN = saved.gh
+    if (saved.github === undefined) delete process.env.GITHUB_TOKEN
+    else process.env.GITHUB_TOKEN = saved.github
+  })
+
+  it('carries the token this service actually has', () => {
+    delete process.env.GITHUB_TOKEN
+    process.env.GH_TOKEN = 'test-token-value'
+    // GH_TOKEN is the name set on the deployed service. Reading only
+    // GITHUB_TOKEN would leave the limit at sixty an hour on the one
+    // deployment that matters, and every test here would still pass.
+    expect(githubReadHeaders().Authorization).toBe('Bearer test-token-value')
+  })
+
+  it('accepts GITHUB_TOKEN as well, so a differently-configured host still works', () => {
+    delete process.env.GH_TOKEN
+    process.env.GITHUB_TOKEN = 'other-token'
+    expect(githubReadHeaders().Authorization).toBe('Bearer other-token')
+  })
+
+  it('stays anonymous when there is no token, rather than failing to start', () => {
+    delete process.env.GH_TOKEN
+    delete process.env.GITHUB_TOKEN
+    const headers = githubReadHeaders()
+    expect(headers.Authorization).toBeUndefined()
+    // A local run with no secrets must behave as it always has.
+    expect(headers.Accept).toBe('application/vnd.github+json')
+  })
+
+  it('always asks for the JSON media type', () => {
+    process.env.GH_TOKEN = 'x'
+    expect(githubReadHeaders().Accept).toBe('application/vnd.github+json')
+  })
+})
+
+/**
+ * One 403 used to take the whole round with it - no review, no choice, no
+ * dispatch - and 135 of 136 round failures measured on 2026-09-06 were exactly
+ * that, leaving the swarm with ZERO bees for half the day.
+ *
+ * `globalThis.fetch` is replaced and restored here rather than mocked with
+ * `mock.module`, which is process-global in bun and cannot be undone: a fake
+ * left behind by this file would be inherited by every test that ran after it.
+ */
+describe('openIssues when GitHub refuses', () => {
+  const realFetch = globalThis.fetch
+
+  afterEach(() => {
+    globalThis.fetch = realFetch
+  })
+
+  const page = (n: number) =>
+    new Response(
+      JSON.stringify(
+        Array.from({ length: n }, (_, i) => ({
+          number: i + 1,
+          title: 't',
+          body: 'b',
+        })),
+      ),
+      { status: 200, headers: { 'content-type': 'application/json' } },
+    )
+
+  it('treats a refusal on a later page as a truncated list, not a dead round', async () => {
+    let call = 0
+    globalThis.fetch = (async () => {
+      call += 1
+      return call === 1
+        ? page(100)
+        : new Response('rate limited', { status: 403 })
+    }) as typeof fetch
+
+    const got = await openIssues('owner/repo')
+    // The work from page one survives, and the caller is told it is partial.
+    expect(got.issues).toHaveLength(100)
+    expect(got.complete).toBe(false)
+  })
+
+  it('still throws when the FIRST page is refused, because nothing was read', async () => {
+    globalThis.fetch = (async () =>
+      new Response('rate limited', { status: 403 })) as typeof fetch
+    // A round with no issue list at all has nothing to decide against, so this
+    // one must stay fatal - tolerating it would dispatch against an empty board.
+    await expect(openIssues('owner/repo')).rejects.toThrow(
+      'GitHub returned 403',
+    )
+  })
+
+  it('marks a short page complete, as it always has', async () => {
+    globalThis.fetch = (async () => page(7)) as typeof fetch
+    const got = await openIssues('owner/repo')
+    expect(got.issues).toHaveLength(7)
+    expect(got.complete).toBe(true)
+  })
+})

trios/agent-server/apps/server/tests/api/queen-round.test.ts

@@ -14,10 +14,7 @@ import {
   runRound,
 } from '../../src/api/services/queen-tick'
 import { logger } from '../../src/lib/logger'
-import {
-  queendPathEnvVar,
-  resolveQueendPath,
-} from '../__helpers__/queend-path'
+import { queendPathEnvVar, resolveQueendPath } from '../__helpers__/queend-path'
 
 /**
  * The round itself, driven against the real policy binary.
@@ -98,12 +95,19 @@ const isDispatchInsert = (sql: string) =>
  * order is the thing under test in two of the cases below and a fake that
  * encodes it would agree with whatever the code does.
  */
-function roundPool(finished: FinishedRow[] = []) {
+function roundPool(finished: FinishedRow[] = [], throwOn?: string) {
   const queries: Array<{ sql: string; params: unknown[] }> = []
   const pool = {
     query: async (sql: string, params: unknown[] = []) => {
       queries.push({ sql: String(sql), params })
       const text = String(sql)
+      // One named statement can be made to fail, so a test can ask what the
+      // round does when a single piece of bookkeeping breaks. Recorded first,
+      // so the failing statement still shows up in `sql()` as having been
+      // attempted - a query that throws is a query the round issued.
+      if (throwOn && text.includes(throwOn)) {
+        throw new Error('deadlock detected')
+      }
       if (text.includes('FROM queen_registry')) {
         return { rowCount: 1, rows: [{ tasks: [] }] }
       }
@@ -237,6 +241,32 @@ describe('queen round, lease lost', () => {
       expect(sql().some(isDispatchInsert)).toBe(true)
     },
   )
+
+  /**
+   * BOOKKEEPING MUST NOT BE ABLE TO IDLE THE SWARM.
+   *
+   * The review used to throw straight out of the round, so a transient database
+   * error cost five minutes of every bee. One round measured on 2026-09-06 died
+   * exactly that way, on `deadlock detected`, and 135 others died a page
+   * earlier on a GitHub 403 - together leaving the swarm with ZERO bees running
+   * for half of a twelve-hour window.
+   *
+   * Nothing is lost by continuing: the review re-reads every unjudged dispatch
+   * next round by construction. What IS lost by throwing is the dispatch that
+   * would have happened, and no later round gives those minutes back.
+   *
+   * `FROM queen_dispatch d` is the review's SELECT and the only query in the
+   * round that uses that alias, so failing it targets the review and nothing
+   * else. Reverting the `.catch()` turns this red: the round stops before
+   * `dispatchBee` and the INSERT never appears.
+   */
+  it.if(present)('dispatches even when the review throws', async () => {
+    const { pool, sql } = roundPool([], 'FROM queen_dispatch d')
+    const result = await runRound(pool, 'me', 7, { held: true }, [ISSUE])
+
+    expect(result.choice?.chosen).toBe(ISSUE)
+    expect(sql().some(isDispatchInsert)).toBe(true)
+  })
 })
 
 /**