diff --git a/trios/agent-server/Dockerfile b/trios/agent-server/Dockerfile index 17ae8ea5f..472040ed7 100644 --- a/trios/agent-server/Dockerfile +++ b/trios/agent-server/Dockerfile @@ -54,6 +54,45 @@ RUN cp "$(swift build -c release --show-bin-path)/queend" /queen-core/queend \ && /queen-core/queend /dev/null; \ test -x /queen-core/queend +# The T27 compiler, for the bees and for the review of the bees. +# +# MEASURED 2026-09-10 (gHashTag/t27#3560): 34 finished bee branches harvested +# from this container, every one reviewed as "met" on the bee's own verdict +# block. Run through `t27c` on a Mac: 20 did not parse, 4 parsed with DISCARDED +# tokens, 1 regressed typecheck, 9 held. Neither the bee nor the reviewer had +# the compiler; both were guessing, and the review had no way to say so. +# +# Built the way t27's own Dockerfile builds it: `bootstrap/` alone, standalone, +# with the repository's root Cargo.lock (bootstrap/Cargo.lock is stale against +# bootstrap/Cargo.toml and fails `--locked`). A sparse, blob-less clone of two +# directories, not the 400 MB repository. The commit that was built is written +# to /usr/local/share/t27c/REF and shipped, so a review can say WHICH compiler +# it measured with. +# +# T27_REF is an ARG, not a pin: the bees work against t27 master and the +# compiler that judges them should be the one master builds. The cost is that +# a cached layer serves the sha it recorded, not today's - the REF file is the +# truth, not the ARG. Bump the ARG to rebuild. +# +# Build time: 15 min on 2 vCPU (sandbox, cold cache, 2026-09-13). Nothing here +# is parallel with the Swift stage by BuildKit's choice, not ours. +FROM rust:1-bookworm AS t27c +ARG T27_REF=master +WORKDIR /src +RUN git clone --depth 1 --filter=blob:none --sparse --branch "${T27_REF}" \ + https://github.com/gHashTag/t27.git t27 \ + && cd t27 \ + && git sparse-checkout set bootstrap gen \ + && git rev-parse HEAD > /src/REF \ + && echo "t27c built from gHashTag/t27@$(cat /src/REF) (ref ${T27_REF})" +# Standalone: the root Cargo.toml names workspace members the sparse checkout +# does not have, so `bootstrap/` is copied out from under it and built alone. +RUN cp -r t27/bootstrap /build && cp t27/Cargo.lock /build/Cargo.lock \ + && rm -rf /build/target \ + && cd /build && cargo build --release \ + && strip /build/target/release/t27c \ + && /build/target/release/t27c --version + FROM oven/bun:1.3.6 AS runtime WORKDIR /app @@ -148,6 +187,16 @@ COPY --from=queen-core /queen-core/PROOF /app/queen-core-linux.proof COPY --from=queen-core /queen-core/queend /usr/local/bin/queend COPY --from=queen-core /usr/lib/swift/linux /usr/lib/swift/linux +# The compiler, on PATH for the bee (su resets PATH to the login default, which +# includes /usr/local/bin) and for the review, which runs it as the bee. Built +# against bookworm's glibc 2.36; the binary links only libc, libm and libgcc_s +# (ldd, 2026-09-13), so it loads on bookworm or anything newer that oven/bun +# ships on. `--version` below is the proof that it loads, taken at build time +# rather than at the first review. +COPY --from=t27c /build/target/release/t27c /usr/local/bin/t27c +COPY --from=t27c /src/REF /usr/local/share/t27c/REF +RUN /usr/local/bin/t27c --version && echo "t27c from gHashTag/t27@$(cat /usr/local/share/t27c/REF)" + # Bind wide, because the platform routes to this container from outside and a # server on loopback accepts nothing while looking perfectly healthy from # inside. PORT is injected by the platform at deploy time; 8080 is only the diff --git a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts index f4e0a820d..69031ed17 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts @@ -23,10 +23,9 @@ * publication step still belongs to a machine that has the credential. */ -import { spawn } from 'node:child_process' -import { execFileSync } from 'node:child_process' -import { statSync } from 'node:fs' +import { execFileSync, spawn } from 'node:child_process' import { randomUUID } from 'node:crypto' +import { statSync } from 'node:fs' import type { Pool } from 'pg' import { logger } from '../../lib/logger' import { shellArgv } from '../../tools/filesystem/bash' @@ -620,6 +619,235 @@ export async function committedFileCount(issue: number): Promise { return (await committedFiles(issue)).length } +/** + * What `t27c` says about ONE `.t27` file a bee committed. + * + * `present` is false for a file the branch deleted - there is no spec to read. + * `baseTypechecks` is null for a file the branch created - there is no earlier + * version to regress against. + */ +export interface SpecWitness { + file: string + present: boolean + /** `t27c parse` exited 0 - no hard parse error. */ + parses: boolean + /** `t27c parse-complete` consumed the whole file: no TRUNCATE, no DISCARD. */ + complete: boolean + discardedTokens: number + /** Occurrences of the literal `TODO: Implement` stub marker. */ + stubMarkers: number + /** `t27c typecheck` exited 0 on the committed file. */ + typechecks: boolean + /** The same, on the base ref's version of the file; null when it is new. */ + baseTypechecks: boolean | null + /** The first error line the compiler printed, or ''. */ + error: string +} + +/** + * The machine's side of a review. + * + * `absent` means the compiler is not on this image (or cannot run): nothing + * was measured, and the caller must not read that as "nothing failed". + */ +export type Witness = + | { kind: 'absent'; detail: string } + | { kind: 'witnessed'; t27c: string; specs: SpecWitness[] } + +/** The compiler the review runs. `T27C_BIN` overrides for a test or a Mac. */ +function t27cBinary(): string { + return process.env.T27C_BIN || 't27c' +} + +/** + * The witness script, one file at a time, run AS THE BEE in the repository + * root. Positional: $1 branch, $2 file, $3 base ref, $4 t27c binary. + * + * Everything is read from the COMMIT (`git show branch:file`), never from the + * worktree, because the worktree may hold edits the bee never committed and + * the commit is the deliverable. Each measurement prints one `W ` line; the + * TypeScript side parses those and nothing else, so compiler chatter cannot be + * mistaken for a result. + */ +const WITNESS_SCRIPT = String.raw` +set -u +branch="$1"; file="$2"; base="$3"; t27c="$4" +tmp="$(mktemp -d)" || exit 97 +trap 'rm -rf "$tmp"' EXIT +mkdir -p "$tmp/specs/one" +spec="$tmp/specs/one/$(basename "$file")" +if ! git show "$branch:$file" > "$spec" 2>/dev/null; then echo 'W absent'; exit 0; fi +if "$t27c" parse "$spec" > "$tmp/parse.out" 2>&1; then + echo 'W parse ok' +else + echo "W parse fail $(grep -m1 -i 'error' "$tmp/parse.out" | cut -c1-240)" +fi +"$t27c" parse-complete --specs-dir "$tmp/specs" 2>&1 | grep -E 'consume all|TRUNCATE|DISCARD|do not parse' | sed 's/^ */W pc /' +echo "W todo $(grep -c 'TODO: Implement' "$spec" || true)" +if "$t27c" typecheck "$spec" > /dev/null 2>&1; then echo 'W typecheck ok'; else echo 'W typecheck fail'; fi +if git show "$base:$file" > "$tmp/base.t27" 2>/dev/null; then + if "$t27c" typecheck "$tmp/base.t27" > /dev/null 2>&1; then echo 'W base ok'; else echo 'W base fail'; fi +else + echo 'W base new' +fi +` + +/** + * Turn the witness script's `W ` lines into one record. Exported for the + * tests: the shell is the part that needs a container, the reading is not. + */ +export function readWitnessLines(file: string, out: string): SpecWitness { + const w: SpecWitness = { + file, + present: true, + parses: false, + complete: false, + discardedTokens: 0, + stubMarkers: 0, + typechecks: false, + baseTypechecks: null, + error: '', + } + // The completeness report is four counters; the file is complete only when + // the one spec scanned landed in "parse and consume all". A report that + // never arrived (compiler crashed, output cut) leaves `complete` false: an + // unmeasured file is not a clean one. + let consumedAll = -1 + for (const raw of out.split('\n')) { + const line = raw.trim() + if (!line.startsWith('W ')) continue + const body = line.slice(2) + if (body === 'absent') { + w.present = false + return w + } + if (body === 'parse ok') w.parses = true + else if (body.startsWith('parse fail')) { + w.parses = false + w.error = body.slice('parse fail'.length).trim() + } else if (body.startsWith('pc ')) { + const m = body.match(/^pc\s+(.*?)\s+(\d+)(?:\s+\((\d+) token)?/) + if (!m) continue + const [, label, count, tokens] = m + if (label.includes('consume all')) consumedAll = Number(count) + else if (label.includes('DISCARD')) + w.discardedTokens = Number(tokens ?? count) + } else if (body.startsWith('todo ')) { + w.stubMarkers = Number(body.slice(5).trim()) || 0 + } else if (body === 'typecheck ok') w.typechecks = true + else if (body === 'typecheck fail') w.typechecks = false + else if (body === 'base ok') w.baseTypechecks = true + else if (body === 'base fail') w.baseTypechecks = false + else if (body === 'base new') w.baseTypechecks = null + } + w.complete = w.parses && consumedAll === 1 && w.discardedTokens === 0 + return w +} + +/** + * WHAT the compiler says about the `.t27` files a bee's branch changed. + * + * The review used to accept on the bee's own "met" lines. Harvested on + * 2026-09-10 (gHashTag/t27#3560): of 34 finished bee branches whose verdict + * blocks were read as met, 20 did not parse, 4 parsed with DISCARDED tokens + * and 1 regressed typecheck - 9 of 34 held up when `t27c` was run on the + * commit. The bee's word is the claim; this is the measurement, taken by the + * same three commands the operator ran by hand, on the same commit. + * + * Only `.t27` files are witnessed; a branch that changed none returns an + * empty `specs`. A missing compiler returns `absent`, which the review treats + * as "could not check" - never as a pass. + */ +export async function witnessSpecs( + issue: number, + files: string[], +): Promise { + const specs = files.filter((f) => f.endsWith('.t27')) + const bin = t27cBinary() + if (specs.length === 0) return { kind: 'witnessed', t27c: bin, specs: [] } + const root = workspaceRoot() + const version = await run(bin, ['--version'], root, 15_000) + if (version.code !== 0) { + return { + kind: 'absent', + detail: `${bin} --version exited ${version.code}: ${version.out.slice(0, 200)}`, + } + } + const base = process.env.TRIOS_REPO_REF || 'origin/dev' + const branch = `queen-${issue}` + const out: SpecWitness[] = [] + for (const file of specs) { + const r = await run( + 'sh', + ['-c', WITNESS_SCRIPT, 'witness', branch, file, base, bin], + root, + 120_000, + ) + const w = readWitnessLines(file, r.out) + if (r.code !== 0 && r.code !== 1) { + // The script itself failed (mktemp, kill on timeout): nothing measured. + w.present = true + w.parses = false + w.complete = false + w.error = w.error || `witness script exited ${r.code}` + } + out.push(w) + } + return { + kind: 'witnessed', + t27c: version.out.split('\n')[0].trim(), + specs: out, + } +} + +/** + * The witness as verdict lines, in the shape the review policy already + * weighs. One line per measurement per file, met or unmet, so `queend`'s one + * rule decides send-back versus escalate and nothing is decided here. + * + * The typecheck line is a RATCHET, not a gate, matching the repository's own + * corpus check: a file that failed typecheck before the bee touched it may + * still fail; a file that passed, or did not exist, must pass. + */ +export function witnessVerdicts( + witness: Witness, +): Array<{ criterion: string; met: boolean }> { + if (witness.kind !== 'witnessed') return [] + const lines: Array<{ criterion: string; met: boolean }> = [] + for (const s of witness.specs) { + if (!s.present) continue + const why = s.error + ? ` (${s.error})` + : s.discardedTokens > 0 + ? ` (parse-complete DISCARDED ${s.discardedTokens} token(s))` + : s.parses && !s.complete + ? ' (parse-complete did not consume the whole file)' + : '' + lines.push({ + criterion: `t27c: ${s.file} parses clean${s.complete ? '' : why}`, + met: s.complete, + }) + lines.push({ + criterion: `t27c: ${s.file} has no 'TODO: Implement' stub markers${ + s.stubMarkers > 0 ? ` (${s.stubMarkers} found)` : '' + }`, + met: s.stubMarkers === 0, + }) + const regressed = !s.typechecks && s.baseTypechecks !== false + lines.push({ + criterion: `t27c: ${s.file} typecheck does not regress${ + regressed + ? s.baseTypechecks === null + ? ' (new file fails typecheck)' + : ' (passed on the base ref, fails on this branch)' + : '' + }`, + met: !regressed, + }) + } + return lines +} + export function workspaceRoot(): string { // The directory name is DERIVED from the repo URL, exactly as the entrypoint // derives it — `REPO_NAME="$(basename "$TRIOS_REPO_URL" .git)"`. Hardcoding @@ -720,12 +948,14 @@ export function volumeUsedPercent(dir = workspaceRoot()): number | null { * is never worth it. Nor does it touch the newest few, which are likely to be * running right now. */ -export async function reapWorktrees(opts: { - high?: number - low?: number - keepNewest?: number - volumeUsed?: (dir: string) => number | null -} = {}): Promise<{ +export async function reapWorktrees( + opts: { + high?: number + low?: number + keepNewest?: number + volumeUsed?: (dir: string) => number | null + } = {}, +): Promise<{ before: number | null after: number | null removed: string[] @@ -734,7 +964,8 @@ export async function reapWorktrees(opts: { }> { const high = opts.high ?? Number(process.env.QUEEN_VOLUME_HIGH ?? 80) const low = opts.low ?? Number(process.env.QUEEN_VOLUME_LOW ?? 55) - const keepNewest = opts.keepNewest ?? Number(process.env.QUEEN_VOLUME_KEEP ?? 6) + const keepNewest = + opts.keepNewest ?? Number(process.env.QUEEN_VOLUME_KEEP ?? 6) const root = workspaceRoot() const measure = opts.volumeUsed ?? volumeUsedPercent const before = measure(root) @@ -796,7 +1027,6 @@ export async function reapWorktrees(opts: { return result } - /** * Link this worktree's node_modules into a shared store instead of installing. * diff --git a/trios/agent-server/apps/server/src/api/services/queen-tick.ts b/trios/agent-server/apps/server/src/api/services/queen-tick.ts index 60f943eaf..1d88c7b64 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-tick.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-tick.ts @@ -46,6 +46,9 @@ import { reapDispatchesFromPreviousBoot, reapStalledDispatches, setDurableCloseListener, + type Witness, + witnessSpecs, + witnessVerdicts, workspaceRoot, } from './queen-dispatch' import { @@ -1278,6 +1281,19 @@ export function briefFor( 'unchecked criterion is not a pass, and saying so plainly costs you', 'nothing.', '', + // The compiler, named, and the fact that the review runs it. Harvested + // 2026-09-10 (gHashTag/t27#3560): 34 branches whose bees had written + // "met", 9 of which parsed clean when `t27c` was actually run. The other 25 + // bees were not lying so much as guessing, because nothing told them the + // compiler was on the machine or that anyone would run it after them. + 'The T27 compiler is installed: `t27c` is on your PATH (/usr/local/bin/t27c).', + 'For every `.t27` file you change, run `t27c parse ` and', + '`t27c typecheck ` yourself before you answer, and quote the result.', + 'The review runs the same commands on your COMMIT - parse, parse-complete', + "and typecheck - and the compiler's answer stands above your verdict line.", + 'A file with a parse error, a DISCARDED token run, or a `TODO: Implement`', + 'stub marker is unmet whatever the line says.', + '', '## Out of scope', '', 'Anything the issue does not ask for. Work that seems obviously needed and', @@ -1291,6 +1307,16 @@ export function briefFor( 'by the operator. A failed push reads as a failed task; a commit is the', 'deliverable.', '', + // The trailer, in the exact form the repository's traceability gate + // accepts. The 9 bee commits carried into gHashTag/t27#3560 all had to be + // rewritten by hand: they closed with "Resolves gHashTag/t27#N", which + // reads well and matches nothing - the L1 gate wants a bare `#N`. + `End your commit message with the line \`Closes #${issue}\` - exactly that`, + 'form, on its own line, bare issue number. "Resolves owner/repo#N" does not', + "pass the repository's traceability gate and the commit is rewritten by hand.", + "Sealing (`t27c seal`) and the `docs/now/` entry are the operator's at", + 'harvest time, not yours: they fall outside your boundary.', + '', // The template, one numbered slot per criterion (#1421). Emitted only when // the task states criteria, so a task with none is unchanged: its bee // states its own criteria first and still needs the standing generic @@ -1447,6 +1473,10 @@ export function workerSystemPrompt( } lines.push( 'Everything you write is English. When you stop, answer every acceptance criterion in turn: met, not met, or could not check.', + // Said twice on purpose - once here, once in the brief - because the + // system prompt survives a context that the brief may have scrolled out + // of. A bee that finishes without the trailer costs a hand rewrite. + `The T27 compiler t27c is installed on this machine; run \`t27c parse\` and \`t27c typecheck\` on every .t27 file you change, because the review runs them on your commit. Your final commit message ends with the line \`Closes #${issue}\`.`, ) return lines.join(' ') } @@ -1747,25 +1777,58 @@ export async function reviewFinishedDispatches( // is missing entirely is excluded: no verdicts at all is the torn-transcript // signature the wait state exists for, and the frozen-wait valve releases // it if the transcript never does arrive. + // The machine's answer, next to the bee's. Every `.t27` file the branch + // changed is read from the COMMIT and run through `t27c parse`, + // `parse-complete` and `typecheck`; each measurement becomes a verdict line + // the policy weighs exactly like the bee's own. Measured 2026-09-10 + // (gHashTag/t27#3560): 34 branches this review had passed on the bee's + // word, 9 held up under the compiler - 20 did not parse at all. A review + // that reads "met" and does not run the compiler is not a review. + // + // Taken only once the bee has judged anything: a bee with no verdict block + // is the torn-transcript case, and a compiler's yes must not stand in for + // the answers the bee never wrote. + const witness: Witness | null = + verdicts.length > 0 + ? await witnessSpecs(row.issue as number, files) + : null + const machine = witness ? witnessVerdicts(witness) : [] + const machineFailed = machine.filter((v) => !v.met).map((v) => v.criterion) + const specCount = files.filter((f) => f.endsWith('.t27')).length const questioned = verdicts.length > 0 ? [ ...verdicts.map((v) => ({ criterion: v.criterion, met: v.met })), ...unjudged.map((criterion) => ({ criterion, met: false })), + ...machine, ] : verdicts.map((v) => ({ criterion: v.criterion, met: v.met })) - const answer = await askQueend({ - kind: 'review', - verdicts: questioned, - totalCriteria, - committedFiles: files.length, - priorSendBacks, - }).catch(() => null) - const state = String(answer?.verdict ?? 'wait') + // No compiler on this image while the branch changed specs: nothing was + // measured, so nothing is accepted. This asks a PERSON rather than waiting, + // because a wait here would never resolve on its own - the frozen-wait + // valve would fail the dispatch after six hours and return the issue to + // the pool, losing a finished branch to a missing binary. `escalate` keeps + // the branch on the board with the reason written down. + const unwitnessed = witness?.kind === 'absent' && specCount > 0 + const answer = unwitnessed + ? null + : await askQueend({ + kind: 'review', + verdicts: questioned, + totalCriteria, + committedFiles: files.length, + priorSendBacks, + }).catch(() => null) + const state = unwitnessed ? 'escalate' : String(answer?.verdict ?? 'wait') // Judged versus unjudged, recorded per dispatch (#1420, FR-001): "2 of 5 // judged" is a fact about the worker's reporting, not about the work, and - // the two belong in the record as separate numbers. - const failed = verdicts.filter((v) => !v.met).map((v) => v.criterion) + // the two belong in the record as separate numbers. The compiler's failed + // lines join the judged-and-failed list: they were checked, and found + // wanting, which is what that list means. + const failed = [ + ...verdicts.filter((v) => !v.met).map((v) => v.criterion), + ...machineFailed, + ] logger.info('Queen reviewed her own work', { issue: row.issue, verdict: state, @@ -1775,12 +1838,29 @@ export async function reviewFinishedDispatches( source: row.criteria_source ?? 'none', priorSendBacks, strays: strays.length, + specs: specCount, + t27c: witness?.kind === 'witnessed' ? witness.t27c : 'absent', + machineUnmet: machineFailed.length, }) + if (unwitnessed) { + logger.warn( + 'Queen could not witness the specs: t27c is not on this image', + { + issue: row.issue, + specs: specCount, + detail: witness?.kind === 'absent' ? witness.detail : '', + }, + ) + } // The send-back message the worker reads, with the two lists under distinct // headings (#1420, FR-002): what it tested and failed, and what it never // wrote a verdict line for at all. - const note = - state === 'sendBack' + const note = unwitnessed + ? `${specCount} .t27 file(s) changed but t27c is not available on this ` + + 'image, so the review could not run t27c parse / parse-complete / ' + + 'typecheck on the commit; a reviewer with t27c must, before merging. ' + + (witness?.kind === 'absent' ? witness.detail : '') + : state === 'sendBack' ? sendBackMessage(String(answer?.note ?? ''), failed, unjudged) : String(answer?.note ?? answer?.refusal ?? '') // Whether this attempt counts against the retry ceiling (#1420, FR-003). @@ -1837,7 +1917,11 @@ export function parseVerdictBlock( // Trying each and keeping the longest parse is stable under either // convention, so a worker running an older brief is not punished for it. const starts: number[] = [] - for (let i = text.indexOf('## VERDICT'); i >= 0; i = text.indexOf('## VERDICT', i + 1)) { + for ( + let i = text.indexOf('## VERDICT'); + i >= 0; + i = text.indexOf('## VERDICT', i + 1) + ) { starts.push(i) } if (!starts.length) return [] diff --git a/trios/agent-server/apps/server/tests/api/queen-witness.test.ts b/trios/agent-server/apps/server/tests/api/queen-witness.test.ts new file mode 100644 index 000000000..4ea7e3e0b --- /dev/null +++ b/trios/agent-server/apps/server/tests/api/queen-witness.test.ts @@ -0,0 +1,424 @@ +/** + * The review runs the compiler on the bee's commit, and what it finds outranks + * the bee's verdict line. + * + * Measured 2026-09-10 (gHashTag/t27#3560): 34 finished bee branches, every one + * reviewed as met on the bee's own VERDICT block; 20 did not parse, 4 parsed + * with DISCARDED tokens, 1 regressed typecheck, 9 held. The reviewer had no + * `t27c`, so it had never been able to check - and never said so. + * + * Three layers, tested separately: + * - reading the witness script's lines (pure, recorded from a real t27c); + * - turning a witness into verdict lines the policy weighs (pure); + * - the review itself: no compiler + specs on the branch => escalate with + * the reason written down, never accept. + * The end-to-end case with a real compiler is gated on one being present. + */ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' +import { spawnSync } from 'node:child_process' +import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import type { Pool } from 'pg' +import { + readWitnessLines, + type SpecWitness, + type Witness, + witnessSpecs, + witnessVerdicts, +} from '../../src/api/services/queen-dispatch' +import { + briefFor, + reviewFinishedDispatches, + workerSystemPrompt, +} from '../../src/api/services/queen-tick' + +const ISSUE = 3542 + +// Recorded 2026-09-13 from `t27c 0.2.0` (gHashTag/t27 master 4dfb1ab) run +// through the witness script on a real commit. If the compiler's report format +// changes, these fixtures are the first thing to re-record. +const CLEAN = [ + 'W parse ok', + 'W pc parse and consume all 1', + 'W pc parse but TRUNCATE 0', + 'W pc parse but DISCARD 0 (0 token(s))', + 'W pc do not parse 0', + 'W todo 0', + 'W typecheck ok', + 'W base ok', +].join('\n') + +const DISCARD = [ + 'W parse ok', + 'W pc /tmp/tmp.DF3dq4PGT3/specs/one/logging.t27: DISCARDED 1 top-level token(s)', + 'W pc parse and consume all 0', + 'W pc parse but TRUNCATE 0', + 'W pc parse but DISCARD 1 (1 token(s))', + 'W pc do not parse 0', + 'W todo 0', + 'W typecheck ok', + 'W base fail', +].join('\n') + +const STUB = [ + 'W parse ok', + 'W pc parse and consume all 1', + 'W pc parse but TRUNCATE 0', + 'W pc parse but DISCARD 0 (0 token(s))', + 'W pc do not parse 0', + 'W todo 1', + 'W typecheck ok', + 'W base ok', +].join('\n') + +const NO_PARSE = [ + 'W parse fail Error: Parse error: parse error at module level near line 130: unexpected token after expression statement: Ident', + 'W pc parse and consume all 0', + 'W pc parse but TRUNCATE 0', + 'W pc parse but DISCARD 0 (0 token(s))', + 'W pc do not parse 1', + 'W todo 0', + 'W typecheck fail', + 'W base new', +].join('\n') + +describe('readWitnessLines, the compiler report read back', () => { + it('reads a clean file as complete with no stubs', () => { + const w = readWitnessLines('specs/a.t27', CLEAN) + expect(w.present).toBe(true) + expect(w.parses).toBe(true) + expect(w.complete).toBe(true) + expect(w.discardedTokens).toBe(0) + expect(w.stubMarkers).toBe(0) + expect(w.typechecks).toBe(true) + expect(w.baseTypechecks).toBe(true) + }) + + it('reads a DISCARD as parsed but not complete', () => { + const w = readWitnessLines('specs/a.t27', DISCARD) + expect(w.parses).toBe(true) + expect(w.complete).toBe(false) + expect(w.discardedTokens).toBe(1) + expect(w.baseTypechecks).toBe(false) + }) + + it('counts stub markers', () => { + expect(readWitnessLines('specs/a.t27', STUB).stubMarkers).toBe(1) + }) + + it('keeps the first error line of a hard parse failure', () => { + const w = readWitnessLines('specs/a.t27', NO_PARSE) + expect(w.parses).toBe(false) + expect(w.complete).toBe(false) + expect(w.error).toContain('Parse error') + expect(w.baseTypechecks).toBeNull() + }) + + it('reads a deleted file as not present', () => { + expect(readWitnessLines('specs/a.t27', 'W absent').present).toBe(false) + }) + + it('does not read compiler chatter as a measurement', () => { + // Nothing but noise: no `W ` line at all. An unmeasured file is not clean. + const w = readWitnessLines('specs/a.t27', 'Node {\n kind: Module\n}\n') + expect(w.parses).toBe(false) + expect(w.complete).toBe(false) + }) +}) + +describe('witnessVerdicts, the measurement as verdict lines', () => { + const witnessed = (specs: SpecWitness[]): Witness => ({ + kind: 'witnessed', + t27c: 't27c 0.2.0', + specs, + }) + + it('says nothing when the compiler was absent', () => { + expect(witnessVerdicts({ kind: 'absent', detail: 'no t27c' })).toEqual([]) + }) + + it('emits three met lines for a clean file', () => { + const lines = witnessVerdicts( + witnessed([readWitnessLines('specs/a.t27', CLEAN)]), + ) + expect(lines).toHaveLength(3) + expect(lines.every((l) => l.met)).toBe(true) + expect(lines[0].criterion).toBe('t27c: specs/a.t27 parses clean') + }) + + it('fails the parse line on a DISCARD and names the count', () => { + const lines = witnessVerdicts( + witnessed([readWitnessLines('specs/a.t27', DISCARD)]), + ) + const parse = lines[0] + expect(parse.met).toBe(false) + expect(parse.criterion).toContain('DISCARDED 1 token') + // The base already failed typecheck: not a regression, so not held + // against the bee. A ratchet, not a gate. + expect(lines[2].met).toBe(true) + }) + + it('fails the stub line and names how many', () => { + const lines = witnessVerdicts( + witnessed([readWitnessLines('specs/a.t27', STUB)]), + ) + expect(lines[1].met).toBe(false) + expect(lines[1].criterion).toContain('(1 found)') + }) + + it('holds a new file that fails typecheck as a regression', () => { + const lines = witnessVerdicts( + witnessed([readWitnessLines('specs/a.t27', NO_PARSE)]), + ) + expect(lines[0].met).toBe(false) + expect(lines[0].criterion).toContain('Parse error') + expect(lines[2].met).toBe(false) + expect(lines[2].criterion).toContain('new file fails typecheck') + }) + + it('skips a deleted file', () => { + expect( + witnessVerdicts(witnessed([readWitnessLines('specs/a.t27', 'W absent')])), + ).toEqual([]) + }) +}) + +describe('the brief names the compiler and the trailer', () => { + it('tells the bee t27c is installed and that the review runs it', () => { + const text = briefFor(ISSUE, 'gHashTag/t27', ['specs/x.t27'], 'body', [ + 'x parses', + ]) + expect(text).toContain('t27c parse ') + expect(text).toContain('t27c typecheck ') + expect(text).toContain('The review runs the same commands on your COMMIT') + }) + + it('dictates the exact trailer the traceability gate accepts', () => { + const text = briefFor(ISSUE, 'gHashTag/t27', ['specs/x.t27'], 'body', [ + 'x parses', + ]) + expect(text).toContain(`\`Closes #${ISSUE}\``) + expect(text).toContain('"Resolves owner/repo#N" does not') + }) + + it('repeats both in the system prompt', () => { + const text = workerSystemPrompt(ISSUE, 'gHashTag/t27', '/w/t27', []) + expect(text).toContain('t27c') + expect(text).toContain(`Closes #${ISSUE}`) + }) +}) + +// --- the review, end to end ------------------------------------------------ + +interface FinishedRow { + issue: number + conversation_id: string + criteria: string[] + criteria_source: string + send_backs: number + owned_paths: string[] + said: string +} + +const CRITERIA = ['specs/x.t27 parses clean', 'no stub markers remain'] + +const block = (met: boolean): string => + [ + '## VERDICT', + ...CRITERIA.map((c) => `- ${c}: ${met ? 'met' : 'unmet'}`), + ].join('\n') + +function finishedRow(said: string): FinishedRow { + return { + issue: ISSUE, + conversation_id: '00000000-0000-0000-0000-000000000dd6', + criteria: CRITERIA, + criteria_source: 'stated', + send_backs: 0, + owned_paths: ['specs/x.t27'], + said, + } +} + +function reviewPool(finished: FinishedRow[]) { + const queries: Array<{ sql: string; params: unknown[] }> = [] + const pool = { + query: async (sql: string, params: unknown[] = []) => { + queries.push({ sql: String(sql), params }) + if (String(sql).includes('FROM queen_dispatch d')) { + return { rowCount: finished.length, rows: finished } + } + return { rowCount: 0, rows: [] } + }, + } as unknown as Pool + return { pool, queries } +} + +const reviewUpdate = (queries: Array<{ sql: string; params: unknown[] }>) => + queries.find( + (q) => + q.sql.includes('UPDATE queen_dispatch') && + q.sql.includes('review_state ='), + ) + +/** A repository whose `queen-` branch committed the given files off `main`. */ +function repoWithCommit(files: Array<{ path: string; body: string }>): string { + const root = mkdtempSync(join(tmpdir(), 'queen-witness-')) + const repo = join(root, 'BrowserOS') + mkdirSync(repo, { recursive: true }) + const git = (...args: string[]) => + spawnSync('git', args, { cwd: repo, encoding: 'utf8' }) + git('init', '-b', 'main') + git('config', 'user.email', 'bee@example.invalid') + git('config', 'user.name', 'a bee') + writeFileSync(join(repo, 'README.md'), 'base\n') + git('add', '-A') + git('commit', '-m', 'base') + git('checkout', '-b', `queen-${ISSUE}`) + for (const file of files) { + mkdirSync(dirname(join(repo, file.path)), { recursive: true }) + writeFileSync(join(repo, file.path), file.body) + } + git('add', '-A') + git('commit', '-m', 'work') + git('checkout', 'main') + return root +} + +const T27C = [process.env.T27C_BIN, '/usr/local/bin/t27c'].find( + (p): p is string => Boolean(p) && existsSync(p as string), +) +const QUEEND = [ + process.env.TRIOS_QUEEND_PATH, + join(import.meta.dir, '../../../../queen-core/.build/release/queend'), + '/usr/local/bin/queend', +].find((p): p is string => Boolean(p) && existsSync(p as string)) + +const saved: Record = {} +const KEYS = [ + 'WORKSPACE_DIR', + 'TRIOS_REPO_REF', + 'TRIOS_REPO_URL', + 'T27C_BIN', + 'TRIOS_QUEEND_PATH', + 'TRIOS_TOOL_SHELL_USER', +] + +beforeEach(() => { + for (const key of KEYS) { + saved[key] = process.env[key] + delete process.env[key] + } + process.env.TRIOS_REPO_REF = 'main' +}) + +afterEach(() => { + for (const key of KEYS) { + if (saved[key] === undefined) delete process.env[key] + else process.env[key] = saved[key] + } +}) + +describe('the review, without a compiler', () => { + it('escalates a branch that changed specs and says the compiler is missing', async () => { + process.env.WORKSPACE_DIR = repoWithCommit([ + { path: 'specs/x.t27', body: 'module M\nfn a() -> i32 { return 1 }\n' }, + ]) + process.env.T27C_BIN = join(tmpdir(), 'no-such-t27c-anywhere') + const { pool, queries } = reviewPool([finishedRow(block(true))]) + const out = await reviewFinishedDispatches(pool) + expect(out.acted).toEqual([`#${ISSUE}:escalate`]) + const update = reviewUpdate(queries) + expect(update?.params[1]).toBe('escalate') + expect(String(update?.params[2])).toContain('t27c is not available') + expect(String(update?.params[2])).toContain('1 .t27 file(s)') + }) + + it('reports absent, not clean, from witnessSpecs itself', async () => { + process.env.WORKSPACE_DIR = repoWithCommit([ + { path: 'specs/x.t27', body: 'module M\n' }, + ]) + process.env.T27C_BIN = join(tmpdir(), 'no-such-t27c-anywhere') + const w = await witnessSpecs(ISSUE, ['specs/x.t27']) + expect(w.kind).toBe('absent') + }) + + it('has nothing to witness on a branch that changed no spec', async () => { + process.env.T27C_BIN = join(tmpdir(), 'no-such-t27c-anywhere') + const w = await witnessSpecs(ISSUE, ['docs/a.md', 'src/b.ts']) + expect(w).toEqual({ + kind: 'witnessed', + t27c: process.env.T27C_BIN, + specs: [], + }) + }) +}) + +describe('the review, with the compiler', () => { + it.if(Boolean(T27C))('measures a clean commit as complete', async () => { + process.env.WORKSPACE_DIR = repoWithCommit([ + { path: 'specs/x.t27', body: 'module M\nfn a() -> i32 { return 1 }\n' }, + ]) + process.env.T27C_BIN = T27C + const w = await witnessSpecs(ISSUE, ['specs/x.t27']) + expect(w.kind).toBe('witnessed') + if (w.kind !== 'witnessed') return + expect(w.specs).toHaveLength(1) + expect(w.specs[0].complete).toBe(true) + expect(w.specs[0].baseTypechecks).toBeNull() + }) + + it.if(Boolean(T27C))('measures a stray brace as a DISCARD', async () => { + process.env.WORKSPACE_DIR = repoWithCommit([ + { + path: 'specs/x.t27', + body: 'module M\nfn a() -> i32 { return 1 }\n}\n', + }, + ]) + process.env.T27C_BIN = T27C + const w = await witnessSpecs(ISSUE, ['specs/x.t27']) + if (w.kind !== 'witnessed') throw new Error('expected a witness') + expect(w.specs[0].parses).toBe(true) + expect(w.specs[0].complete).toBe(false) + expect(w.specs[0].discardedTokens).toBeGreaterThan(0) + }) + + it.if(Boolean(T27C && QUEEND))( + 'sends back a bee whose "met" the compiler contradicts', + async () => { + process.env.WORKSPACE_DIR = repoWithCommit([ + { + path: 'specs/x.t27', + body: 'module M\nfn a() -> i32 {\n // TODO: Implement\n return 1\n}\n}\n', + }, + ]) + process.env.T27C_BIN = T27C + process.env.TRIOS_QUEEND_PATH = QUEEND + const { pool, queries } = reviewPool([finishedRow(block(true))]) + const out = await reviewFinishedDispatches(pool) + expect(out.acted).toEqual([`#${ISSUE}:sendBack`]) + const update = reviewUpdate(queries) + const note = String(update?.params[2]) + expect(note).toContain('parses clean') + expect(note).toContain('stub markers') + // Judged and found wanting: this attempt spends the retry ceiling. + expect(update?.params[4]).toBe(true) + }, + ) + + it.if(Boolean(T27C && QUEEND))( + 'accepts a bee whose "met" the compiler confirms', + async () => { + process.env.WORKSPACE_DIR = repoWithCommit([ + { path: 'specs/x.t27', body: 'module M\nfn a() -> i32 { return 1 }\n' }, + ]) + process.env.T27C_BIN = T27C + process.env.TRIOS_QUEEND_PATH = QUEEND + const { pool, queries } = reviewPool([finishedRow(block(true))]) + const out = await reviewFinishedDispatches(pool) + expect(out.acted).toEqual([`#${ISSUE}:accept`]) + expect(reviewUpdate(queries)?.params[1]).toBe('accept') + }, + ) +})