test: enforce headless token budgets and release comparisons

.github/workflows/ci.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main]
 
 permissions:
   contents: read
@@ -40,5 +39,33 @@ jobs:
           sudo apt-get install --no-install-recommends -y ripgrep
           rg --version
 
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+        with:
+          python-version: "3.12"
+
+      # Setup may download pinned wheels/ranks. Capture and accounting are offline.
+      - name: Provision reference tokenizer
+        run: python3 -I -B script/token-budget/setup.py --directory "$RUNNER_TEMP/token-budget-environment"
+
+      - name: Enforce actual SDK prefix token budgets
+        run: '"$RUNNER_TEMP/token-budget-environment/venv/bin/python3" -I -B script/token-budget/run.py --output "$RUNNER_TEMP/token-budget-evidence"'
+
+      - name: Verify token fixture loopback binding
+        working-directory: script/token-budget/release-fixtures
+        run: bun test token-stub.test.ts
+
+      - name: Verify token-budget fault controls
+        run: '"$RUNNER_TEMP/token-budget-environment/venv/bin/python3" -I -B -m unittest discover -s script/token-budget -p "test_*.py" -v'
+        env:
+          TOKEN_BUDGET_TEST_CAMPAIGN: ${{ runner.temp }}/token-budget-evidence
+
+      - name: Retain token-budget captures
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: token-budget-evidence
+          path: ${{ runner.temp }}/token-budget-evidence
+          if-no-files-found: warn
+
       - name: Run Tests
         run: bun turbo test --concurrency=100%

EVENTS.md

@@ -105,16 +105,16 @@ Emitted once when the session begins.
 
 ### `tool_catalog`
 
-Emitted once per session, immediately after `session_start` and before the first model turn (present even for zero-turn or immediately-failing runs). Lists every tool and skill that was resolved for the session so consumers can structurally verify tool exposure without string-matching the model's prose.
+Emitted once per session, immediately after `session_start` and before the first model turn (present even for zero-turn or immediately-failing runs). Reports builtin registry and resolved MCP tool inventory, plus in-scope skills. This session-level snapshot supports inventory diagnostics; it is not the effective tool list for each model request.
 
 ```json
 {
   "type": "tool_catalog",
   "sessionID": "ses_…",
   "timestamp": 1741500000000,
   "tools": [
-    { "name": "record_finding", "source": "mcp", "server": "aictrl" },
-    { "name": "record_review_completed", "source": "mcp", "server": "aictrl" },
+    { "name": "aictrl_record_finding", "source": "mcp", "server": "aictrl" },
+    { "name": "aictrl_record_review_completed", "source": "mcp", "server": "aictrl" },
     { "name": "bash", "source": "builtin" },
     { "name": "read", "source": "builtin" }
   ],
@@ -125,13 +125,13 @@ Emitted once per session, immediately after `session_start` and before the first
 }
 ```
 
-#### `tools[]` — builtin and MCP tools resolved for this session
+#### `tools[]` — builtin registry and resolved MCP inventory for this session
 
-- **`name`** (string, **required**) — the tool/function name as exposed to the model. Mirrors upstream `ToolListItem.id`. For MCP tools this is `{serverName}_{toolName}` (both sanitised to `[a-zA-Z0-9_-]`).
+- **`name`** (string, **required**) — the tool/function identifier used when a tool is dispatched. Mirrors upstream `ToolListItem.id`. For MCP tools this is `{serverName}_{toolName}` (both sanitised to `[a-zA-Z0-9_-]`). The identifier does not guarantee inclusion in a model request.
 - **`source`** (string, **required**) — `"builtin"` for tools built into the CLI; `"mcp"` for tools provided by a connected MCP server.
 - **`server`** (string, optional) — the MCP server (client name from config) that provides this tool. Present only when `source` is `"mcp"`.
 
-`description` and `parameters` are intentionally omitted to keep the event lean. The primary consumer only needs `name` + `source` to verify tool exposure.
+`description` and `parameters` are intentionally omitted to keep the event lean. `name` + `source` identify inventory entries and their provenance; they do not prove dispatch exposure.
 
 #### `skills[]` — skill packs resolved for this session (separate from `tools[]`)
 
@@ -140,9 +140,9 @@ Only skills in scope for the run are listed; see [Skill Events](#skill-events).
 - **`name`** (string, **required**) — skill name from `SKILL.md` frontmatter.
 - **`version`** (string or null, **required**) — version string from the `version` field in `SKILL.md` frontmatter. `null` when no version is declared.
 
-> **Use case:** The server-side completion gate (aictrl-dev/aictrl #3216) uses `tools[]` to verify that `record_finding` and `record_review_completed` were actually in the model's function list at dispatch time, and `skills[]` to record which skill version produced the review. Detects the "silent success" failure mode where a missing MCP server lets a run complete green without ever having the review tools available.
+> **Use case:** A server-side completion gate (aictrl-dev/aictrl #3216) can use `tools[]` to detect missing expected inventory and `skills[]` to record resolved skill versions. To verify that `aictrl_record_finding` and `aictrl_record_review_completed` were offered on a particular turn, it must inspect that actual model request's dispatched function list after filtering. Inventory presence alone proves neither dispatch nor a successful review-tool call.
 >
-> **Note on builtin tool filtering:** The `source: "builtin"` entries in `tools[]` reflect the _instance-level superset_ registered in `ToolRegistry`. Per-model filters (e.g. `apply_patch` only on gpt-\*, `codesearch`/`websearch` only for aictrl provider) are applied at dispatch time inside `resolveTools` and are **not** reflected here. Consumers should treat the presence of a builtin tool name as "registered and potentially available", not as "guaranteed to appear in the model's function list". The strong structural guarantee (tool present ↔ tool in dispatch list) applies only to `source: "mcp"` entries.
+> **Inventory versus dispatch:** Builtin entries reflect the _instance-level superset_ registered in `ToolRegistry`; model-specific builtin filters are applied when resolving each request. MCP entries come from connected servers' resolved tool catalogs. Both sources can be excluded by effective work-tool selection, permissions or request-specific disabling. The shared MCP naming helper guarantees identifier consistency, not inclusion in the dispatched function list.
 
 ### `session_complete`
 

README.md

@@ -63,7 +63,7 @@ Stdin is read only when a standalone `-` is a message operand, or when no messag
 
 The marker can appear before or after `--`; repeated markers still read stdin once. Markers are removed from the model message, command arguments and derived session title. `--` ends option parsing, so `run -- -` also selects stdin. An operand such as `"literal -"` remains ordinary text. An explicit empty operand (`run ""`) does not select implicit stdin. Selected stdin is read to EOF and appended after one actual newline. Ordinary prompt operands are joined with spaces without adding artificial quotes or backslash escapes; quotes, backslashes and whitespace inside an operand remain its text.
 
-Option values belong to their options: `--model=-` or `--command=-` does not select stdin, and scalar options also accept a separated hyphen value. Use `--file=-` to attach a file literally named `-`; `--file -` is missing its required file value. Multiple file values and repeated `--skill` flags retain their existing behavior. Required option values are checked by the argument parser; explicit `--title ""` remains supported.
+Scalar required-value options own a separated hyphen: `--model -` and `--command -` do not select stdin, nor do their `=` forms. Optional `--port` leaves a separated `-` as the stdin operand (`--port -`). Use `--file=-` to attach a file literally named `-`; `--file -` is missing its required file value. Multiple file values and repeated `--skill` flags retain their existing behavior. Required option values are checked by the argument parser; explicit `--title ""` remains supported.
 
 An explicit marker reads stdin even from a terminal. Any selected stream can wait for EOF indefinitely; this change adds no stdin timeout. Callers must close selected input or apply their own supervisor deadline.
 

RELEASING.md

@@ -10,6 +10,29 @@
    bun test test/cli/run-mcp-discovery.test.ts test/mcp/discovery-recovery.test.ts test/cli/publish-workflow.test.ts test/cli/run-provider-finish.test.ts test/cli/run-signal-cancellation.test.ts test/cli/classify-session-error.test.ts test/session/idle.test.ts test/session/processor-idle.test.ts
    ```
 
+   Before publication, run the separate current-versus-previous native release
+   comparison. Prepare and review explicit candidate and previous published binary
+   artifacts, their SHA256/version/provenance pins and one existing immutable
+   fixture image using [the manifest contract](script/token-budget/release-fixtures/README.md).
+   Provision the pinned Python counter in a fresh directory (requires Python3.12
+   and network access during setup only), then capture and count offline:
+
+   ```bash
+   python3.12 -I -B script/token-budget/setup.py --directory /absolute/new-counter-env
+   bun run --no-install --no-env-file script/token-budget/release.ts /absolute/release-inputs.json /absolute/new-release-evidence
+   /absolute/new-counter-env/venv/bin/python3 -I -B script/token-budget/release-count.py /absolute/new-release-evidence/campaign.json
+   ```
+
+   Run these commands from the repository root. Require valid current-default and
+   previous-default task runs, raw requests/responses and actual Read-result
+   evidence, verified versions/binary hashes, successful cleanup and
+   `release-comparison.json`. Review first-input and session input/output/total
+   deltas before publication and record any accepted regression with rationale.
+   Minimal arms are measured only when each actual binary advertises `--tools`;
+   an unsupported previous arm must be labeled UNSUPPORTED, never synthesized
+   through permission settings. This scripted reference-token gate is separate
+   from source CI token budgets and from provider billing or model-quality claims.
+
    Wait for CI build, workspace typecheck and tests to pass and address reviews.
 
 3. Merge the approved PR and record the resulting `main` commit SHA. Confirm
@@ -43,15 +66,15 @@
 publish workflow publishes each as a platform package. It then lists them in
 the wrapper's `optionalDependencies`:
 
-| Package | `os` | `cpu` | `libc` |
-| --- | --- | --- | --- |
-| `@aictrl/cli-linux-x64-baseline` | linux | x64 | glibc |
-| `@aictrl/cli-linux-x64-baseline-musl` | linux | x64 | musl |
-| `@aictrl/cli-linux-arm64` | linux | arm64 | glibc |
-| `@aictrl/cli-linux-arm64-musl` | linux | arm64 | musl |
-| `@aictrl/cli-darwin-x64-baseline` | darwin | x64 | |
-| `@aictrl/cli-darwin-arm64` | darwin | arm64 | |
-| `@aictrl/cli-windows-x64-baseline` | win32 | x64 | |
+| Package                               | `os`   | `cpu` | `libc` |
+| ------------------------------------- | ------ | ----- | ------ |
+| `@aictrl/cli-linux-x64-baseline`      | linux  | x64   | glibc  |
+| `@aictrl/cli-linux-x64-baseline-musl` | linux  | x64   | musl   |
+| `@aictrl/cli-linux-arm64`             | linux  | arm64 | glibc  |
+| `@aictrl/cli-linux-arm64-musl`        | linux  | arm64 | musl   |
+| `@aictrl/cli-darwin-x64-baseline`     | darwin | x64   |        |
+| `@aictrl/cli-darwin-arm64`            | darwin | arm64 |        |
+| `@aictrl/cli-windows-x64-baseline`    | win32  | x64   |        |
 
 npm installs only the package that matches the host. Every x64 package is the
 baseline (no AVX2) build: Bun ships the same binary for its baseline and AVX2

packages/cli/README.md

@@ -104,7 +104,7 @@ Aictrl is built to be used in non-interactive environments.
 ### Stdin Piping
 
 ```bash
-cat diff.txt | aictrl run "review this code change"
+cat diff.txt | aictrl run "review this code change" -
 ```
 
 ### JSON Events
@@ -277,6 +277,25 @@ If your agents should commit or open PRs, supply your own procedure through a
 skill, `AGENTS.md` or the `instructions` config, and run the CLI in an isolated
 environment with credentials scoped to what the task needs.
 
+## Project instruction isolation
+
+Set `AICTRL_DISABLE_PROJECT_CONFIG=true` to disable automatic project instruction
+discovery as well as project configuration. This excludes root `AGENTS.md`,
+`.opencode/AGENTS.md`, and nested `AGENTS.md` files that would otherwise be
+attached when a tool reads a project file.
+
+Trusted global instruction files (`~/.config/aictrl/AGENTS.md` and
+`AICTRL_CONFIG_DIR/AGENTS.md`) and explicit `instructions` entries in caller
+configuration remain available. Relative caller instructions resolve from
+`AICTRL_CONFIG_DIR` when project configuration is disabled; without that directory,
+relative entries are skipped. Absolute paths and URLs remain explicit inputs.
+
+This controls automatic instruction discovery. Project files and tool output
+remain untrusted task data, so CI jobs still need scoped credentials, configured
+permissions, and process isolation. Keep any existing instruction-file deletion
+workaround until the installed release includes this behavior and has been
+verified.
+
 ## Local Development
 
 ```bash

packages/cli/src/session/instruction.ts

@@ -19,7 +19,9 @@ function globalFiles() {
     files.push(path.join(Flag.AICTRL_CONFIG_DIR, "AGENTS.md"))
   }
   files.push(path.join(Global.Path.config, "AGENTS.md"))
-  files.push(path.join(Instance.directory, ".opencode", "AGENTS.md"))
+  if (!Flag.AICTRL_DISABLE_PROJECT_CONFIG) {
+    files.push(path.join(Instance.directory, ".opencode", "AGENTS.md"))
+  }
   return files
 }
 
@@ -181,6 +183,7 @@ export namespace InstructionPrompt {
   }
 
   export async function resolve(messages: MessageV2.WithParts[], filepath: string, messageID: string) {
+    if (Flag.AICTRL_DISABLE_PROJECT_CONFIG) return []
     const system = await systemPaths()
     const already = loaded(messages)
     const results: { filepath: string; content: string }[] = []

packages/cli/test/cli/fixture/command-template-capture.ts

@@ -25,10 +25,12 @@ export async function capture(input: Input) {
     const ownedFile = input.ownedRedirect ? path.join(cwd, "target; /usr/bin/printf INERT_REDIRECT") : undefined
     if (ownedFile) await fs.mkdir(path.dirname(ownedFile), { recursive: true })
     const requests: Record<string, unknown>[] = []
+    let requestArrivals = 0
     const server = Bun.serve({
       hostname: "127.0.0.1",
       port: 0,
       async fetch(request) {
+        requestArrivals++
         if (new URL(request.url).pathname !== "/v1/chat/completions") throw new Error("Unexpected provider path")
         requests.push((await request.json()) as Record<string, unknown>)
         const chunks = [
@@ -165,7 +167,7 @@ export async function capture(input: Input) {
             stages.push(
               setTimeout(() => {
                 eofAt = performance.now() - started
-                requestsBeforeEOF = requests.length
+                requestsBeforeEOF = requestArrivals
                 child.stdin!.end()
               }, 100),
             )
@@ -249,6 +251,7 @@ export async function capture(input: Input) {
       firstBytesAt,
       eofAt,
       requestsBeforeEOF,
+      requestArrivals,
       command,
       cwd,
       code,