diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3a1c34a..dbe2dff0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,6 @@ on: push: branches: [main] pull_request: - branches: [main] permissions: contents: read @@ -40,5 +39,33 @@ jobs: sudo apt-get install --no-install-recommends -y ripgrep rg --version + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + + # Setup may download pinned wheels/ranks. Capture and accounting are offline. + - name: Provision reference tokenizer + run: python3 -I -B script/token-budget/setup.py --directory "$RUNNER_TEMP/token-budget-environment" + + - name: Enforce actual SDK prefix token budgets + run: '"$RUNNER_TEMP/token-budget-environment/venv/bin/python3" -I -B script/token-budget/run.py --output "$RUNNER_TEMP/token-budget-evidence"' + + - name: Verify token fixture loopback binding + working-directory: script/token-budget/release-fixtures + run: bun test token-stub.test.ts + + - name: Verify token-budget fault controls + run: '"$RUNNER_TEMP/token-budget-environment/venv/bin/python3" -I -B -m unittest discover -s script/token-budget -p "test_*.py" -v' + env: + TOKEN_BUDGET_TEST_CAMPAIGN: ${{ runner.temp }}/token-budget-evidence + + - name: Retain token-budget captures + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: token-budget-evidence + path: ${{ runner.temp }}/token-budget-evidence + if-no-files-found: warn + - name: Run Tests run: bun turbo test --concurrency=100% diff --git a/EVENTS.md b/EVENTS.md index 85c9fdd9..78dc5ec9 100644 --- a/EVENTS.md +++ b/EVENTS.md @@ -105,7 +105,7 @@ Emitted once when the session begins. ### `tool_catalog` -Emitted once per session, immediately after `session_start` and before the first model turn (present even for zero-turn or immediately-failing runs). Lists every tool and skill that was resolved for the session so consumers can structurally verify tool exposure without string-matching the model's prose. +Emitted once per session, immediately after `session_start` and before the first model turn (present even for zero-turn or immediately-failing runs). Reports builtin registry and resolved MCP tool inventory, plus in-scope skills. This session-level snapshot supports inventory diagnostics; it is not the effective tool list for each model request. ```json { @@ -113,8 +113,8 @@ Emitted once per session, immediately after `session_start` and before the first "sessionID": "ses_…", "timestamp": 1741500000000, "tools": [ - { "name": "record_finding", "source": "mcp", "server": "aictrl" }, - { "name": "record_review_completed", "source": "mcp", "server": "aictrl" }, + { "name": "aictrl_record_finding", "source": "mcp", "server": "aictrl" }, + { "name": "aictrl_record_review_completed", "source": "mcp", "server": "aictrl" }, { "name": "bash", "source": "builtin" }, { "name": "read", "source": "builtin" } ], @@ -125,13 +125,13 @@ Emitted once per session, immediately after `session_start` and before the first } ``` -#### `tools[]` — builtin and MCP tools resolved for this session +#### `tools[]` — builtin registry and resolved MCP inventory for this session -- **`name`** (string, **required**) — the tool/function name as exposed to the model. Mirrors upstream `ToolListItem.id`. For MCP tools this is `{serverName}_{toolName}` (both sanitised to `[a-zA-Z0-9_-]`). +- **`name`** (string, **required**) — the tool/function identifier used when a tool is dispatched. Mirrors upstream `ToolListItem.id`. For MCP tools this is `{serverName}_{toolName}` (both sanitised to `[a-zA-Z0-9_-]`). The identifier does not guarantee inclusion in a model request. - **`source`** (string, **required**) — `"builtin"` for tools built into the CLI; `"mcp"` for tools provided by a connected MCP server. - **`server`** (string, optional) — the MCP server (client name from config) that provides this tool. Present only when `source` is `"mcp"`. -`description` and `parameters` are intentionally omitted to keep the event lean. The primary consumer only needs `name` + `source` to verify tool exposure. +`description` and `parameters` are intentionally omitted to keep the event lean. `name` + `source` identify inventory entries and their provenance; they do not prove dispatch exposure. #### `skills[]` — skill packs resolved for this session (separate from `tools[]`) @@ -140,9 +140,9 @@ Only skills in scope for the run are listed; see [Skill Events](#skill-events). - **`name`** (string, **required**) — skill name from `SKILL.md` frontmatter. - **`version`** (string or null, **required**) — version string from the `version` field in `SKILL.md` frontmatter. `null` when no version is declared. -> **Use case:** The server-side completion gate (aictrl-dev/aictrl #3216) uses `tools[]` to verify that `record_finding` and `record_review_completed` were actually in the model's function list at dispatch time, and `skills[]` to record which skill version produced the review. Detects the "silent success" failure mode where a missing MCP server lets a run complete green without ever having the review tools available. +> **Use case:** A server-side completion gate (aictrl-dev/aictrl #3216) can use `tools[]` to detect missing expected inventory and `skills[]` to record resolved skill versions. To verify that `aictrl_record_finding` and `aictrl_record_review_completed` were offered on a particular turn, it must inspect that actual model request's dispatched function list after filtering. Inventory presence alone proves neither dispatch nor a successful review-tool call. > -> **Note on builtin tool filtering:** The `source: "builtin"` entries in `tools[]` reflect the _instance-level superset_ registered in `ToolRegistry`. Per-model filters (e.g. `apply_patch` only on gpt-\*, `codesearch`/`websearch` only for aictrl provider) are applied at dispatch time inside `resolveTools` and are **not** reflected here. Consumers should treat the presence of a builtin tool name as "registered and potentially available", not as "guaranteed to appear in the model's function list". The strong structural guarantee (tool present ↔ tool in dispatch list) applies only to `source: "mcp"` entries. +> **Inventory versus dispatch:** Builtin entries reflect the _instance-level superset_ registered in `ToolRegistry`; model-specific builtin filters are applied when resolving each request. MCP entries come from connected servers' resolved tool catalogs. Both sources can be excluded by effective work-tool selection, permissions or request-specific disabling. The shared MCP naming helper guarantees identifier consistency, not inclusion in the dispatched function list. ### `session_complete` diff --git a/README.md b/README.md index adccc41b..bdb3d4c8 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Stdin is read only when a standalone `-` is a message operand, or when no messag The marker can appear before or after `--`; repeated markers still read stdin once. Markers are removed from the model message, command arguments and derived session title. `--` ends option parsing, so `run -- -` also selects stdin. An operand such as `"literal -"` remains ordinary text. An explicit empty operand (`run ""`) does not select implicit stdin. Selected stdin is read to EOF and appended after one actual newline. Ordinary prompt operands are joined with spaces without adding artificial quotes or backslash escapes; quotes, backslashes and whitespace inside an operand remain its text. -Option values belong to their options: `--model=-` or `--command=-` does not select stdin, and scalar options also accept a separated hyphen value. Use `--file=-` to attach a file literally named `-`; `--file -` is missing its required file value. Multiple file values and repeated `--skill` flags retain their existing behavior. Required option values are checked by the argument parser; explicit `--title ""` remains supported. +Scalar required-value options own a separated hyphen: `--model -` and `--command -` do not select stdin, nor do their `=` forms. Optional `--port` leaves a separated `-` as the stdin operand (`--port -`). Use `--file=-` to attach a file literally named `-`; `--file -` is missing its required file value. Multiple file values and repeated `--skill` flags retain their existing behavior. Required option values are checked by the argument parser; explicit `--title ""` remains supported. An explicit marker reads stdin even from a terminal. Any selected stream can wait for EOF indefinitely; this change adds no stdin timeout. Callers must close selected input or apply their own supervisor deadline. diff --git a/RELEASING.md b/RELEASING.md index d8d1e639..09a0d6b2 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -10,6 +10,29 @@ bun test test/cli/run-mcp-discovery.test.ts test/mcp/discovery-recovery.test.ts test/cli/publish-workflow.test.ts test/cli/run-provider-finish.test.ts test/cli/run-signal-cancellation.test.ts test/cli/classify-session-error.test.ts test/session/idle.test.ts test/session/processor-idle.test.ts ``` + Before publication, run the separate current-versus-previous native release + comparison. Prepare and review explicit candidate and previous published binary + artifacts, their SHA256/version/provenance pins and one existing immutable + fixture image using [the manifest contract](script/token-budget/release-fixtures/README.md). + Provision the pinned Python counter in a fresh directory (requires Python3.12 + and network access during setup only), then capture and count offline: + + ```bash + python3.12 -I -B script/token-budget/setup.py --directory /absolute/new-counter-env + bun run --no-install --no-env-file script/token-budget/release.ts /absolute/release-inputs.json /absolute/new-release-evidence + /absolute/new-counter-env/venv/bin/python3 -I -B script/token-budget/release-count.py /absolute/new-release-evidence/campaign.json + ``` + + Run these commands from the repository root. Require valid current-default and + previous-default task runs, raw requests/responses and actual Read-result + evidence, verified versions/binary hashes, successful cleanup and + `release-comparison.json`. Review first-input and session input/output/total + deltas before publication and record any accepted regression with rationale. + Minimal arms are measured only when each actual binary advertises `--tools`; + an unsupported previous arm must be labeled UNSUPPORTED, never synthesized + through permission settings. This scripted reference-token gate is separate + from source CI token budgets and from provider billing or model-quality claims. + Wait for CI build, workspace typecheck and tests to pass and address reviews. 3. Merge the approved PR and record the resulting `main` commit SHA. Confirm @@ -43,15 +66,15 @@ publish workflow publishes each as a platform package. It then lists them in the wrapper's `optionalDependencies`: -| Package | `os` | `cpu` | `libc` | -| --- | --- | --- | --- | -| `@aictrl/cli-linux-x64-baseline` | linux | x64 | glibc | -| `@aictrl/cli-linux-x64-baseline-musl` | linux | x64 | musl | -| `@aictrl/cli-linux-arm64` | linux | arm64 | glibc | -| `@aictrl/cli-linux-arm64-musl` | linux | arm64 | musl | -| `@aictrl/cli-darwin-x64-baseline` | darwin | x64 | | -| `@aictrl/cli-darwin-arm64` | darwin | arm64 | | -| `@aictrl/cli-windows-x64-baseline` | win32 | x64 | | +| Package | `os` | `cpu` | `libc` | +| ------------------------------------- | ------ | ----- | ------ | +| `@aictrl/cli-linux-x64-baseline` | linux | x64 | glibc | +| `@aictrl/cli-linux-x64-baseline-musl` | linux | x64 | musl | +| `@aictrl/cli-linux-arm64` | linux | arm64 | glibc | +| `@aictrl/cli-linux-arm64-musl` | linux | arm64 | musl | +| `@aictrl/cli-darwin-x64-baseline` | darwin | x64 | | +| `@aictrl/cli-darwin-arm64` | darwin | arm64 | | +| `@aictrl/cli-windows-x64-baseline` | win32 | x64 | | npm installs only the package that matches the host. Every x64 package is the baseline (no AVX2) build: Bun ships the same binary for its baseline and AVX2 diff --git a/packages/cli/README.md b/packages/cli/README.md index 367fcf63..7cd9bf95 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -104,7 +104,7 @@ Aictrl is built to be used in non-interactive environments. ### Stdin Piping ```bash -cat diff.txt | aictrl run "review this code change" +cat diff.txt | aictrl run "review this code change" - ``` ### JSON Events @@ -277,6 +277,25 @@ If your agents should commit or open PRs, supply your own procedure through a skill, `AGENTS.md` or the `instructions` config, and run the CLI in an isolated environment with credentials scoped to what the task needs. +## Project instruction isolation + +Set `AICTRL_DISABLE_PROJECT_CONFIG=true` to disable automatic project instruction +discovery as well as project configuration. This excludes root `AGENTS.md`, +`.opencode/AGENTS.md`, and nested `AGENTS.md` files that would otherwise be +attached when a tool reads a project file. + +Trusted global instruction files (`~/.config/aictrl/AGENTS.md` and +`AICTRL_CONFIG_DIR/AGENTS.md`) and explicit `instructions` entries in caller +configuration remain available. Relative caller instructions resolve from +`AICTRL_CONFIG_DIR` when project configuration is disabled; without that directory, +relative entries are skipped. Absolute paths and URLs remain explicit inputs. + +This controls automatic instruction discovery. Project files and tool output +remain untrusted task data, so CI jobs still need scoped credentials, configured +permissions, and process isolation. Keep any existing instruction-file deletion +workaround until the installed release includes this behavior and has been +verified. + ## Local Development ```bash diff --git a/packages/cli/src/session/instruction.ts b/packages/cli/src/session/instruction.ts index e1708489..3c05d773 100644 --- a/packages/cli/src/session/instruction.ts +++ b/packages/cli/src/session/instruction.ts @@ -19,7 +19,9 @@ function globalFiles() { files.push(path.join(Flag.AICTRL_CONFIG_DIR, "AGENTS.md")) } files.push(path.join(Global.Path.config, "AGENTS.md")) - files.push(path.join(Instance.directory, ".opencode", "AGENTS.md")) + if (!Flag.AICTRL_DISABLE_PROJECT_CONFIG) { + files.push(path.join(Instance.directory, ".opencode", "AGENTS.md")) + } return files } @@ -181,6 +183,7 @@ export namespace InstructionPrompt { } export async function resolve(messages: MessageV2.WithParts[], filepath: string, messageID: string) { + if (Flag.AICTRL_DISABLE_PROJECT_CONFIG) return [] const system = await systemPaths() const already = loaded(messages) const results: { filepath: string; content: string }[] = [] diff --git a/packages/cli/test/cli/fixture/command-template-capture.ts b/packages/cli/test/cli/fixture/command-template-capture.ts index 6fe45938..a6fc9dea 100644 --- a/packages/cli/test/cli/fixture/command-template-capture.ts +++ b/packages/cli/test/cli/fixture/command-template-capture.ts @@ -25,10 +25,12 @@ export async function capture(input: Input) { const ownedFile = input.ownedRedirect ? path.join(cwd, "target; /usr/bin/printf INERT_REDIRECT") : undefined if (ownedFile) await fs.mkdir(path.dirname(ownedFile), { recursive: true }) const requests: Record[] = [] + let requestArrivals = 0 const server = Bun.serve({ hostname: "127.0.0.1", port: 0, async fetch(request) { + requestArrivals++ if (new URL(request.url).pathname !== "/v1/chat/completions") throw new Error("Unexpected provider path") requests.push((await request.json()) as Record) const chunks = [ @@ -165,7 +167,7 @@ export async function capture(input: Input) { stages.push( setTimeout(() => { eofAt = performance.now() - started - requestsBeforeEOF = requests.length + requestsBeforeEOF = requestArrivals child.stdin!.end() }, 100), ) @@ -249,6 +251,7 @@ export async function capture(input: Input) { firstBytesAt, eofAt, requestsBeforeEOF, + requestArrivals, command, cwd, code, diff --git a/packages/cli/test/cli/fixture/integrated-capture.ts b/packages/cli/test/cli/fixture/integrated-capture.ts new file mode 100644 index 00000000..ca91b001 --- /dev/null +++ b/packages/cli/test/cli/fixture/integrated-capture.ts @@ -0,0 +1,403 @@ +import { spawn, spawnSync } from "node:child_process" +import fs from "node:fs/promises" +import { existsSync, readFileSync } from "node:fs" +import os from "node:os" +import { Database } from "bun:sqlite" +import path from "node:path" + +const guard = path.join(import.meta.dir, "stdin-guard.ts") +const pause = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +export type Input = { + name: string + entry: "index" | "headless" + operands: string[] + stdin?: string + open?: boolean + tools?: string + configSelection?: string[] + extraArgs?: string[] + attach?: boolean + read?: boolean + profile?: boolean + globalInstructions?: boolean + disableProject?: boolean + denyRead?: boolean + deadlineMs?: number +} +export async function capture(input: Input) { + const cwd = await fs.mkdtemp(path.join(os.tmpdir(), "aictrl-command-arguments-")) + const evidence = process.env.COMMAND_ARGUMENTS_EVIDENCE + const cleanup: (() => void | Promise)[] = [ + async () => { + if (!evidence) await fs.rm(cwd, { recursive: true, force: true }) + }, + ] + let failed = false + try { + const files: Record = { + "AGENTS.md": "ROOT_AUTO_MARKER", + ".opencode/AGENTS.md": "OPENCODE_AUTO_MARKER", + "nested/AGENTS.md": "NESTED_AUTO_MARKER", + "nested/fixture.txt": "READ_FIXTURE_EXACT\n", + "config/aictrl/AGENTS.md": "TRUSTED_GLOBAL_MARKER", + "profile/AGENTS.md": "TRUSTED_PROFILE_MARKER", + "explicit.txt": "EXPLICIT_CALLER_MARKER", + "project-rule.txt": "PROJECT_RULE_MARKER", + } + if (input.profile === false) delete files["profile/AGENTS.md"] + const initialized = spawnSync("git", ["init", "--quiet"], { + cwd, + env: { + PATH: [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter), + HOME: path.join(cwd, "home"), + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + }, + }) + if (initialized.status !== 0) throw new Error("Owned fixture git init failed: " + initialized.stderr.toString()) + if (input.globalInstructions === false) delete files["config/aictrl/AGENTS.md"] + for (const [name, text] of Object.entries(files)) { + await fs.mkdir(path.dirname(path.join(cwd, name)), { recursive: true }) + await fs.writeFile(path.join(cwd, name), text) + } + await fs.writeFile(path.join(cwd, "aictrl.json"), JSON.stringify({ instructions: ["project-rule.txt"] })) + const http: { method: string; path: string; body: unknown }[] = [] + const requests: Record[] = [] + let requestArrivals = 0 + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + requestArrivals++ + const route = new URL(request.url).pathname + const arrival: (typeof http)[number] = { method: request.method, path: route, body: undefined } + http.push(arrival) + const body = request.method === "POST" ? await request.json() : null + arrival.body = body + if (route !== "/v1/chat/completions") return Response.json({ unexpected: route }, { status: 500 }) + requests.push(body as Record) + const call = input.read && requests.length === 1 + + const chunks = [ + { + choices: [ + { + index: 0, + delta: { + role: "assistant", + ...(call + ? { + tool_calls: [ + { + index: 0, + id: "read_integrated_1", + type: "function", + function: { + name: "read", + arguments: JSON.stringify({ filePath: path.join(cwd, "nested/fixture.txt") }), + }, + }, + ], + } + : { content: "INTEGRATED_DONE" }), + }, + finish_reason: null, + }, + ], + }, + { + choices: [{ index: 0, delta: {}, finish_reason: call ? "tool_calls" : "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }, + ] + return new Response(chunks.map((value) => `data: ${JSON.stringify(value)}\n\n`).join("") + "data: [DONE]\n\n", { + headers: { "content-type": "text/event-stream" }, + }) + }, + }) + cleanup.push(() => server.stop(true)) + const config = { + enabled_providers: ["fixture"], + provider: { + fixture: { + npm: "@ai-sdk/openai-compatible", + options: { apiKey: "fixture", baseURL: server.url.origin + "/v1" }, + models: { fixture: { name: "fixture", tool_call: true, limit: { context: 100000, output: 1000 } } }, + }, + }, + agent: { title: { disable: true } }, + permission: { "*": "allow", ...(input.denyRead ? { read: "deny" } : {}) }, + instructions: [path.join(cwd, "explicit.txt")], + plugin: [new URL("./integrated-observer-plugin.ts", import.meta.url).href], + command: { + argumentprobe: { template: "STATIC A<$1> B<$2> C<$3> RAW<$ARGUMENTS>" }, + }, + ...(input.configSelection !== undefined ? { tool_allowlist: input.configSelection } : {}), + } + await fs.writeFile(path.join(cwd, "observations.jsonl"), "") + const ledger = path.join(cwd, "network.jsonl") + await fs.writeFile(ledger, "") + const env = { + PATH: [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter), + HOME: path.join(cwd, "home"), + AICTRL_TEST_HOME: path.join(cwd, "home"), + AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(cwd, "managed"), + XDG_DATA_HOME: path.join(cwd, "data"), + XDG_CONFIG_HOME: path.join(cwd, "config"), + XDG_CACHE_HOME: path.join(cwd, "cache"), + XDG_STATE_HOME: path.join(cwd, "state"), + BUN_RUNTIME_TRANSPILER_CACHE_PATH: path.join(cwd, "bun-cache"), + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + TZ: "UTC", + AICTRL_DISABLE_DEFAULT_PLUGINS: "true", + AICTRL_DISABLE_AUTOCOMPACT: "true", + STDIN_FIXTURE_ORIGIN: server.url.origin, + STDIN_FIXTURE_LEDGER: ledger, + AICTRL_CONFIG_CONTENT: JSON.stringify(config), + AICTRL_DISABLE_PROJECT_CONFIG: String(input.disableProject !== false), + ...(input.profile === false ? {} : { AICTRL_CONFIG_DIR: path.join(cwd, "profile") }), + COMMAND_TEMPLATE_OBSERVATIONS: path.join(cwd, "observations.jsonl"), + } + const source = path.resolve(import.meta.dir, `../../../src/${input.entry}.ts`) + const command = [ + "run", + "--no-install", + "--no-env-file", + "--preload", + guard, + "--conditions=browser", + source, + "run", + "--format", + "json", + "--model", + "fixture/fixture", + ...(input.attach ? ["--attach", server.url.origin] : []), + ...(input.tools !== undefined ? ["--tools", input.tools] : []), + ...(input.extraArgs ?? []), + "--command", + "argumentprobe", + ...input.operands, + ] + const child = spawn(process.execPath, command, { + cwd, + env, + detached: true, + stdio: ["stdin" in input || input.open ? "pipe" : "ignore", "pipe", "pipe"], + }) + cleanup.push(async () => { + if (!child.pid) return + try { + process.kill(-child.pid, "SIGKILL") + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return + throw error + } + await pause(150) + try { + process.kill(-child.pid, 0) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return + throw error + } + throw new Error("Fixture process group remains after final cleanup") + }) + let stdout = "", + stderr = "", + deadline = false + const exists = () => { + try { + process.kill(-child.pid!, 0) + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false + throw error + } + } + let staged = false + let firstBytesAt: number | undefined + let eofAt: number | undefined + let requestsBeforeEOF: number | undefined + let beforeEOF: { http: unknown[]; dbExists: boolean; projectCacheExists: boolean; observations: string } | undefined + const started = performance.now() + const stages: ReturnType[] = [] + child.stdin?.on("error", (error: NodeJS.ErrnoException) => { + if (error.code !== "EPIPE") stderr += String(error) + }) + child.stdout!.on("data", (value) => { + stdout += value.toString() + if ("stdin" in input && !staged && stdout.includes('"type":"invocation_start"')) { + staged = true + stages.push( + setTimeout(() => { + firstBytesAt = performance.now() - started + child.stdin!.write(input.stdin ?? "") + stages.push( + setTimeout(() => { + eofAt = performance.now() - started + requestsBeforeEOF = requestArrivals + beforeEOF = { + http: http.map((arrival) => ({ ...arrival })), + dbExists: existsSync(path.join(cwd, "data/aictrl/aictrl.db")), + projectCacheExists: existsSync(path.join(cwd, ".git/aictrl")), + observations: readFileSync(path.join(cwd, "observations.jsonl"), "utf8"), + } + child.stdin!.end() + }, 100), + ) + }, 100), + ) + } + }) + child.stderr!.on("data", (value) => { + stderr += value.toString() + }) + const timer = setTimeout(() => { + deadline = true + if (exists()) process.kill(-child.pid!, "SIGTERM") + }, input.deadlineMs ?? 12000) + const escalation = setTimeout( + () => { + if (deadline && exists()) process.kill(-child.pid!, "SIGKILL") + }, + (input.deadlineMs ?? 12000) + 200, + ) + cleanup.push(() => { + for (const stage of stages) clearTimeout(stage) + clearTimeout(timer) + clearTimeout(escalation) + child.stdin?.destroy() + }) + let code: number | null = null + try { + code = await new Promise((resolve, reject) => { + child.once("error", reject) + child.once("close", (value) => resolve(value)) + }) + } catch (error) { + failed = true + throw error + } finally { + try { + for (const stage of stages) clearTimeout(stage) + child.stdin?.destroy() + clearTimeout(timer) + clearTimeout(escalation) + if (exists()) { + process.kill(-child.pid!, "SIGTERM") + await pause(150) + if (exists()) process.kill(-child.pid!, "SIGKILL") + } + } catch (error) { + console.error("Fixture child cleanup failed", { directory: cwd, error }) + if (!failed) throw error + } + } + // Partial fetch/install observations only; coverage is documented in README.md. + const network = (await fs.readFile(ledger, "utf8")) + .split("\n") + .filter(Boolean) + .map((value) => JSON.parse(value)) + const events = stdout + .split("\n") + .filter((value) => value.startsWith("{")) + .map((value) => JSON.parse(value)) + const dbPath = path.join(cwd, "data/aictrl/aictrl.db") + let sessions: { id: string; title: string }[] = [], + messages: { data: Record }[] = [] + const dbExists = await fs.stat(dbPath).then( + () => true, + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return false + throw error + }, + ) + if (dbExists) { + const db = new Database(dbPath, { readonly: true }) + try { + sessions = db.query("SELECT id,title FROM session").all() as typeof sessions + messages = (db.query("SELECT data FROM message").all() as { data: string }[]).map((row) => ({ + data: JSON.parse(row.data), + })) + } finally { + db.close() + } + } + const result = { + input, + config, + http, + dbExists, + sessions, + messages, + projectCacheExists: await fs.stat(path.join(cwd, ".git/aictrl")).then( + () => true, + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return false + throw error + }, + ), + firstBytesAt, + eofAt, + requestsBeforeEOF, + requestArrivals, + beforeEOF, + command, + cwd, + code, + deadline, + groupGone: !exists(), + stdout, + stderr, + requests, + events, + network, + observations: (await fs.readFile(path.join(cwd, "observations.jsonl"), "utf8")) + .split("\n") + .filter(Boolean) + .map((row) => JSON.parse(row)), + } + if (evidence) { + await fs.mkdir(evidence, { recursive: true }) + const prefix = input.entry + await fs.writeFile(path.join(evidence, `${prefix}-${input.name}.json`), JSON.stringify(result, null, 2) + "\n") + } + if (deadline) throw new Error("Command argument fixture deadline fired: " + JSON.stringify(result)) + if (!result.groupGone) throw new Error("Command argument fixture process group remains") + if (network.some((row) => row.allowed === false)) + throw new Error("Command argument fixture blocked external attempt") + return result + } catch (error) { + failed = true + throw error + } finally { + const errors: unknown[] = [] + for (const dispose of cleanup.reverse()) { + try { + await dispose() + } catch (error) { + errors.push(error) + } + } + if (errors.length) { + console.error("Fixture cleanup failed", { directory: cwd, errors }) + if (!failed) throw new AggregateError(errors, "Fixture cleanup failed") + } + } +} +export function userText(body: Record) { + const messages = body.messages as { role: string; content: string | { type: string; text?: string }[] }[] + return messages + .filter((message) => message.role === "user") + .map((message) => + typeof message.content === "string" + ? message.content + : message.content + .filter((part) => part.type === "text") + .map((part) => part.text ?? "") + .join(""), + ) + .join("\n") +} diff --git a/packages/cli/test/cli/fixture/integrated-observer-plugin.ts b/packages/cli/test/cli/fixture/integrated-observer-plugin.ts new file mode 100644 index 00000000..42d9d3d0 --- /dev/null +++ b/packages/cli/test/cli/fixture/integrated-observer-plugin.ts @@ -0,0 +1,16 @@ +import { appendFileSync } from "node:fs" +import type { Plugin } from "@aictrl/plugin" +const record = (data: unknown) => + appendFileSync(process.env.COMMAND_TEMPLATE_OBSERVATIONS!, JSON.stringify(data) + "\n") +record({ kind: "plugin-import" }) +export const IntegratedObserver: Plugin = async () => { + record({ kind: "plugin-init" }) + return { + "command.execute.before": async (input) => { + record({ kind: "before", input }) + }, + event: async ({ event }) => { + if (event.type === "command.executed") record({ kind: "executed", event }) + }, + } +} diff --git a/packages/cli/test/cli/request-arrival.test.ts b/packages/cli/test/cli/request-arrival.test.ts new file mode 100644 index 00000000..9e14d766 --- /dev/null +++ b/packages/cli/test/cli/request-arrival.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, test } from "bun:test" +import http from "node:http" +import fs from "node:fs/promises" +import path from "node:path" +import ts from "typescript" + +// Exercise the actual owned handler bodies without starting a CLI or claiming +// that a CLI request has been reproduced before stdin EOF. +const fixtures = [ + ["template", "fixture/command-template-capture.ts", "/v1/chat/completions"], + ["integrated", "fixture/integrated-capture.ts", "/v1/chat/completions"], + ["stdin", "run-stdin.test.ts", "/session"], +] as const + +describe("server-observed request arrival", () => { + test.each(fixtures)("%s records arrival while JSON is incomplete", async (name, file, route) => { + const source = ts.createSourceFile( + file, + await fs.readFile(path.join(import.meta.dir, file), "utf8"), + ts.ScriptTarget.Latest, + true, + ) + let handler: ts.MethodDeclaration | undefined + const visit = (node: ts.Node) => { + if (handler) return + if (ts.isCallExpression(node) && node.expression.getText(source) === "Bun.serve") { + const options = node.arguments[0] + if (!options || !ts.isObjectLiteralExpression(options)) throw new Error("Fixture serve options changed") + handler = options.properties.find( + (property): property is ts.MethodDeclaration => + ts.isMethodDeclaration(property) && property.name.getText(source) === "fetch", + ) + if (!handler) throw new Error("Fixture fetch handler missing") + return + } + ts.forEachChild(node, visit) + } + visit(source) + if (!handler) throw new Error("Fixture server missing") + const compiled = new Bun.Transpiler({ loader: "ts" }).transformSync( + `const fixture = { ${handler.getText(source)} }`, + ) + const state = { requests: [] as Record[], http: [] as Record[] } + const capture = new Function( + "state", + "path", + `const requests = state.requests, http = state.http; + let requestArrivals = 0, stream; + const input = { read: false }, cwd = "/owned-arrival-control", sessionID = "ses_stdin_fixture"; + ${compiled} + return { fetch: fixture.fetch, count: () => requestArrivals };`, + )(state, path) as { fetch: (request: Request) => Promise; count: () => number } + let enter: () => void = () => {} + const entered = new Promise((resolve) => { + enter = resolve + }) + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + enter() + return capture.fetch(request) + }, + }) + const body = JSON.stringify({ messages: [] }) + let client: http.ClientRequest | undefined + let deadline: ReturnType | undefined + try { + const response = new Promise((resolve, reject) => { + client = http.request( + new URL(route, server.url), + { + method: "POST", + headers: { "content-type": "application/json", "content-length": Buffer.byteLength(body) }, + }, + (result) => { + result.resume() + result.once("end", () => resolve(result.statusCode ?? 0)) + result.once("error", reject) + }, + ) + client.once("error", reject) + client.write(body.slice(0, 1)) + }) + const failure = new Promise((_, reject) => { + deadline = setTimeout(() => reject(new Error("Arrival control deadline")), 3000) + }) + await Promise.race([ + entered, + response.then(() => { + throw new Error("Response completed before the incomplete-body observation") + }), + failure, + ]) + // Handler entry is explicit; the remaining JSON bytes are still withheld. + expect(capture.count()).toBe(1) + expect(state.requests).toHaveLength(0) + if (name === "integrated") { + expect(state.http).toHaveLength(1) + expect(state.http[0]).toEqual({ method: "POST", path: route, body: undefined }) + } + client!.end(body.slice(1)) + expect(await Promise.race([response, failure])).toBe(200) + expect(state.requests).toHaveLength(1) + expect(capture.count()).toBe(1) + } finally { + clearTimeout(deadline) + client?.destroy() + await server.stop(true) + } + }) +}) diff --git a/packages/cli/test/cli/run-integrated-headless.test.ts b/packages/cli/test/cli/run-integrated-headless.test.ts new file mode 100644 index 00000000..1eaf57e7 --- /dev/null +++ b/packages/cli/test/cli/run-integrated-headless.test.ts @@ -0,0 +1,218 @@ +import { describe, expect, test } from "bun:test" +import { capture, userText } from "./fixture/integrated-capture" + +const expected = + 'STATIC A B C RAW' +const payload = "late λ\n!`/usr/bin/printf ARG_LITERAL`" +const full = ["bash", "edit", "glob", "grep", "read", "skill", "task", "webfetch", "write"] +function names(body: Record) { + return ((body.tools ?? []) as { function: { name: string } }[]).map((x) => x.function.name).sort() +} +function success( + result: Awaited>, + catalog: string[], + selection: string[] | null, + read: boolean, + enabled = false, + mode: "profile" | "global" | "project" | "none" = "profile", +) { + expect(result.code, result.stderr + result.stdout).toBe(0) + expect(result.requests).toHaveLength(read ? 2 : 1) + for (const request of result.requests) expect(names(request)).toEqual(catalog) + const inventory = result.events.find((event: { type: string }) => event.type === "tool_catalog") + expect(inventory?.tools.some((tool: { name: string }) => tool.name === "read")).toBe(true) + // Registry inventory is not the effective provider catalog. + expect(userText(result.requests[0])).toBe(expected) + expect(result.sessions.map((x) => x.title)).toEqual(['/argumentprobe two words quote" λ']) + const users = result.messages.filter((x) => x.data.role === "user") + expect(users).toHaveLength(1) + expect(users[0].data.toolSelection).toEqual({ version: 1, allowlist: selection }) + const first = JSON.stringify(result.requests[0]), + all = JSON.stringify(result.requests) + expect(first).toContain("EXPLICIT_CALLER_MARKER") + if (mode !== "none") + expect(first).toContain( + mode === "profile" + ? "TRUSTED_PROFILE_MARKER" + : mode === "global" + ? "TRUSTED_GLOBAL_MARKER" + : "OPENCODE_AUTO_MARKER", + ) + if (mode !== "global") expect(first).not.toContain("TRUSTED_GLOBAL_MARKER") + if (mode !== "profile") expect(first).not.toContain("TRUSTED_PROFILE_MARKER") + for (const marker of ["ROOT_AUTO_MARKER", "OPENCODE_AUTO_MARKER", "NESTED_AUTO_MARKER", "PROJECT_RULE_MARKER"]) { + if (enabled && (marker !== "OPENCODE_AUTO_MARKER" || mode === "project")) expect(all).toContain(marker) + else expect(all).not.toContain(marker) + } + if (read) { + const tool = (result.requests[1].messages as { role: string; content: unknown }[]).filter((x) => x.role === "tool") + expect(JSON.stringify(tool)).toContain("READ_FIXTURE_EXACT") + } + expect(result.firstBytesAt).toBeGreaterThan(90) + expect(result.eofAt!).toBeGreaterThan(result.firstBytesAt!) + expect(result.requestsBeforeEOF).toBe(0) + const before = result.observations.filter((x: { kind: string }) => x.kind === "before"), + executed = result.observations.filter((x: { kind: string }) => x.kind === "executed") + expect(result.observations.filter((x: { kind: string }) => x.kind === "plugin-import")).toHaveLength(1) + expect(result.observations.filter((x: { kind: string }) => x.kind === "plugin-init")).toHaveLength(1) + expect(before).toHaveLength(1) + expect(executed).toHaveLength(1) + expect(before[0].input.arguments).toBe('two words quote" λ\n' + payload) + expect(executed[0].event.properties.arguments).toBe('two words quote" λ\n' + payload) +} + +describe("coupled selector stdin command and instruction acceptance", () => { + for (const entry of ["index", "headless"] as const) { + test(`${entry}: selected Read consumes exact command data and excludes discovered instructions`, async () => { + const result = await capture({ + entry, + name: "selected-read", + tools: "read", + operands: ["two words", "-", 'quote" λ'], + stdin: payload, + read: true, + }) + success(result, ["read"], ["read"], true) + }, 15000) + test(`${entry}: explicit empty tools preserves command data title and events`, async () => { + const result = await capture({ + entry, + name: "empty-tools", + tools: "", + operands: ["two words", "-", 'quote" λ'], + stdin: payload, + }) + success(result, [], [], false) + }, 15000) + } + for (const item of [ + { name: "default-nine", catalog: full, selection: null, read: true }, + { name: "config-read", configSelection: ["read"], catalog: ["read"], selection: ["read"], read: true }, + { + name: "cli-replaces-empty-config", + configSelection: [], + tools: "read", + catalog: ["read"], + selection: ["read"], + read: true, + }, + { name: "config-empty", configSelection: [], catalog: [], selection: [], read: false }, + { name: "permissions-still-deny", tools: "read", denyRead: true, catalog: [], selection: ["read"], read: false }, + { + name: "trusted-global-control", + profile: false, + catalog: ["read"], + selection: ["read"], + tools: "read", + read: true, + }, + { + name: "opencode-enabled-control", + profile: false, + globalInstructions: false, + disableProject: false, + catalog: ["read"], + selection: ["read"], + tools: "read", + read: true, + }, + { + name: "opencode-disabled-control", + profile: false, + globalInstructions: false, + catalog: ["read"], + selection: ["read"], + tools: "read", + read: true, + }, + { + name: "enabled-discovery-control", + tools: "read", + disableProject: false, + catalog: ["read"], + selection: ["read"], + read: true, + }, + ]) + test( + item.name, + async () => { + const result = await capture({ + entry: "headless", + ...item, + operands: ["two words", "-", 'quote" λ'], + stdin: payload, + }) + success( + result, + item.catalog, + item.selection, + item.read, + item.disableProject === false, + item.name === "opencode-disabled-control" + ? "none" + : item.name === "opencode-enabled-control" + ? "project" + : item.name === "trusted-global-control" + ? "global" + : "profile", + ) + }, + 15000, + ) + for (const item of [ + { name: "attach-explicit-empty-before-stdin", attach: true, tools: "" }, + { name: "malformed-tools-before-stdin", tools: " read" }, + { name: "repeated-tools-before-stdin", extraArgs: ["--tools", "read", "--tools", "bash"] }, + { name: "missing-tools-before-stdin", extraArgs: ["--tools"] }, + ]) + test( + item.name, + async () => { + const result = await capture({ entry: "headless", ...item, operands: ["-"], open: true }) + expect(result.code, result.stderr + result.stdout).toBe(1) + expect(result.http).toEqual([]) + expect(result.requests).toEqual([]) + expect(result.dbExists).toBe(false) + expect(result.projectCacheExists).toBe(false) + expect(result.observations).toEqual([]) + }, + 15000, + ) + test("actual coupled helper deadline never passes as a negative result", async () => { + await expect( + capture({ entry: "index", name: "forced-deadline", tools: "read", operands: ["-"], open: true, deadlineMs: 1 }), + ).rejects.toThrow("deadline fired") + }, 15000) +}) + +test("config-derived empty attach selection rejects after selected EOF without any receiver or local mutation", async () => { + const result = await capture({ + entry: "headless", + name: "attach-config-empty-late-EOF", + attach: true, + configSelection: [], + operands: ["-"], + stdin: "selected data stays local", + }) + expect(result.firstBytesAt).toBeGreaterThan(90) + expect(result.eofAt!).toBeGreaterThan(result.firstBytesAt!) + expect(result.beforeEOF).toEqual({ http: [], dbExists: false, projectCacheExists: false, observations: "" }) + expect(result.code, result.stderr + result.stdout).toBe(1) + expect(result.http).toEqual([]) + expect(result.requests).toEqual([]) + expect(result.dbExists).toBe(false) + expect(result.projectCacheExists).toBe(false) + expect(result.observations).toEqual([]) + const error = result.events.find((event: { type: string }) => event.type === "invocation_error") + expect(error?.code).toBe("INVOCATION_INVALID_ARGUMENTS") + expect(error?.phase).toBe("bootstrap") + expect(result.stderr).toContain( + "Work-tool selection is not supported with --attach: this client cannot verify server-side enforcement.", + ) + expect(result.events.map((event: { type: string }) => event.type)).toEqual([ + "invocation_start", + "invocation_error", + "invocation_complete", + ]) +}, 15000) diff --git a/packages/cli/test/cli/run-stdin.test.ts b/packages/cli/test/cli/run-stdin.test.ts index cda80200..eeaa7499 100644 --- a/packages/cli/test/cli/run-stdin.test.ts +++ b/packages/cli/test/cli/run-stdin.test.ts @@ -33,10 +33,12 @@ async function run(input: { const requests: { method: string; path: string; body: Record | undefined }[] = [] let stream: ReadableStreamDefaultController | undefined const sessionID = "ses_stdin_fixture" + let requestArrivals = 0 const server = Bun.serve({ hostname: "127.0.0.1", port: 0, async fetch(request) { + requestArrivals++ const url = new URL(request.url) const body = request.method === "POST" ? ((await request.json()) as Record) : undefined requests.push({ method: request.method, path: url.pathname, body }) @@ -146,7 +148,7 @@ async function run(input: { staged.push( setTimeout(() => { eofAt = performance.now() - started - requestsBeforeEOF = requests.length + requestsBeforeEOF = requestArrivals child.stdin.end() }, 150), ) @@ -231,6 +233,7 @@ async function run(input: { firstBytesAt, eofAt, requestsBeforeEOF, + requestArrivals, stdout, stderr, requests, diff --git a/packages/cli/test/session/instruction-isolation.test.ts b/packages/cli/test/session/instruction-isolation.test.ts new file mode 100644 index 00000000..b9888e68 --- /dev/null +++ b/packages/cli/test/session/instruction-isolation.test.ts @@ -0,0 +1,157 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import path from "path" +import { InstructionPrompt } from "../../src/session/instruction" +import { Instance } from "../../src/project/instance" +import { Global } from "../../src/global" +import { HeadlessRun } from "../../src/flag/headless-run" +import { ReadTool } from "../../src/tool/read" +import { tmpdir } from "../fixture/fixture" + +describe("project instruction isolation (#250)", () => { + const keys = ["AICTRL_DISABLE_PROJECT_CONFIG", "AICTRL_CONFIG_DIR", "AICTRL_CONFIG_CONTENT"] as const + let env: (string | undefined)[] + let config: string + let headless: boolean + + beforeEach(() => { + env = keys.map((key) => process.env[key]) + config = Global.Path.config + headless = HeadlessRun.enabled() + for (const key of keys) delete process.env[key] + }) + + afterEach(() => { + keys.forEach((key, index) => { + if (env[index] === undefined) delete process.env[key] + else process.env[key] = env[index] + }) + Global.Path.config = config + if (headless) HeadlessRun.enable() + else HeadlessRun.disable() + }) + + test.each([false, true])("disabled=%s controls root and .opencode automatic instructions", async (disabled) => { + await using global = await tmpdir() + Global.Path.config = global.path + if (disabled) process.env.AICTRL_DISABLE_PROJECT_CONFIG = "true" + await using tmp = await tmpdir({ + init: async (dir) => { + await Bun.write(path.join(dir, "AGENTS.md"), "ROOT_ISOLATION_MARKER") + await Bun.write(path.join(dir, ".opencode", "AGENTS.md"), "OPENCODE_ISOLATION_MARKER") + }, + }) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const paths = await InstructionPrompt.systemPaths() + expect(paths.has(path.join(tmp.path, "AGENTS.md"))).toBe(!disabled) + expect(paths.has(path.join(tmp.path, ".opencode", "AGENTS.md"))).toBe(!disabled) + const instructions = (await InstructionPrompt.system()).join("\n") + expect(instructions.includes("ROOT_ISOLATION_MARKER")).toBe(!disabled) + expect(instructions.includes("OPENCODE_ISOLATION_MARKER")).toBe(!disabled) + }, + }) + }) + + test.each([ + [false, "resolve"], + [true, "resolve"], + [false, "read"], + [true, "read"], + ] as const)("disabled=%s controls nested %s instructions", async (disabled, mode) => { + await using global = await tmpdir() + Global.Path.config = global.path + if (disabled) process.env.AICTRL_DISABLE_PROJECT_CONFIG = "true" + await using tmp = await tmpdir({ + init: async (dir) => { + await Bun.write(path.join(dir, "nested", "AGENTS.md"), "NESTED_ISOLATION_MARKER") + await Bun.write(path.join(dir, "nested", "file.txt"), "ordinary file content") + }, + }) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const filepath = path.join(tmp.path, "nested", "file.txt") + if (mode === "resolve") { + const results = await InstructionPrompt.resolve([], filepath, "resolve-isolation") + expect( + results + .map((item) => item.content) + .join("\n") + .includes("NESTED_ISOLATION_MARKER"), + ).toBe(!disabled) + return + } + const read = await ReadTool.init() + const result = await read.execute( + { filePath: filepath }, + { + sessionID: "isolation-test", + messageID: "read-isolation", + callID: "read-isolation", + agent: "build", + abort: AbortSignal.any([]), + messages: [], + metadata: () => {}, + ask: async () => {}, + }, + ) + expect(result.output).toContain("ordinary file content") + expect(result.output.includes("NESTED_ISOLATION_MARKER")).toBe(!disabled) + expect(result.output.includes("")).toBe(!disabled) + expect(result.metadata.loaded).toEqual(disabled ? [] : [path.join(tmp.path, "nested", "AGENTS.md")]) + }, + }) + }) + + test.each([false, true])( + "headless=%s retains trusted and explicit caller instructions when disabled", + async (active) => { + if (active) HeadlessRun.enable() + else HeadlessRun.disable() + process.env.AICTRL_DISABLE_PROJECT_CONFIG = "true" + await using global = await tmpdir({ + init: async (dir) => { + await Bun.write(path.join(dir, "AGENTS.md"), "GLOBAL_ISOLATION_MARKER") + }, + }) + Global.Path.config = global.path + await using profile = await tmpdir({ + init: async (dir) => { + await Bun.write(path.join(dir, "AGENTS.md"), "PROFILE_ISOLATION_MARKER") + await Bun.write(path.join(dir, "relative.md"), "RELATIVE_ISOLATION_MARKER") + }, + }) + await using caller = await tmpdir({ + init: async (dir) => { + await Bun.write(path.join(dir, "caller.md"), "CALLER_ISOLATION_MARKER") + }, + }) + process.env.AICTRL_CONFIG_CONTENT = JSON.stringify({ instructions: [path.join(caller.path, "caller.md")] }) + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const instructions = (await InstructionPrompt.system()).join("\n") + expect(instructions).toContain("GLOBAL_ISOLATION_MARKER") + expect(instructions).toContain("CALLER_ISOLATION_MARKER") + }, + }) + process.env.AICTRL_CONFIG_DIR = profile.path + process.env.AICTRL_CONFIG_CONTENT = JSON.stringify({ + instructions: ["relative.md", path.join(caller.path, "caller.md")], + }) + await using other = await tmpdir() + await Instance.provide({ + directory: other.path, + fn: async () => { + const instructions = (await InstructionPrompt.system()).join("\n") + expect(instructions).toContain("PROFILE_ISOLATION_MARKER") + expect(instructions).not.toContain("GLOBAL_ISOLATION_MARKER") + expect(instructions).toContain("RELATIVE_ISOLATION_MARKER") + expect(instructions).toContain("CALLER_ISOLATION_MARKER") + }, + }) + }, + ) +}) diff --git a/script/token-budget/README.md b/script/token-budget/README.md new file mode 100644 index 00000000..d5a7c934 --- /dev/null +++ b/script/token-budget/README.md @@ -0,0 +1,31 @@ +# Fixed SDK prefix token guard (Linux) + +This guard starts the public CLI twice per configuration with fresh HOME/XDG state and receives its actual OpenAI-compatible SDK request at a loopback endpoint. It verifies the complete nine-tool default catalog and the explicit `--tools read,bash,edit,write` four-tool catalog. It does not import tests, reconstruct production schemas, invoke a paid model, or replace the existing 15,000-character Qwen guard. + +Prerequisites: Linux x86_64, Python 3.12 with `venv`/`pip`, Bun from `package.json`, real `rg` and Git on PATH, and the repository's installed frozen Bun dependencies and built SDK. Missing prerequisites fail explicitly. + +```sh +# Online setup only: creates a NEW owned directory, pinned official PyPI wheels, +# and checksum-verified OpenAI o200k rank data. It never installs JS packages. +python3.12 -I -B script/token-budget/setup.py --directory /tmp/my-token-environment + +# Offline actual CLI capture and accounting; output must be a NEW directory. +/tmp/my-token-environment/venv/bin/python3 -I -B script/token-budget/run.py \ + --output /tmp/my-token-evidence +``` + +The guard uses an exclusive nonblocking lock in the owner-only `/tmp/aictrl-token-budget` directory. Concurrent invocations fail before changing the fixed workspace. Each cold child runs in the literal `/tmp/aictrl-token-budget/workspace`, newly initialized as a Git repository, with UTC and Date fixed before production imports to 21 October 2026. This controls all three raw cwd occurrences, including the two in Bash descriptions/schema. No text normalization is applied. Existing unowned, shared, or symlink fixture directories are rejected. The directory/lock remain for safe reuse; raw captures remain in the requested evidence directory. + +The before-import guard observes `globalThis.fetch` calls in the CLI process and rejects origins other than the loopback fixture. It rejects `fetch.preconnect` and direct `Bun.spawn` calls whose executable is `bun` and whose next argument is exactly `add` or `install`. This ledger is partial: other network APIs, `Bun.spawnSync`, Node child-process APIs, shell-launched programs, and package-manager aliases are outside those controls. A clean ledger establishes no rejected operation on the instrumented paths; it does not establish complete network or installation accounting. Child environments are explicitly allowlisted, without provider credentials or inherited project configuration. These trusted fixture controls are not an OS network sandbox. + +The real CLI has a mandatory 15-second deadline; each outer capture has 25 seconds. A fresh forked Linux subreaper owns only that invocation's capture and adopted descendants, including descendants that create a new session. After the capture parent exits, already-dead children are reaped without being classified as live survivors. Live adopted children are terminated and reaped, and the capture is rejected even if its parent exited successfully. Cleanup repeatedly handles newly adopted children and fails if it cannot verify completion within its cleanup deadline. The caller's other children are outside this ownership scope. The result records the supervisor and descendant cleanup evidence; the legacy `remaining_group_killed` field now indicates any observed live adopted descendant, not just the original process group. + +Evidence output must be a new directory under an existing parent. Its root and per-run directories are owner-only (`0700`), so raw fixture requests and allowlisted environment records are not exposed by an ambient permissive umask. Existing output directories are never overwritten. The fixed workspace directory uses the same ownership/private-directory validation in production and lock controls. + +`count.py` verifies the pinned tokenizer version, exact rank SHA256, raw environment, catalog, single request and the recorded process/partial-fetch-ledger evidence. `reference.py` denies socket/DNS use during accounting. The enforced metric is `o200k_base` encoding of compact sorted Unicode JSON containing actual SDK system-role messages, serialized tools, and instructions when present. The fixture user text is excluded. The full first model input is separately reported using `instructions/messages/input/tools/response_format/text`; transport controls are excluded. System/tool diagnostic counts use separate tokenizations and are not additive BPE contributions. + +`budgets.json` contains two fixed explicit ceilings. Raising either is a reviewed source edit requiring raw evidence and an explanation. `--observe` records an evaluation without enforcing ceilings and is never used by CI. `--source` selects an isolated source copy for QA source-growth controls; CI always uses the repository entry. The default two cold repetitions must produce identical canonical prefix bytes for each arm. + +CI provisions the Python runtime, lock and rank data before capture; capture/accounting do not download. Raw requests, canonical prefixes, network/exit/deadline logs and the complete reference report are retained as `token-budget-evidence`, including when enforcement fails. These counts are reference overhead measurements, not provider billing, cache savings, quality or performance estimates. Release comparisons have a separate current-versus-previous protocol; these two selector arms do not substitute for that requirement. + +Initial portable observation: default **3,441** prefix tokens and selected **2,001**, at the controlled path above (15,453 and 8,973 canonical characters). Both raw request bodies and canonical prefixes were byte-identical across two cold captures per arm. Initial ceilings equal those observed counts with **zero headroom**: deterministic prompt/tool growth should require an explicit reviewed explanation rather than consume a hidden allowance. Full first model input was separately 3,457/2,017. The longer earlier evaluation cwd produced different counts and was not used as a ceiling. diff --git a/script/token-budget/budgets.json b/script/token-budget/budgets.json new file mode 100644 index 00000000..7763cec9 --- /dev/null +++ b/script/token-budget/budgets.json @@ -0,0 +1,4 @@ +{ + "default": 3441, + "four": 2001 +} diff --git a/script/token-budget/capture.ts b/script/token-budget/capture.ts new file mode 100644 index 00000000..8f17815a --- /dev/null +++ b/script/token-budget/capture.ts @@ -0,0 +1,144 @@ +import path from "node:path" +import fs from "node:fs/promises" +import { guard } from "./guard" +const out = process.env.TOKEN_BUDGET_OUTPUT +const work = process.env.TOKEN_BUDGET_WORKSPACE +const source = process.env.TOKEN_BUDGET_SOURCE +if (!out || !work || !source) throw new Error("Explicit output/workspace/source required") +const binpath = [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter) +const rg = Bun.which("rg", { PATH: binpath }) +if (!rg) throw new Error("Real ripgrep prerequisite missing") +const requests: string[] = [] +const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + idleTimeout: 0, + async fetch(request) { + if (new URL(request.url).pathname !== "/v1/chat/completions") throw new Error("Unexpected endpoint " + request.url) + if (request.method !== "POST") throw new Error("Expected POST") + requests.push(await request.text()) + const chunks = [ + { + id: "eval", + object: "chat.completion.chunk", + choices: [{ index: 0, delta: { role: "assistant", content: "Fixture complete" } }], + }, + { id: "eval", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }, + ] + return new Response( + chunks + .map((chunk) => "data: " + JSON.stringify(chunk)) + .concat("data: [DONE]") + .join("\n\n") + "\n\n", + { headers: { "content-type": "text/event-stream" } }, + ) + }, +}) +await fs.mkdir(out, { recursive: true }) +await Bun.write(path.join(out, "guard.ts"), guard) +await Bun.write(path.join(out, "network.jsonl"), "") +const config = { + enabled_providers: ["fixture"], + provider: { + fixture: { + npm: "@ai-sdk/openai-compatible", + options: { apiKey: "fixture", baseURL: server.url.origin + "/v1" }, + models: { fixture: { name: "fixture", tool_call: true, limit: { context: 200000, output: 1000 } } }, + }, + }, + agent: { title: { disable: true } }, + permission: { "*": "allow" }, +} +const command = [ + process.execPath, + "run", + "--no-install", + "--no-env-file", + "--preload", + path.join(import.meta.dir, "clock.js"), + "--preload", + path.join(out, "guard.ts"), + "--conditions=browser", + source, + "run", + "--model", + "fixture/fixture", + "--agent", + "build", + "--format", + "json", + ...(process.env.TOKEN_BUDGET_ARM === "four" ? ["--tools", "read,bash,edit,write"] : []), + "Say Fixture complete without tools.", +] +const env = { + PATH: binpath, + SHELL: "/bin/bash", + TZ: "UTC", + HOME: path.join(out, "home"), + AICTRL_TEST_HOME: path.join(out, "home"), + AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(out, "managed"), + XDG_DATA_HOME: path.join(out, "data"), + XDG_CONFIG_HOME: path.join(out, "config"), + XDG_STATE_HOME: path.join(out, "state"), + XDG_CACHE_HOME: path.join(out, "cache"), + BUN_RUNTIME_TRANSPILER_CACHE_PATH: path.join(out, "bun-cache"), + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + AICTRL_DISABLE_PROJECT_CONFIG: "true", + AICTRL_DISABLE_DEFAULT_PLUGINS: "true", + AICTRL_DISABLE_AUTOCOMPACT: "true", + AICTRL_MODEL_STREAM_IDLE_TIMEOUT_MS: "3000", + AICTRL_CONFIG_CONTENT: JSON.stringify(config), + SELECTOR_FIXTURE_ORIGIN: server.url.origin, + SELECTOR_FIXTURE_GUARD_LOG: path.join(out, "network.jsonl"), +} +const child = Bun.spawn(command, { cwd: work, env, stdin: "ignore", stdout: "pipe", stderr: "pipe" }) +let deadline = false +const timer = setTimeout(() => { + deadline = true + child.kill("SIGKILL") +}, 15000) +try { + const [stdout, stderr, exit] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]) + clearTimeout(timer) + await Bun.write(path.join(out, "stdout.jsonl"), stdout) + await Bun.write(path.join(out, "stderr.log"), stderr) + for (const [index, body] of requests.entries()) await Bun.write(path.join(out, `request-${index}.json`), body) + const network = (await Bun.file(path.join(out, "network.jsonl")).text()) + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)) + const result = { exit, deadline, command, environment: env, rg, requests: requests.length, network } + await Bun.write(path.join(out, "result.json"), JSON.stringify(result, null, 2) + "\n") + if (deadline || exit !== 0 || requests.length !== 1 || network.some((row) => !row.allowed)) + throw new Error( + "Capture failed: " + JSON.stringify({ exit, deadline, requests: requests.length, network }) + stderr.slice(-2000), + ) + const events = stdout + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)) + if ( + events + .filter((event) => event.type === "text") + .map((event) => event.part?.text) + .join("") !== "Fixture complete" + ) + throw new Error("Exact assistant response missing") + if ( + events.filter((event) => event.type === "invocation_complete" && event.status === "completed").length !== 1 || + events.some((event) => event.type === "error" || event.type === "tool_use") + ) + throw new Error("Unexpected invocation lifecycle") +} catch (error) { + console.error(String(error)) + process.exitCode = 1 +} finally { + clearTimeout(timer) + child.kill("SIGKILL") + server.stop(true) +} diff --git a/script/token-budget/clock.js b/script/token-budget/clock.js new file mode 100644 index 00000000..f1c11845 --- /dev/null +++ b/script/token-budget/clock.js @@ -0,0 +1,13 @@ +const epoch = 1792584000000 // 2026-10-21T12:00:00.000Z +const NativeDate = Date +globalThis.Date = new Proxy(NativeDate, { + construct(target, args, next) { + return Reflect.construct(target, args.length ? args : [epoch], next) + }, + apply() { + return new NativeDate(epoch).toString() + }, + get(target, key, receiver) { + return key === "now" ? () => epoch : Reflect.get(target, key, receiver) + }, +}) diff --git a/script/token-budget/count.py b/script/token-budget/count.py new file mode 100644 index 00000000..13a8dc38 --- /dev/null +++ b/script/token-budget/count.py @@ -0,0 +1,91 @@ +"""Count actual SDK captures; no production schema reassembly or normalization.""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from reference import canonical, input_text, load_encoding, RANK_SHA256 + +EXPECTED = {'default': ['bash', 'edit', 'glob', 'grep', 'read', 'skill', 'task', 'webfetch', 'write'], + 'four': ['bash', 'edit', 'read', 'write']} +ENVIRONMENT = "\n Working directory: /tmp/aictrl-token-budget/workspace\n Is directory a git repo: yes\n Platform: linux\n Today's date: Wed Oct 21 2026\n" + + +def analyze(output, observe=False): + encoding = load_encoding() + count = lambda value: len(encoding.encode_ordinary(canonical(value))) + campaign = json.loads((output / 'campaign.json').read_text()) + runs = campaign['runs'] + if campaign['workspace'] != '/tmp/aictrl-token-budget/workspace' or campaign['raw_normalization'] is not False: + raise ValueError('Wrong raw fixture conditions') + repeats = campaign['repeats'] + if not isinstance(repeats, int) or repeats < 2 or len(runs) != 2 * repeats: + raise ValueError('Require complete repeated two-arm capture') + rows = [] + for arm, names in EXPECTED.items(): + for repeat in range(1, repeats + 1): + out = output / f'{arm}-{repeat}' + controller = json.loads((out / 'controller-result.json').read_text()) + if controller not in runs or controller['exit'] or controller['timedOut'] or controller['remaining_group_killed']: + raise ValueError('Unsuccessful controller or surviving process group') + run = json.loads((out / 'result.json').read_text()) + if run['exit'] or run['deadline'] or run['requests'] != 1 or any(not x['allowed'] for x in run['network']): + raise ValueError('Invalid capture exit/deadline/request/network evidence') + ledger = [json.loads(line) for line in (out / 'network.jsonl').read_text().splitlines() if line] + if ledger != run['network'] or len(ledger) != 1 or ledger[0]['path'] != '/v1/chat/completions': + raise ValueError('Unaccounted network ledger or auxiliary request') + if len(list(out.glob('request-*.json'))) != 1: + raise ValueError('Unaccounted request capture') + raw = (out / 'request-0.json').read_text() + body = json.loads(raw) + if body.get('model') != 'fixture': + raise ValueError('Wrong fixture model') + tools = body['tools'] + if sorted(tool['function']['name'] for tool in tools) != names: + raise ValueError('Wrong actual tool catalog: ' + arm) + systems = [m for m in body['messages'] if m['role'] == 'system'] + users = [m for m in body['messages'] if m['role'] == 'user'] + if len(systems) != 1 or len(users) != 1 or len(body['messages']) != 2: + raise ValueError('Unexpected single-turn message roles') + text = systems[0]['content'] + if 'You are aictrl' not in text: + raise ValueError('Base prompt absent') + if re.findall(r'.*?', text, re.S) != [ENVIRONMENT]: + raise ValueError('Uncontrolled raw environment') + prefix = {'messages': systems, 'tools': tools} + if 'instructions' in body: + prefix['instructions'] = body['instructions'] + payload = canonical(prefix) + (out / 'prefix.canonical.json').write_text(payload) + rows.append({'arm': arm, 'repeat': repeat, 'raw_body_sha256': hashlib.sha256(raw.encode()).hexdigest(), + 'prefix_sha256': hashlib.sha256(payload.encode()).hexdigest(), 'prefix_chars': len(payload), + 'prefix_reference_tokens': count(prefix), + 'full_first_model_input_reference_tokens': len(encoding.encode_ordinary(input_text(raw))), + 'system_separate_tokens': count({'messages': systems}), 'tools_separate_tokens': count({'tools': tools}), + 'tool_names': names}) + for arm in EXPECTED: + if len({r['prefix_sha256'] for r in rows if r['arm'] == arm}) != 1: + raise ValueError('Raw prefix differs between cold repetitions: ' + arm) + budgets = None if observe else json.loads((Path(__file__).parent / 'budgets.json').read_text()) + violations = [] if observe else [f"{r['arm']}: {r['prefix_reference_tokens']} > {budgets[r['arm']]}" for r in rows + if r['prefix_reference_tokens'] > budgets[r['arm']]] + report = {'metric': 'raw SDK prefix o200k_base reference tokens; not provider billing/cache/quality/speed', + 'raw_normalization': False, 'tokenizer': 'tiktoken0.12.0/o200k_base', 'rank_sha256': RANK_SHA256, + 'piece_warning': 'System/tool counts encode separate projections and are not additive BPE contributions.', + 'budgets': budgets, 'violations': violations, 'rows': rows} + (output / 'reference-counts.json').write_text(json.dumps(report, indent=2) + '\n') + print(json.dumps(report, indent=2)) + if violations: + raise RuntimeError('Fixed prefix budget exceeded; review source change or explicit budget edit: ' + '; '.join(violations)) + return report + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('output', type=Path) + parser.add_argument('--observe', action='store_true') + args = parser.parse_args() + analyze(args.output, args.observe) diff --git a/script/token-budget/guard.ts b/script/token-budget/guard.ts new file mode 100644 index 00000000..d7f70ac1 --- /dev/null +++ b/script/token-budget/guard.ts @@ -0,0 +1,25 @@ +const guard = [ + 'import { appendFileSync } from "node:fs"', + "const original = globalThis.fetch", + "const spawn = Bun.spawn.bind(Bun)", + "const checked = (...args) => {", + " const command = Array.isArray(args[0]) ? args[0] : args[0]?.cmd", + " if (command && /(?:^|\\/)bun$/.test(String(command[0])) && ['add','install'].includes(command[1])) throw new Error('Fixture forbids dependency installation')", + " return spawn(...args)", + "}", + "Bun.spawn = checked", + "globalThis.fetch = Object.assign(async (input, init) => {", + " const url = new URL(input instanceof Request ? input.url : String(input))", + " if (process.env.SELECTOR_FIXTURE_CODEX === 'true' && url.origin === 'https://chatgpt.com' && url.pathname === '/backend-api/codex/responses') {", + ' appendFileSync(process.env.SELECTOR_FIXTURE_GUARD_LOG,JSON.stringify({origin:url.origin,path:url.pathname,allowed:true,intercepted:true})+"\\n")', + ' appendFileSync(process.env.SELECTOR_FIXTURE_CODEX_LOG,JSON.stringify(JSON.parse(String(init?.body)))+"\\n")', + " const events=[{type:'response.created',response:{id:'resp-fixture',created_at:1,model:'gpt-5.2',service_tier:null}},{type:'response.output_item.added',output_index:0,item:{type:'message',id:'item-fixture'}},{type:'response.output_text.delta',item_id:'item-fixture',delta:'done',logprobs:null},{type:'response.output_item.done',output_index:0,item:{type:'message',id:'item-fixture'}},{type:'response.completed',response:{incomplete_details:null,usage:{input_tokens:1,input_tokens_details:null,output_tokens:1,output_tokens_details:null},service_tier:null}}]", + " return new Response(events.map(event=>'data: '+JSON.stringify(event)).concat('data: [DONE]').join('\\n\\n')+'\\n\\n',{headers:{'content-type':'text/event-stream'}})", + " }", + " const allowed = url.origin === process.env.SELECTOR_FIXTURE_ORIGIN", + ' appendFileSync(process.env.SELECTOR_FIXTURE_GUARD_LOG, JSON.stringify({ origin: url.origin, path: url.pathname, allowed }) + "\\n")', + ' if (!allowed) throw new Error("Selector fixture blocked nonlocal fetch: " + url.origin)', + " return original(input, init)", + '}, { preconnect: () => { throw new Error("Selector fixture forbids preconnect") } })', +].join("\n") +export { guard } diff --git a/script/token-budget/reference.py b/script/token-budget/reference.py new file mode 100644 index 00000000..05839ab8 --- /dev/null +++ b/script/token-budget/reference.py @@ -0,0 +1,63 @@ +"""Pinned offline reference accounting shared by CI and release comparisons.""" +import hashlib +import importlib.metadata +import json +from pathlib import Path +import os +import socket +import sys + +INPUT_FIELDS = ('instructions', 'messages', 'input', 'tools', 'response_format', 'text') +RANK_SHA256 = '446a9538cb6c348e3516120d7c08b09f57c36495e2acfffe59a5bf8b0cfb1a2d' +RANK_KEY = hashlib.sha1(b'https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken').hexdigest() + + +def canonical(value): + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':')) + + +def input_text(body): + payload = json.loads(body) + if not isinstance(payload, dict): + raise ValueError('Request body must be an object') + selected = {key: payload[key] for key in INPUT_FIELDS if key in payload} + if not any(key in selected for key in ('instructions', 'messages', 'input')): + raise ValueError('No model input found') + return canonical(selected) + + +def output_text(response): + if response['kind'] == 'text': + return response['text'] + if response['kind'] == 'tool': + return canonical({'name': response['name'], 'arguments': response['arguments']}) + raise ValueError('Unknown response kind: ' + str(response.get('kind'))) + + +def load_encoding(): + if sys.version_info[:2] != (3, 12): + raise RuntimeError('Require provisioned Python 3.12') + if importlib.metadata.version('tiktoken') != '0.12.0': + raise RuntimeError('Require pinned tiktoken 0.12.0') + cache = Path(sys.prefix).parent / 'cache' + rank = cache / RANK_KEY + if not rank.is_file() or hashlib.sha256(rank.read_bytes()).hexdigest() != RANK_SHA256: + raise RuntimeError('Missing or mismatched pinned o200k rank data; rerun explicit setup') + attempts = [] + + def deny(*args, **kwargs): + attempts.append(repr(args)) + raise RuntimeError('Offline reference counter forbids network') + + socket.socket.connect = deny + socket.socket.connect_ex = deny + socket.create_connection = deny + socket.getaddrinfo = deny + os.environ['TIKTOKEN_CACHE_DIR'] = str(cache) + import tiktoken + encoding = tiktoken.get_encoding('o200k_base') + if attempts: + raise RuntimeError('Unexpected tokenizer network attempt') + if len(encoding.encode_ordinary('hello world')) != 2: + raise RuntimeError('Known-string tokenizer parity failed') + return encoding diff --git a/script/token-budget/release-count.py b/script/token-budget/release-count.py new file mode 100644 index 00000000..a5059889 --- /dev/null +++ b/script/token-budget/release-count.py @@ -0,0 +1,92 @@ +"""Account the explicit release comparison with the shared pinned offline counter.""" +import gzip +import hashlib +import json +from pathlib import Path +import statistics +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from reference import input_text, output_text, load_encoding + + +def analyze(campaign_path): + campaign = json.loads(campaign_path.read_text()) + target = campaign_path.with_name('release-comparison.json') + if target.exists(): + raise ValueError('Refuse overwriting accounting results') + if campaign['protocolVersion'] != 3 or campaign['failures'] or campaign['REPS'] < 2: + raise ValueError('Incomplete or failed release campaign') + checks = campaign.get('cleanupChecks') + if not isinstance(checks, list) or not checks or not isinstance(checks[-1], dict): + raise ValueError('Incomplete release campaign: final cleanup evidence missing') + final = checks[-1].get('finalCampaignCheck') + if not isinstance(final, dict) or not {'exit', 'timedOut', 'stdout'} <= final.keys(): + raise ValueError('Incomplete release campaign: final cleanup evidence missing') + if final['exit'] != 0 or final['timedOut'] or final['stdout'].strip(): + raise ValueError('Final campaign container cleanup not verified') + encoding = load_encoding() + arms = {} + for arm in campaign['HARNESSES']: + sessions = [] + runs = campaign['raw']['short'][arm] + if len(runs) != campaign['REPS']: + raise ValueError('Incomplete arm: ' + arm) + for run in runs: + if not run['valid'] or run['exit'] or run['timedOut'] or run['mainRequests'] != 4: + raise ValueError('Invalid capture: ' + run['runId']) + log = campaign_path.parent / 'campaign-logs' / (run['runId'] + '.requests-with-bodies.jsonl.gz') + records = [json.loads(line) for line in gzip.decompress(log.read_bytes()).decode().splitlines()] + counted = [] + for r in records: + if r['kind'] not in ('main', 'aux'): + continue + body = r['body'].encode() + if len(body) != r['bytes'] or hashlib.sha256(body).hexdigest() != r['bodySha256']: + raise ValueError('Raw body hash/byte mismatch') + response = r['responseBody'].encode() + if len(response) != r['responseDelivery']['bodyBytes']: + raise ValueError('Raw SSE response byte mismatch') + counted.append({'kind': r['kind'], 'step': r['step'], + 'request_sha256': hashlib.sha256(body).hexdigest(), + 'response_sha256': hashlib.sha256(response).hexdigest(), + 'input': len(encoding.encode_ordinary(input_text(r['body']))), + 'output': len(encoding.encode_ordinary(output_text(r['response'])))}) + main = [r for r in records if r['kind'] == 'main'] + if len(main) != 4 or [r['step'] for r in main] != [0, 1, 2, 3]: + raise ValueError('Expected exactly ordered scripted steps') + for i in range(3): + call = main[i]['response'] + result = next((r for r in main[i+1]['toolResults'] if r['callId'] == f'call_{i}'), None) + if call['kind'] != 'tool' or call['name'] != 'read' or call['arguments']['filePath'] != f'/workspace/notes-{i}.txt' or not result or not result['complete'] or result['matchedLines'] != 20 or any(result.get(k) for k in ['errorNotice','truncationNotice','nativeSuppressed','nativeGuardrailRefusal']): + raise ValueError('Missing actual exact20line ordered Read result') + if len(main[-1]['toolResults']) != 3 or main[-1]['response']['text'] != 'Done.': + raise ValueError('Missing final completion') + if len(counted) != run['mainRequests'] + run['auxRequests']: + raise ValueError('Request count mismatch') + incoming = sum(r['input'] for r in counted) + outgoing = sum(r['output'] for r in counted) + sessions.append({'runId': run['runId'], 'first_input': next(r['input'] for r in counted if r['kind']=='main'), + 'session_input': incoming, 'session_output': outgoing, 'session_total': incoming+outgoing, + 'capture_sha256': hashlib.sha256(log.read_bytes()).hexdigest(), 'requests': counted}) + arms[arm] = {'sessions': sessions, 'median': {k: statistics.median(r[k] for r in sessions) + for k in ('first_input','session_input','session_output','session_total')}} + if not {'current-default','previous-default'} <= arms.keys(): + raise ValueError('Both actual default versions required') + unsupported = {name+'-selected': 'UNSUPPORTED: actual run --help lacks --tools; no permission/config substitute' + for name, supported in campaign['support'].items() if not supported} + deltas = {k: arms['current-default']['median'][k] - arms['previous-default']['median'][k] + for k in ('first_input','session_input','session_output','session_total')} + result = {'measurement': 'offline o200k_base reference tokens; tiktoken0.12.0; no billing/cache/quality/speed claim', + 'campaign_sha256': hashlib.sha256(campaign_path.read_bytes()).hexdigest(), + 'accounting_source_sha256': hashlib.sha256(Path(__file__).with_name('reference.py').read_bytes()).hexdigest(), + 'arms': arms, 'unsupported': unsupported, 'current_minus_previous_default': deltas, + 'gate': 'Complete valid task/provenance/cleanup comparison required; numerical release deltas require reviewer acceptance, not an automatic no-increase policy'} + target.write_text(json.dumps(result, indent=2)+'\n') + print(json.dumps({k:v['median'] for k,v in arms.items()}, indent=2)) + + +if __name__ == '__main__': + if len(sys.argv) != 2: + raise SystemExit('Usage: OWNED_ENV/venv/bin/python3 -I -B script/token-budget/release-count.py CAMPAIGN.json') + analyze(Path(sys.argv[1])) diff --git a/script/token-budget/release-fixtures/LICENSE b/script/token-budget/release-fixtures/LICENSE new file mode 100644 index 00000000..f44de99e --- /dev/null +++ b/script/token-budget/release-fixtures/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 aictrl + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/script/token-budget/release-fixtures/README.md b/script/token-budget/release-fixtures/README.md new file mode 100644 index 00000000..b5b23b58 --- /dev/null +++ b/script/token-budget/release-fixtures/README.md @@ -0,0 +1,58 @@ +The scripted protocol is adapted from the MIT-licensed +[aictrl public benchmark](https://github.com/aictrl-dev/skill-md-research/blob/cc47b1e5f5277de19c222f7216081110571c5504/experiments/headless-cli-benchmark/harness/token-stub.ts). +The vendored copy adds raw SSE response capture. `LICENSE` retains its notice. + +Release comparisons use explicit local native binaries and an explicit existing +Linux amd64 fixture image. The image must provide Node, a fresh empty runner HOME, +UID10001 and `/workspace` containing the benchmark's 55 exact fixtures. The +controller verifies every file hash before a task. It never pulls or builds an +image, downloads an artifact, installs a dependency, or authenticates a provider. +Artifacts must be obtained and their publication/build provenance reviewed first. +`previous.provenance` is a pinned published npm wrapper/platform inspection +manifest containing `npmWrapperVersion`, `publishedPackage.version`, and +`installedPackages.platformPackages` binary version/hash. `current.provenance` +is a pinned completed build manifest with `complete`, `version`, `binarySha256`. +These records bind identity; reviewers still validate their authenticity and the +candidate's source/build relationship. + +Input manifest (paths resolve relative to the manifest): + +```json +{ + "fixtureImage": "sha256:<64 hex digits>", + "repeats": 2, + "current": { + "path": "current-aictrl", + "sha256": "<64 hex digits>", + "version": "", + "provenance": { "path": "current-build.json", "sha256": "<64 hex digits>" } + }, + "previous": { + "path": "previous-aictrl", + "sha256": "<64 hex digits>", + "version": "", + "provenance": { "path": "previous-npm-inspection.json", "sha256": "<64 hex digits>" } + } +} +``` + +The same fixture image, read-only binary bind, fresh HOME, UID10001, 2 CPU, +2 GiB memory, permissive tool permissions, non-GPT `fixture/fixture` model and +scripted short task apply to both versions. Host networking permits the loopback +stub; this is not an OS egress sandbox. The controller receives an allowlisted +process environment without provider credentials. No real inference occurs. +`--tools read,bash,edit,write` adds a minimal arm only when the binary's actual +`run --help` advertises `--tools`. Previous releases without it are explicitly +unsupported; permission configuration never substitutes for schema selection. + +Each session must contain four ordered model requests, three actual exact +20-line Read results and native Done with exit0. Raw request JSON, raw SSE +responses, stdout, stderr, binary/help/fixture preflight and owned-container +cleanup remain in the new output directory, including failures. The shared +pinned offline `reference.py` counts canonical model-facing fields and all +request input/response output, including repeated history and auxiliary calls. +The comparison records first input, session input/output/total and default +current-minus-previous deltas. These are reference tokens, not provider billing, +cache, quality, reasoning or performance measurements. Review numerical release +deltas explicitly; the release gate does not silently accept an increase or +reuse CI budget thresholds as a cross-version claim. diff --git a/script/token-budget/release-fixtures/token-stub.test.ts b/script/token-budget/release-fixtures/token-stub.test.ts new file mode 100644 index 00000000..a3d04292 --- /dev/null +++ b/script/token-budget/release-fixtures/token-stub.test.ts @@ -0,0 +1,89 @@ +import { expect, test } from "bun:test" +import { MARKER, Stub } from "./token-stub" + +for (const api of ["chat", "responses"] as const) { + const body = (main: boolean) => ({ + model: "fixture", + ...(api === "chat" + ? { + messages: [ + { role: "user", content: main ? `${MARKER} short` : "Generate a title" }, + { role: "tool", tool_call_id: "foreign_history_id", content: "Earlier unrelated result" }, + ], + tools: main + ? [ + { + type: "function", + function: { + name: "read", + parameters: { + type: "object", + properties: { filePath: { type: "string" } }, + required: ["filePath"], + }, + }, + }, + ] + : [], + } + : { + input: [ + { role: "user", content: main ? `${MARKER} short` : "Generate a title" }, + { type: "function_call_output", call_id: "foreign_history_id", output: "Earlier unrelated result" }, + ], + tools: main + ? [ + { + type: "function", + name: "read", + parameters: { type: "object", properties: { filePath: { type: "string" } }, required: ["filePath"] }, + }, + ] + : [], + }), + }) + + test(`${api}: auxiliary foreign history does not advance or reject the main script`, async () => { + const stub = new Stub(0) + stub.reset("short") + try { + const response = await fetch( + new URL(api === "chat" ? "/v1/chat/completions" : "/v1/responses", stub.server.url), + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body(false)), + }, + ) + expect(response.status).toBe(200) + expect(await response.text()).toContain("Bench title") + expect(stub.log).toHaveLength(1) + expect(stub.log[0]).toMatchObject({ kind: "aux", step: 0 }) + expect(stub.log[0].toolResults?.[0]?.callId).toBe("foreign_history_id") + expect(stub.log.some((row) => row.error)).toBe(false) + } finally { + await stub.server.stop(true) + } + }) + + test(`${api}: main foreign history still fails explicit scripted-ID validation`, async () => { + const stub = new Stub(0) + stub.reset("short") + try { + const response = await fetch( + new URL(api === "chat" ? "/v1/chat/completions" : "/v1/responses", stub.server.url), + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body(true)), + }, + ) + expect(response.status).toBe(400) + expect(await response.text()).toContain("Tool results lost all scripted call IDs") + expect(stub.log.some((row) => row.error?.includes("Tool results lost all scripted call IDs"))).toBe(true) + expect(stub.log.some((row) => row.kind === "main")).toBe(false) + } finally { + await stub.server.stop(true) + } + }) +} diff --git a/script/token-budget/release-fixtures/token-stub.ts b/script/token-budget/release-fixtures/token-stub.ts new file mode 100644 index 00000000..71d1a901 --- /dev/null +++ b/script/token-budget/release-fixtures/token-stub.ts @@ -0,0 +1,498 @@ +// Token-capture snapshot: adds output argument evidence only. +/** + * Scripted model stub shared by every harness. It speaks two wire APIs on the same port: + * POST /v1/chat/completions (OpenAI chat completions, streaming) aictrl, opencode, pi + * POST /v1/responses (OpenAI Responses API, streaming) codex + * Standard OpenAI model discovery is supported and logged separately. Other backend + * probes are logged and answered 404, so stray calls show up in the results. + * + * The next turn is keyed by the latest returned scripted call ID, so retries cannot + * shift it and pruning older results does not restart the loop. History regression fails. The tool to call is + * picked from the request's own tool list: `read` when the harness has one, otherwise its + * shell tool with `cat`. Argument names come from the tool's JSON schema. + * + * A request is "main" when it carries tools and the BENCH marker from the task prompt. + * Anything else (title generation and the like) is "aux": it gets a short text reply and + * does not advance the script. + */ + +export type Scenario = "trivial" | "short" | "long" | "version" +export type Req = { + at: number + epoch: number + path: string + api: "chat" | "responses" | "other" + kind: "main" | "aux" | "discovery" | "other" + step: number + bytes: number + codeUnits: number + toolResults?: FixtureEvidence[] + error?: string + response?: { + kind: "tool" | "text" + callId?: string + name?: string + arguments?: Record + text?: string + } + responseDelivery?: { + stage: "planned" | "body-created" | "handler-returning" + plannedAt: number + plannedEpoch: number + bodyCreatedAt?: number + bodyCreatedEpoch?: number + handlerReturningAt?: number + handlerReturningEpoch?: number + bodyBytes?: number + completionId?: string + completionCreated?: number + clientAbortedAt?: number + clientAbortedEpoch?: number + } + responseBody?: string + body?: string +} + +export const fixturePath = (scenario: Scenario, index: number) => + `/workspace/${scenario === "long" ? "big" : "notes"}-${index}.txt` +export const MARKER = "BENCHTASK" +export const PROTOCOL_VERSION = 3 +const CHAT_NONCE = crypto.randomUUID().replaceAll("-", "") +let chatCompletionSequence = 0 + +export class Stub { + scenario: Scenario = "trivial" + longSteps = 50 + latency: number + log: Req[] = [] + keepBodies = false + server: ReturnType + + constructor(latency: number) { + this.latency = latency + this.server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + idleTimeout: 0, + fetch: async (req) => { + try { + return await this.handle(req) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + console.error(`Stub rejected request: ${message}`) + this.log.push({ + at: performance.now(), + epoch: Date.now(), + path: new URL(req.url).pathname, + api: "other", + kind: "other", + step: -1, + bytes: 0, + codeUnits: 0, + error: message, + }) + return Response.json({ error: { message, type: "benchmark_validation_error" } }, { status: 400 }) + } + }, + }) + } + + get port() { + return this.server.port + } + + reset(scenario: Scenario) { + this.scenario = scenario + this.log = [] + } + + private async handle(req: Request) { + const at = performance.now() + const epoch = Date.now() + const path = new URL(req.url).pathname + const body = req.method === "POST" ? await req.text() : "" + const api = path.endsWith("/chat/completions") ? "chat" : path.endsWith("/responses") ? "responses" : "other" + if (req.method === "GET" && ["/v1/models", "/models", "/v1/models/fixture", "/models/fixture"].includes(path)) { + this.log.push({ + at, + epoch, + path: `GET ${path}`, + api: "other", + kind: "discovery", + step: -1, + bytes: 0, + codeUnits: 0, + }) + const model = { + id: "fixture", + object: "model", + created: 1, + owned_by: "benchmark", + context_window: 200_000, + context_length: 200_000, + max_output_tokens: 1000, + } + return Response.json(path.endsWith("/fixture") ? model : { object: "list", data: [model] }) + } + if (api === "other") { + this.log.push({ + at, + epoch, + path: `${req.method} ${path}`, + api, + kind: "other", + step: -1, + bytes: Buffer.byteLength(body, "utf8"), + codeUnits: body.length, + }) + return new Response("not found", { status: 404 }) + } + const json = JSON.parse(body) + const tools = toolList(json, api) + const main = tools.length > 0 && body.includes(MARKER) + const resultEvidence = fixtureEvidence(json, api, this.scenario) + const done = main ? scriptProgress(resultEvidence) : 0 + const furthest = Math.max(0, ...this.log.filter((r) => r.kind === "main").map((r) => r.step)) + if (main && done < furthest) throw new Error(`Tool history regressed from scripted step ${furthest} to ${done}`) + const turn = main ? this.next(done, tools) : { text: "Bench title" } + const firstMain = main && !this.log.some((r) => r.kind === "main") + const evidence: Req = { + at, + epoch, + path, + api, + kind: main ? "main" : "aux", + step: done, + bytes: Buffer.byteLength(body, "utf8"), + codeUnits: body.length, + toolResults: resultEvidence, + response: turn.call + ? { kind: "tool", callId: turn.call.id, name: turn.call.name, arguments: turn.call.args } + : { kind: "text", text: turn.text }, + responseDelivery: { stage: "planned", plannedAt: performance.now(), plannedEpoch: Date.now() }, + body: this.keepBodies || firstMain ? body : undefined, + } + this.log.push(evidence) + const delivery = evidence.responseDelivery! + req.signal.addEventListener( + "abort", + () => { + delivery.clientAbortedAt = performance.now() + delivery.clientAbortedEpoch = Date.now() + }, + { once: true }, + ) + await Bun.sleep(this.latency) + const completion = + api === "chat" + ? { id: `chatcmpl-bench-${CHAT_NONCE}-${++chatCompletionSequence}`, created: Math.floor(Date.now() / 1000) } + : undefined + const sse = api === "chat" ? chatSSE(turn, completion!) : responsesSSE(turn, json) + if (this.keepBodies) evidence.responseBody = sse + delivery.stage = "body-created" + delivery.bodyCreatedAt = performance.now() + delivery.bodyCreatedEpoch = Date.now() + delivery.bodyBytes = Buffer.byteLength(sse, "utf8") + if (completion) { + delivery.completionId = completion.id + delivery.completionCreated = completion.created + } + const response = new Response(sse, { headers: { "content-type": "text/event-stream" } }) + // This is the fetch handler's return boundary, not proof of network delivery or client receipt. + delivery.stage = "handler-returning" + delivery.handlerReturningAt = performance.now() + delivery.handlerReturningEpoch = Date.now() + return response + } + + private next(done: number, tools: Tool[]): Turn { + const wanted = this.scenario === "short" ? 3 : this.scenario === "long" ? this.longSteps : 0 + if (done >= wanted) return { text: "Done." } + return { call: fixtureCall(tools, fixturePath(this.scenario, done), done) } + } +} + +// ---- tool selection --------------------------------------------------------------------- +type Tool = { name: string; params: any; kind: "function" | "other" } +type Call = { id: string; name: string; args: Record } +type Turn = { text?: string; call?: Call } + +function toolList(json: any, api: "chat" | "responses"): Tool[] { + const raw: any[] = json.tools ?? [] + return raw.map((t) => + api === "chat" + ? { name: t.function?.name, params: t.function?.parameters, kind: "function" as const } + : { + name: t.name ?? t.type, + params: t.parameters, + kind: t.type === "function" ? ("function" as const) : ("other" as const), + }, + ) +} + +const READ_NAMES = ["read", "read_file", "view"] +const SHELL_NAMES = [ + "developer__shell", + "terminal", + "bash", + "shell", + "exec_command", + "shell_command", + "run_shell_command", +] + +/** A call that reads `file`: the harness's read tool if it has one, else `cat` through its shell tool. */ +export function fixtureCall(tools: Tool[], file: string, n: number): Call { + const fn = tools.filter((t) => t.kind === "function") + const read = fn.find((t) => READ_NAMES.includes(t.name)) + if (read) { + const props = read.params?.properties ?? {} + const key = ["filePath", "path", "file_path"].find((k) => k in props) ?? "path" + return { + id: `call_${n}`, + name: read.name, + args: fillRequired(read, { [key]: file, ...(read.name === "read_file" ? { offset: 1, limit: 2000 } : {}) }), + } + } + const shell = SHELL_NAMES.map((name) => fn.find((t) => t.name === name)).find(Boolean) + if (!shell) throw new Error(`no read or shell tool in: ${tools.map((t) => t.name).join(",")}`) + const props = shell.params?.properties ?? {} + const cmd = `cat ${file}` + if ("cmd" in props) return { id: `call_${n}`, name: shell.name, args: fillRequired(shell, { cmd }) } + if (props.command?.type === "array") + return { id: `call_${n}`, name: shell.name, args: fillRequired(shell, { command: ["bash", "-lc", cmd] }) } + return { id: `call_${n}`, name: shell.name, args: fillRequired(shell, { command: cmd }) } +} + +/** Plausible values for required arguments the script does not set (e.g. a bash `description`). */ +function fillRequired(tool: Tool, args: Record) { + const props = tool.params?.properties ?? {} + for (const key of tool.params?.required ?? []) { + if (key in args) continue + const type = props[key]?.type + args[key] = type === "number" || type === "integer" ? 1 : type === "boolean" ? false : "Read the fixture file" + } + return args +} + +/** Latest scripted call ID advances the loop even when a harness prunes older results. + * Identical request retries yield the same next call. Unknown IDs fail explicitly. + */ +export function scriptProgress(results: FixtureEvidence[]): number { + if (!results.length) return 0 + const indices = results + .map((r) => /^call_(\d+)(?:_|$)/.exec(r.callId)) + .filter(Boolean) + .map((m) => Number(m![1])) + if (!indices.length) throw new Error("Tool results lost all scripted call IDs; cannot advance benchmark safely") + return Math.max(...indices) + 1 +} + +// ---- wire formats ----------------------------------------------------------------------- +const USAGE = { prompt: 100, completion: 10 } + +function chatSSE(turn: Turn, completion: { id: string; created: number }) { + const delta = turn.call + ? { + tool_calls: [ + { + index: 0, + id: turn.call.id, + type: "function", + function: { name: turn.call.name, arguments: JSON.stringify(turn.call.args) }, + }, + ], + } + : { content: turn.text } + const chunks = [ + { choices: [{ index: 0, delta: { role: "assistant", ...delta }, finish_reason: null }] }, + { + choices: [{ index: 0, delta: {}, finish_reason: turn.call ? "tool_calls" : "stop" }], + usage: { + prompt_tokens: USAGE.prompt, + completion_tokens: USAGE.completion, + total_tokens: USAGE.prompt + USAGE.completion, + }, + }, + ] + return ( + chunks + .map( + (c) => + `data: ${JSON.stringify({ id: completion.id, object: "chat.completion.chunk", created: completion.created, model: "fixture", ...c })}\n\n`, + ) + .join("") + "data: [DONE]\n\n" + ) +} + +let responseSeq = 0 +function responsesSSE(turn: Turn, json: any) { + const id = `resp_${++responseSeq}` + const base = { + id, + object: "response", + created_at: 1, + model: json.model ?? "fixture", + status: "in_progress", + output: [] as unknown[], + } + const item = turn.call + ? { + type: "function_call", + id: `fc_${responseSeq}`, + call_id: turn.call.id + `_${responseSeq}`, + name: turn.call.name, + arguments: JSON.stringify(turn.call.args), + status: "completed", + } + : { + type: "message", + id: `msg_${responseSeq}`, + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: turn.text, annotations: [] }], + } + const usage = { + input_tokens: USAGE.prompt, + input_tokens_details: { cached_tokens: 0 }, + output_tokens: USAGE.completion, + output_tokens_details: { reasoning_tokens: 0 }, + total_tokens: USAGE.prompt + USAGE.completion, + } + const events: [string, unknown][] = [ + ["response.created", { type: "response.created", response: base }], + [ + "response.output_item.added", + { type: "response.output_item.added", output_index: 0, item: { ...item, status: "in_progress" } }, + ], + ] + if (!turn.call) + events.push([ + "response.output_text.delta", + { type: "response.output_text.delta", item_id: item.id, output_index: 0, content_index: 0, delta: turn.text }, + ]) + events.push( + ["response.output_item.done", { type: "response.output_item.done", output_index: 0, item }], + [ + "response.completed", + { type: "response.completed", response: { ...base, status: "completed", output: [item], usage } }, + ], + ) + return events.map(([event, data]) => `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`).join("") +} + +/** Privacy-safe evidence: retain only lengths, fixture coverage and hashes, never provider prompts. */ +export type FixtureEvidence = { + callId: string + bytes: number + codeUnits: number + sha256: string + expectedLines: number + matchedLines: number + firstMarker: boolean + lastMarker: boolean + complete: boolean + truncationNotice: boolean + errorNotice: boolean + nativeSuppressed: boolean + nativeGuardrailRefusal: boolean +} +export function fixtureLines(scenario: Scenario): string[] { + return scenario === "long" + ? Array.from( + { length: 800 }, + (_, i) => `line ${String(i + 1).padStart(4, "0")} lorem ipsum dolor sit amet consectetur adipiscing`, + ) + : Array.from({ length: 20 }, () => "alpha beta gamma") +} +function resultText(value: any): string { + if (typeof value === "string") return value + if (Array.isArray(value)) return value.map(resultText).join("\n") + if (value && typeof value === "object") { + if (typeof value.text === "string") return value.text + if (value.content !== undefined) return resultText(value.content) + } + return value === undefined ? "" : JSON.stringify(value) +} +export function fixtureEvidence(json: any, api: "chat" | "responses", scenario: Scenario): FixtureEvidence[] { + const items = + api === "chat" + ? (json.messages ?? []).filter((m: any) => m.role === "tool") + : (Array.isArray(json.input) ? json.input : []).filter((m: any) => m.type?.endsWith("_call_output")) + const expected = fixtureLines(scenario) + return items.map((m: any, index: number) => { + const content = resultText(api === "chat" ? m.content : m.output) + const flags = nativeResultFlags(content) + // Native tools can add line numbers/XML envelopes. Require every source line in order, + // including 20 occurrences of the repeated short fixture, without requiring identical wrappers. + let cursor = 0, + matchedLines = 0 + for (const line of expected) { + const at = content.indexOf(line, cursor) + if (at < 0) break + matchedLines++ + cursor = at + line.length + } + return { + callId: String(m.tool_call_id ?? m.call_id ?? index), + bytes: Buffer.byteLength(content, "utf8"), + codeUnits: content.length, + sha256: new Bun.CryptoHasher("sha256").update(content).digest("hex"), + expectedLines: expected.length, + matchedLines, + firstMarker: content.includes(expected[0]), + lastMarker: content.includes(expected.at(-1)!), + complete: matchedLines === expected.length, + truncationNotice: flags.truncated, + errorNotice: + flags.error || /(?:^|\n)(?:error:|traceback|permission denied|cat:.*(?:No such|denied))/i.test(content), + nativeSuppressed: flags.suppressed, + nativeGuardrailRefusal: flags.guardrail, + } + }) +} + +/** Native tools often JSON-encode their result inside OpenAI's string content. + * Respect explicit booleans: the field `truncated: false` is not a notice. + */ +function nativeResultFlags(content: string) { + const flags = { truncated: false, error: false, suppressed: false, guardrail: false } + const humanTruncation = (s: string) => + /(?:\btruncated\b|output.{0,30}(?:limit|cap)|(?:lines|characters|bytes) omitted)/i.test(s) + function walk(value: any) { + if (typeof value === "string") { + try { + walk(JSON.parse(value)) + } catch { + flags.truncated ||= humanTruncation(value) + } + return + } + if (Array.isArray(value)) { + value.forEach(walk) + return + } + if (!value || typeof value !== "object") return + for (const [key, item] of Object.entries(value)) { + if (/^truncat/i.test(key)) { + flags.truncated ||= + item !== false && item !== null && item !== 0 && item !== "" && !(Array.isArray(item) && !item.length) + continue + } + if (key === "guardrail_refusal" && item === true) { + flags.guardrail = true + flags.error = true + } + if ( + (key === "is_error" && item === true) || + (key === "error" && !!item) || + (key === "status" && ["error", "blocked", "refused", "failed"].includes(String(item))) + ) + flags.error = true + if ((key === "content_returned" && item === false) || (key === "dedup" && item === true)) flags.suppressed = true + walk(item) + } + } + walk(content) + return flags +} diff --git a/script/token-budget/release.ts b/script/token-budget/release.ts new file mode 100644 index 00000000..c0a0959f --- /dev/null +++ b/script/token-budget/release.ts @@ -0,0 +1,539 @@ +/** Explicit release binary comparison. No paid provider or image build; capture only. */ +import fs from "node:fs" +import path from "node:path" +import os from "node:os" +import { gzipSync } from "node:zlib" +import { MARKER, PROTOCOL_VERSION, Stub } from "./release-fixtures/token-stub" + +type Binary = { path: string; sha256: string; version: string; provenance: { path: string; sha256: string } } +type Inputs = { fixtureImage: string; current: Binary; previous: Binary; repeats: number } +const SHA = /^[a-f0-9]{64}$/ +const digest = (b: string | Uint8Array) => new Bun.CryptoHasher("sha256").update(b).digest("hex") +const [inputArg, outputArg] = process.argv.slice(2) +if (!inputArg || !outputArg || process.argv.length !== 4) + throw new Error("Usage: bun script/token-budget/release.ts INPUT.json NEW-OUTPUT-DIRECTORY") +const inputPath = path.resolve(inputArg) +const inputBytes = fs.readFileSync(inputPath) +const input = JSON.parse(inputBytes.toString()) as Inputs +if (!/^sha256:[a-f0-9]{64}$/.test(input.fixtureImage) || !Number.isInteger(input.repeats) || input.repeats < 2) + throw new Error("Pinned fixture image and repeats >=2 required") +for (const name of ["current", "previous"] as const) { + const bin = input[name] + if ( + !bin || + !SHA.test(bin.sha256) || + typeof bin.version !== "string" || + !bin.version || + !bin.provenance || + !SHA.test(bin.provenance.sha256) + ) + throw new Error(`Explicit ${name} binary/version/hash/provenance required`) + bin.path = path.resolve(path.dirname(inputPath), bin.path) + if ( + fs.lstatSync(bin.path).isSymbolicLink() || + !fs.statSync(bin.path).isFile() || + digest(fs.readFileSync(bin.path)) !== bin.sha256 + ) + throw new Error(`${name} binary hash/file mismatch`) + const bytes = fs.readFileSync(path.resolve(path.dirname(inputPath), bin.provenance.path)) + if (digest(bytes) !== bin.provenance.sha256) throw new Error(`${name} provenance hash mismatch`) + const provenance = JSON.parse(bytes.toString()) + const published = provenance.installedPackages?.platformPackages?.find( + (item: any) => item.binarySha256 === bin.sha256 && item.version === bin.version, + ) + if ( + name === "current" && + (provenance.complete !== true || provenance.binarySha256 !== bin.sha256 || provenance.version !== bin.version) + ) + throw new Error("Current build provenance must bind complete reviewed build/version/binary") + if ( + name === "previous" && + (!published || provenance.publishedPackage?.version !== bin.version || provenance.npmWrapperVersion !== bin.version) + ) + throw new Error("Previous provenance must bind actual published npm wrapper/platform binary/version") +} +if (input.current.sha256 === input.previous.sha256) + throw new Error( + "Current and previous binaries must have distinct SHA256 hashes; self-comparison is not a release comparison", + ) +const output = path.resolve(outputArg) +fs.mkdirSync(output, { recursive: false }) // refuse existing campaign, including failed runs +const logs = path.join(output, "campaign-logs") +fs.mkdirSync(logs, { mode: 0o700 }) +fs.writeFileSync(path.join(output, "original-inputs.json"), inputBytes, { mode: 0o600 }) +for (const name of ["current", "previous"] as const) { + const bytes = fs.readFileSync(path.resolve(path.dirname(inputPath), input[name].provenance.path)) + fs.writeFileSync(path.join(output, `${name}-provenance.json`), bytes, { mode: 0o600 }) + input[name].provenance.path = path.join(output, `${name}-provenance.json`) +} +fs.writeFileSync(path.join(output, "effective-inputs.json"), JSON.stringify(input, null, 2) + "\n", { mode: 0o600 }) +const hostHome = fs.mkdtempSync(path.join(os.tmpdir(), "release248-docker-")) +const hostEnv = { PATH: "/usr/local/bin:/usr/bin:/bin", HOME: hostHome } +const TIMEOUT = 180_000 +let ownedContainer: string | undefined +let proc: ReturnType | undefined +const campaignId = `release248-${process.pid}-${Date.now()}` +const failures: unknown[] = [] +const cleanupChecks: unknown[] = [] +const errors: unknown[] = [] +const cleanups: { name: typeof ownedContainer; child: typeof proc; promise: Promise }[] = [] +let interruption: { signal: "SIGINT" | "SIGTERM"; error: Error } | undefined +function recordFailure(error: unknown, field: "campaignError" | "cleanupError" = "campaignError") { + if (errors.includes(error)) return + errors.push(error) + failures.push({ [field]: error instanceof Error ? error.message : String(error) }) +} +function checkInterrupted() { + if (interruption) throw interruption.error +} +const arms: string[] = ["current-default", "previous-default"] +const support: Record = {} +const raw: Record> = { short: Object.fromEntries(arms.map((x) => [x, []])) } +const protocolSources = Object.fromEntries( + ["release-fixtures/token-stub.ts", "release.ts", "release-count.py", "reference.py"].map((name) => [ + name, + digest(fs.readFileSync(path.join(import.meta.dir, name))), + ]), +) +const metadata = { + purpose: "explicit current versus previous release reference-token capture; no performance comparison", + network: "host network; loopback scripted stub only is intended, but OS egress is not restricted", + model: "fixture/fixture (non-GPT), no real inference", + prompt: `${MARKER} short: read notes-0.txt, notes-1.txt and notes-2.txt once each in order, then reply Done.`, + permission: { "*": "allow" }, + selector: "--tools read,bash,edit,write, only if actual run help supports it", + inputManifestSha256: digest(inputBytes), + protocolSources, + limitations: [ + "Not provider billing/cache/quality evidence", + "Raw inert requests include repeated history and auxiliary calls", + "Timing is diagnostic only; never compare against historical campaigns", + ], +} +let stub: Stub | undefined +function checkpoint() { + fs.writeFileSync( + path.join(output, "campaign.json"), + JSON.stringify( + { + schemaVersion: 2, + protocolVersion: PROTOCOL_VERSION, + HARNESSES: arms, + REPS: input.repeats, + LONG_REPS: 0, + WARMUP: 0, + LATENCY: 50, + CPUS: "2", + MEMORY: "2g", + TIMEOUT_MS: TIMEOUT, + binaries: input, + support, + campaignMetadata: metadata, + raw, + failures, + cleanupChecks, + }, + null, + 2, + ) + "\n", + ) +} +async function docker(args: string[]) { + checkInterrupted() + const child = Bun.spawn(["docker", ...args], { env: hostEnv, stdin: "ignore", stdout: "pipe", stderr: "pipe" }) + proc = child + let timeout = false + const timer = setTimeout(() => { + timeout = true + child.kill("SIGKILL") + }, TIMEOUT) + try { + const [out, err, code] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]) + if (code || timeout) { + const error = new Error(`Docker preflight failed (${code}, timeout=${timeout}): ${err}`) + recordFailure(error) + await cleanup() + throw error + } + checkInterrupted() + return out + } finally { + clearTimeout(timer) + if (proc === child) proc = undefined + } +} +async function cleanupCommand(args: string[]) { + const child = Bun.spawn(["docker", ...args], { env: hostEnv, stdin: "ignore", stdout: "pipe", stderr: "pipe" }) + let timedOut = false + const timer = setTimeout(() => { + timedOut = true + child.kill("SIGKILL") + }, 15_000) + try { + const [stdout, stderr, exit] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]) + return { stdout, stderr, exit, timedOut } + } finally { + clearTimeout(timer) + } +} +function cleanup() { + const name = ownedContainer + const child = proc + const running = cleanups.find((item) => item.name === name && item.child === child) + if (running) return running.promise + const promise = (async () => { + const failed: unknown[] = [] + try { + if (child && child.exitCode === null) child.kill("SIGKILL") + } catch (error) { + failed.push(error) + } + if (name) { + let removed: Awaited> | undefined + let remaining: Awaited> | undefined + try { + removed = await cleanupCommand(["rm", "-f", name]) + } catch (error) { + failed.push(error) + } + try { + remaining = await cleanupCommand(["ps", "-aq", "--filter", `name=^/${name}$`]) + } catch (error) { + failed.push(error) + } + cleanupChecks.push({ name, removed, remaining, errors: failed.map(String) }) + if ( + !removed || + !remaining || + removed.timedOut || + remaining.timedOut || + remaining.exit !== 0 || + remaining.stdout.trim() || + (removed.exit !== 0 && !removed.stderr.includes("No such container")) + ) + failed.push(new Error(`Unverified container cleanup: ${name}`)) + } + try { + if (child) await child.exited + } catch (error) { + failed.push(error) + } + if (failed.length) { + const error = new AggregateError(failed, failed.map(String).join("; "), { cause: failed[0] }) + recordFailure(error, "cleanupError") + throw error + } + })().finally(() => { + const index = cleanups.findIndex((item) => item.promise === promise) + cleanups.splice(index, 1) + }) + cleanups.push({ name, child, promise }) + return promise +} +const handlers = (["SIGINT", "SIGTERM"] as const).map((signal) => { + const handler = () => { + if (interruption) return + interruption = { signal, error: new Error(`Campaign interrupted by ${signal}`) } + recordFailure(interruption.error) + void cleanup().catch((error) => recordFailure(error, "cleanupError")) + } + process.on(signal, handler) + return { signal, handler } +}) +try { + const image = JSON.parse(await docker(["image", "inspect", input.fixtureImage]))[0] + if ( + image.Id !== input.fixtureImage || + image.Architecture !== "amd64" || + image.Os !== "linux" || + image.Config.Env.some((entry: string) => !/^(PATH|NODE_VERSION|YARN_VERSION|HOME)=/.test(entry)) + ) + throw new Error("Fixture image identity/platform mismatch") + for (const name of ["current", "previous"] as const) { + checkInterrupted() + const binary = input[name] + const script = `const fs=require('fs'),cp=require('child_process'),crypto=require('crypto'); const binary='/release/aictrl'; console.log(JSON.stringify({uid:process.getuid(),home:fs.readdirSync(process.env.HOME),binarySha256:crypto.createHash('sha256').update(fs.readFileSync(binary)).digest('hex'),version:cp.execFileSync(binary,['--version'],{encoding:'utf8'}).trim(),help:cp.execFileSync(binary,['run','--help'],{encoding:'utf8'}),fixtures:Object.fromEntries(['notes.txt','big.txt',...Array.from({length:3},(_,i)=>'notes-'+i+'.txt'),...Array.from({length:50},(_,i)=>'big-'+i+'.txt')].map(name=>{const b=fs.readFileSync('/workspace/'+name);return[name,{bytes:b.length,sha256:crypto.createHash('sha256').update(b).digest('hex')}]}))}));` + ownedContainer = `${campaignId}-preflight-${name}` + const preflight = JSON.parse( + await docker([ + "run", + "--rm", + "--pull=never", + "--name", + ownedContainer, + "--label", + `aictrl.harness-benchmark=${campaignId}`, + "--network", + "none", + "--user", + "10001:10001", + "--mount", + `type=bind,src=${binary.path},dst=/release/aictrl,readonly`, + "--entrypoint", + "node", + input.fixtureImage, + "-e", + script, + ]), + ) + await cleanup() + checkInterrupted() + ownedContainer = undefined + fs.writeFileSync(path.join(output, `${name}-preflight.json`), JSON.stringify(preflight, null, 2) + "\n") + if ( + preflight.uid !== 10001 || + preflight.home.length || + preflight.binarySha256 !== binary.sha256 || + preflight.version !== binary.version + ) + throw new Error(`${name} binary/UID/HOME/version mismatch`) + for (const [filename, item] of Object.entries(preflight.fixtures) as [ + string, + { bytes: number; sha256: string }, + ][]) { + const short = filename.startsWith("notes") + if ( + item.bytes !== (short ? 340 : 48000) || + item.sha256 !== + (short + ? "268a0ba51362b6181510ff8e8cea2804328d526cb07a97ce8fe56d995f5c64a0" + : "1c756a4da138e2789f27bccbd48c08dcf99bcbf51bd27f7a4442b13b28b912db") + ) + throw new Error(`Fixture mismatch:${filename}`) + } + support[name] = /^\s*--tools\s/m.test(preflight.help) + if (support[name]) { + arms.push(`${name}-selected`) + raw.short[`${name}-selected`] = [] + } + } + checkInterrupted() + stub = new Stub(50) + stub.keepBodies = true + for (let rep = 0; rep < input.repeats; rep++) { + for (const arm of rep % 2 ? [...arms].reverse() : arms) { + checkInterrupted() + stub.reset("short") + const runId = `${String(rep + 1).padStart(2, "0")}-${arm}-short` + ownedContainer = `${campaignId}-${runId}` + const config = { + provider: { + fixture: { + npm: "@ai-sdk/openai-compatible", + options: { apiKey: "fixture", baseURL: `http://127.0.0.1:${stub.port}/v1` }, + models: { fixture: { name: "fixture", tool_call: true, limit: { context: 200_000, output: 1000 } } }, + }, + }, + permission: metadata.permission, + } + const binary = input[arm.startsWith("current-") ? "current" : "previous"] + const cliArgv = [ + "/release/aictrl", + "run", + "--format", + "json", + "--model", + "fixture/fixture", + ...(arm.endsWith("-selected") ? ["--tools", "read,bash,edit,write"] : []), + metadata.prompt, + ] + const argv = [ + "docker", + "run", + "--rm", + "--pull=never", + "--name", + ownedContainer, + "--label", + `aictrl.harness-benchmark=${campaignId}`, + "--network", + "host", + "--user", + "10001:10001", + "--cpus", + "2", + "--memory", + "2g", + "--memory-swap", + "2g", + "-e", + `AICTRL_CONFIG_CONTENT=${JSON.stringify(config)}`, + "--mount", + `type=bind,src=${binary.path},dst=/release/aictrl,readonly`, + "--entrypoint", + "/release/aictrl", + input.fixtureImage, + ...cliArgv.slice(1), + ] + fs.writeFileSync( + path.join(logs, `${runId}.argv.json`), + JSON.stringify({ argv, cliArgv, config, inputManifestSha256: digest(inputBytes) }, null, 2) + "\n", + ) + let timeout = false + let cleanupFailure: string | undefined + checkInterrupted() + const child = Bun.spawn(argv, { env: hostEnv, stdin: "ignore", stdout: "pipe", stderr: "pipe" }) + proc = child + const timer = setTimeout(() => { + timeout = true + recordFailure(new Error(`Capture ${runId} timed out`)) + void cleanup().catch((error) => { + cleanupFailure ??= String(error) + }) + }, TIMEOUT) + let stdout = "", + stderr = "", + code: number | null = null + try { + ;[stdout, stderr, code] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]) + } finally { + clearTimeout(timer) + try { + await cleanup() + } catch (error) { + cleanupFailure ??= String(error) + } + if (!cleanupFailure) ownedContainer = undefined + proc = undefined + } + const requests = [...stub.log] + fs.writeFileSync(path.join(logs, `${runId}.stdout.jsonl`), stdout) + fs.writeFileSync(path.join(logs, `${runId}.stderr.log`), stderr) + fs.writeFileSync( + path.join(logs, `${runId}.requests-with-bodies.jsonl.gz`), + gzipSync( + requests + .map((q) => JSON.stringify({ ...q, bodySha256: q.body === undefined ? null : digest(q.body) })) + .join("\n") + "\n", + ), + { mode: 0o600 }, + ) + fs.writeFileSync( + path.join(logs, `${runId}.requests.jsonl`), + requests + .map(({ body, ...q }) => JSON.stringify({ ...q, bodySha256: body === undefined ? null : digest(body) })) + .join("\n") + "\n", + ) + const errors: string[] = [] + if (cleanupFailure) errors.push(cleanupFailure) + const events: any[] = [] + for (const line of stdout.split("\n").filter(Boolean)) { + try { + events.push(JSON.parse(line)) + } catch { + errors.push("Invalid native NDJSON") + } + } + const main = requests.filter((q) => q.kind === "main") + const tools = main.map((q) => JSON.parse(q.body!).tools.map((t: any) => t.function?.name ?? t.name)) as string[][] + for (const names of tools) { + if (names.some((n) => typeof n !== "string") || new Set(names).size !== names.length) + errors.push("Malformed/duplicate schema names") + if (arm.endsWith("-selected") && [...names].sort().join(",") !== "bash,edit,read,write") + errors.push("Selected schema does not contain exactly four work tools") + if (arm.endsWith("-default") && !names.includes("read")) errors.push("Default catalog lacks scripted Read tool") + } + if (main.length !== 4 || main.map((q) => q.step).join(",") !== "0,1,2,3") + errors.push("Expected four main requests / scripted steps0..3") + for (let i = 0; i < 3; i++) { + const call = main[i]?.response + const evidence = main[i + 1]?.toolResults?.find((q) => q.callId === `call_${i}`) + if ( + call?.kind !== "tool" || + call.name !== "read" || + call.arguments?.filePath !== `/workspace/notes-${i}.txt` || + !evidence?.complete || + evidence.matchedLines !== 20 || + evidence.errorNotice || + evidence.truncationNotice || + evidence.nativeSuppressed || + evidence.nativeGuardrailRefusal + ) + errors.push(`Read${i} not exactly validated`) + } + if ( + main.at(-1)?.toolResults?.length !== 3 || + main.at(-1)?.response?.text !== "Done." || + !events.some((e) => e.type === "text" && /^Done\.?$/.test((e.text ?? e.part?.text ?? "").trim())) + ) + errors.push("Missing final three reads / native Done") + if (requests.some((q) => q.kind === "other")) errors.push("Unexpected provider endpoint request") + if ( + events.some((e) => e.type === "error" || e.is_error || e.type === "invocation_failed") || + requests.some((q) => q.error) + ) + errors.push("Native/stub error") + if (code !== 0 || timeout) errors.push(`Exit${code}, timeout=${timeout}`) + const sample = { + runId, + valid: errors.length === 0, + mainRequests: main.length, + auxRequests: requests.filter((q) => q.kind === "aux").length, + discoveryRequests: requests.filter((q) => q.kind === "discovery").length, + otherRequests: requests.filter((q) => q.kind === "other").map((q) => q.path), + toolNames: tools, + exit: code, + timedOut: timeout, + errors, + } + raw.short[arm].push(sample) + if (errors.length) failures.push({ arm, rep, ...sample }) + checkpoint() + if (errors.length) throw new Error(`Capture ${runId} failed: ${errors.join("; ")}`) + } + } +} catch (error) { + recordFailure(error) +} finally { + try { + await cleanup() + } catch (error) { + recordFailure(error, "cleanupError") + } + try { + const remaining = await cleanupCommand(["ps", "-aq", "--filter", `label=aictrl.harness-benchmark=${campaignId}`]) + cleanupChecks.push({ finalCampaignCheck: remaining }) + if (remaining.timedOut || remaining.exit !== 0 || remaining.stdout.trim()) + throw new Error("Campaign-owned containers remain or cleanup state could not be inspected") + } catch (error) { + recordFailure(error, "cleanupError") + } + try { + await stub?.server.stop(true) + } catch (error) { + recordFailure(error, "cleanupError") + } + await Promise.allSettled(cleanups.map((item) => item.promise)) + try { + fs.rmSync(hostHome, { recursive: true, force: true }) + } catch (error) { + recordFailure(error, "cleanupError") + } + try { + checkpoint() + } catch (error) { + recordFailure(error) + } + for (const { signal, handler } of handlers) process.off(signal, handler) +} +if (errors.length) { + const error = new AggregateError(errors, errors.map(String).join("; "), { cause: errors[0] }) + if (!interruption) throw error + console.error(error) + process.exit(128 + (interruption.signal === "SIGINT" ? 2 : 15)) +} +console.log( + `Captured ${input.repeats} sessions per supported arm; containers verified absent in ${output}; run release-count.py next`, +) diff --git a/script/token-budget/requirements.lock b/script/token-budget/requirements.lock new file mode 100644 index 00000000..2ed1de35 --- /dev/null +++ b/script/token-budget/requirements.lock @@ -0,0 +1,8 @@ +# Python 3.12 Linux x86_64; wheels only, official PyPI. Tokenizer stays fixed; support libraries use patched official releases. +tiktoken==0.12.0 --hash=sha256:edde1ec917dfd21c1f2f8046b86348b0f54a2c0547f68149d8600859598769ad +regex==2026.1.15 --hash=sha256:c32bef3e7aeee75746748643667668ef941d28b003bfc89994ecf09a10f7a1b5 +requests==2.34.2 --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 +charset-normalizer==3.4.4 --hash=sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86 +idna==3.20 --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c +urllib3==2.8.0 --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 +certifi==2026.10.19 --hash=sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b diff --git a/script/token-budget/run.py b/script/token-budget/run.py new file mode 100644 index 00000000..8f424768 --- /dev/null +++ b/script/token-budget/run.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +"""Linux-only public CLI SDK capture; no downloads or model traffic.""" +import argparse +import ctypes +import errno +import fcntl +import json +import os +from pathlib import Path +import shutil +import signal +import stat +import subprocess +import sys +import time +import traceback + +ROOT = Path(__file__).resolve().parent +FIXTURE = Path('/tmp/aictrl-token-budget') +WORKSPACE = FIXTURE / 'workspace' + + +def _reap_descendants(limit=2): + # Called only in a fresh forked subreaper: it has no caller-owned children. + # waitid(WNOWAIT) proves each PID is our unreaped direct child. Until we + # explicitly waitpid it, even an exiting child cannot release/reuse its PID. + end = time.monotonic() + limit + surviving = False + records = [] + while True: + children = Path(f'/proc/self/task/{os.getpid()}/children').read_text().split() + if not children: + try: + os.waitpid(-1, os.WNOHANG) + except ChildProcessError: + return surviving, records + raise RuntimeError('Owned child inventory incomplete; refusing unproven cleanup') + for value in children: + pid = int(value) + state = os.waitid(os.P_PID, pid, os.WEXITED | os.WNOHANG | os.WNOWAIT) + if state is not None: + os.waitpid(pid, 0) + records.append({'pid': pid, 'state': 'reaped', 'owner': os.getpid()}) + continue + surviving = True + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + # Still unreaped: this identity cannot have become a foreign PID. + pass + records.append({'pid': pid, 'state': 'live-killed', 'owner': os.getpid()}) + if time.monotonic() >= end: + raise RuntimeError('Owned descendants remain after cleanup deadline: ' + ','.join(children)) + time.sleep(0.005) + + +def _capture(command, cwd, env, limit, cancelled): + child = None + primary = None + timed = False + surviving = False + records = [] + stdout = stderr = '' + try: + child = subprocess.Popen(command, cwd=cwd, env=env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) + end = time.monotonic() + limit + while child.poll() is None: + if cancelled(): + raise InterruptedError('Capture supervisor interrupted') + remaining = end - time.monotonic() + if remaining <= 0: + timed = True + break + try: + stdout, stderr = child.communicate(timeout=min(0.05, remaining)) + except subprocess.TimeoutExpired: + # Recheck the actual parent, even while a descendant holds a pipe. + pass + except BaseException as error: + primary = error + raise + finally: + errors = [] + if child is not None and child.poll() is None: + try: + child.kill() + except ProcessLookupError: + pass + except Exception as error: + errors.append(error) + try: + child.wait(timeout=5) + except Exception as error: + errors.append(error) + try: + surviving, records = _reap_descendants() + except Exception as error: + errors.append(error) + if child is not None: + try: + stdout, stderr = child.communicate(timeout=5) + except Exception as error: + errors.append(error) + finally: + for stream in (child.stdout, child.stderr): + if stream is not None: + try: + stream.close() + except Exception as error: + errors.append(error) + if errors: + detail = 'Owned capture cleanup failed: ' + '; '.join(str(error) for error in errors) + if primary is not None: + primary.add_note(detail) + else: + raise RuntimeError(detail) from errors[0] + return {'exit': child.returncode, 'timedOut': timed, 'remaining_group_killed': surviving, + 'process_group': child.pid, 'supervisor': os.getpid(), + 'descendant_cleanup': records, 'command': command}, stdout, stderr + + +def execute(command, cwd, env, limit=25): + # A separate fork is an ownership boundary, not a network/security sandbox. + # Its only children are the capture and descendants adopted from that capture. + read, write = os.pipe() + try: + supervisor = os.fork() + except BaseException: + os.close(read) + os.close(write) + raise + if supervisor == 0: + status = 1 + try: + os.close(read) + interrupted = False + + def cancel(number, frame): + nonlocal interrupted + interrupted = True + + try: + # Do not retain the caller's unrelated pipes/files in this fork. + for value in os.listdir('/proc/self/fd'): + descriptor = int(value) + if descriptor > 2 and descriptor != write: + try: + os.close(descriptor) + except OSError as error: + if error.errno != errno.EBADF: + raise + signal.signal(signal.SIGTERM, cancel) + signal.signal(signal.SIGINT, cancel) + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), 'Unable to establish owned Linux child subreaper') + result, stdout, stderr = _capture(command, cwd, env, limit, lambda: interrupted) + payload = {'result': result, 'stdout': stdout, 'stderr': stderr} + except BaseException as error: + payload = {'error': {'type': type(error).__name__, 'message': str(error), + 'errno': getattr(error, 'errno', None), 'strerror': getattr(error, 'strerror', None), + 'filename': getattr(error, 'filename', None), + 'traceback': ''.join(traceback.format_exception(error))}} + with os.fdopen(write, 'w') as stream: + json.dump(payload, stream) + status = 0 if 'result' in payload else 1 + finally: + os._exit(status) + os.close(write) + with os.fdopen(read) as stream: + primary = None + try: + # Drain before waitpid: stdout/stderr can exceed the result pipe buffer. + payload = json.load(stream) + except BaseException as error: + primary = error + try: + os.kill(supervisor, signal.SIGTERM) + except ProcessLookupError: + pass + except Exception as cleanup_error: + primary.add_note('Supervisor cancellation failed: ' + str(cleanup_error)) + try: + # Finish owned cleanup and drain its reply before waiting. + stream.read() + except Exception as cleanup_error: + primary.add_note('Supervisor reply drain failed: ' + str(cleanup_error)) + raise + finally: + try: + _, status = os.waitpid(supervisor, 0) + except Exception as cleanup_error: + if primary is None: + raise + primary.add_note('Supervisor reap failed: ' + str(cleanup_error)) + if 'error' in payload: + error = payload['error'] + if error['errno'] is not None: + failure = OSError(error['errno'], error['strerror'], error['filename']) + else: + failure = RuntimeError('Capture supervisor failed: ' + error['type'] + ': ' + error['message']) + failure.add_note(error['traceback']) + raise failure + if not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0: + raise RuntimeError('Capture supervisor exited without successful cleanup: ' + str(status)) + return payload['result'], payload['stdout'], payload['stderr'] + + +def verify(result): + if result['remaining_group_killed']: + raise RuntimeError('Owned descendant survived controller exit; capture rejected after cleanup') + if result['exit'] or result['timedOut']: + raise RuntimeError('Capture failed or exceeded deadline: ' + json.dumps(result)) + + +def private_directory(path, reuse=False): + try: + path.mkdir(mode=0o700) + except FileExistsError: + if not reuse: + raise + info = path.lstat() + if (not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077 + or (not reuse and stat.S_IMODE(info.st_mode) != 0o700)): + raise RuntimeError('Fixture/evidence directory must be an owned private directory, never a symlink') + return path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', type=Path, required=True, help='New evidence directory; never overwritten') + parser.add_argument('--repeats', type=int, default=2) + parser.add_argument('--source', type=Path, default=ROOT.parents[1] / 'packages/cli/src/index.ts', + help='Explicit alternate source entry for isolated source-growth controls; CI uses repository entry') + parser.add_argument('--observe', action='store_true', help='Report observations without enforcing budgets (evaluation only)') + args = parser.parse_args() + if sys.platform != 'linux' or sys.version_info[:2] != (3, 12): + raise RuntimeError('Require Linux and provisioned Python 3.12') + if args.repeats < 2: + raise ValueError('At least two cold captures per arm required') + bun = shutil.which('bun') + if not bun or not shutil.which('rg'): + raise RuntimeError('Require real bun and ripgrep on PATH; install prerequisites before offline capture') + private_directory(FIXTURE, reuse=True) + fd = os.open(FIXTURE / 'lock', os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, 'a') as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise RuntimeError('Exclusive fixed workspace is already in use') + if args.output.is_symlink(): + raise ValueError('Evidence output must be a new directory, never a symlink') + output = args.output.resolve() + if output.is_relative_to(FIXTURE): + raise ValueError('Evidence output must be outside the fixed fixture directory') + private_directory(output) + records = [] + for arm in ['default', 'four']: + for repeat in range(1, args.repeats + 1): + out = output / f'{arm}-{repeat}' + private_directory(out) + if WORKSPACE.is_symlink(): + raise RuntimeError('Refusing a symlink fixed workspace') + if WORKSPACE.exists(): + shutil.rmtree(WORKSPACE) + WORKSPACE.mkdir() + env = {'PATH': os.pathsep.join([str(Path(bun).parent), '/usr/local/bin', '/usr/bin', '/bin']), + 'HOME': str(out / 'controller-home'), 'XDG_CACHE_HOME': str(out / 'controller-cache'), + 'BUN_RUNTIME_TRANSPILER_CACHE_PATH': str(out / 'controller-bun-cache'), + 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null', 'TZ': 'UTC', + 'TOKEN_BUDGET_OUTPUT': str(out), 'TOKEN_BUDGET_WORKSPACE': str(WORKSPACE), + 'TOKEN_BUDGET_SOURCE': str(args.source.resolve(strict=True)), + 'TOKEN_BUDGET_ARM': arm} + subprocess.run(['git', '-c', 'init.defaultBranch=fixture', 'init', '--quiet', str(WORKSPACE)], + env=env, check=True, stdin=subprocess.DEVNULL) + result, stdout, stderr = execute([bun, 'run', '--no-install', '--no-env-file', str(ROOT / 'capture.ts')], ROOT, env) + result.update(arm=arm, repeat=repeat) + (out / 'controller.log').write_text(stdout + stderr) + (out / 'controller-result.json').write_text(json.dumps(result, indent=2) + '\n') + verify(result) + records.append(result) + (output / 'campaign.json').write_text(json.dumps({'runs': records, 'workspace': str(WORKSPACE), + 'raw_normalization': False, 'repeats': args.repeats, 'source_entry': str(args.source.resolve())}, indent=2) + '\n') + subprocess.run([sys.executable, '-I', '-B', str(ROOT / 'count.py'), str(output), + *(['--observe'] if args.observe else [])], check=True, + env={'PATH': env['PATH'], 'HOME': str(output / 'counter-home'), 'TZ': 'UTC'}) + + +if __name__ == '__main__': + main() diff --git a/script/token-budget/setup.py b/script/token-budget/setup.py new file mode 100644 index 00000000..112113f4 --- /dev/null +++ b/script/token-budget/setup.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Online SETUP ONLY: owned Python environment and checksum-verified rank data.""" +import argparse +import hashlib +import json +from pathlib import Path +import platform +import subprocess +import sys +import urllib.request +import venv + +ROOT = Path(__file__).resolve().parent +URL = 'https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken' +SHA = '446a9538cb6c348e3516120d7c08b09f57c36495e2acfffe59a5bf8b0cfb1a2d' +KEY = hashlib.sha1(URL.encode()).hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--directory', type=Path, required=True, help='New owned environment directory') + args = parser.parse_args() + if sys.platform != 'linux' or platform.machine() != 'x86_64' or sys.version_info[:2] != (3, 12): + raise RuntimeError('Lock targets Python 3.12 Linux x86_64') + directory = args.directory.resolve() + directory.mkdir(parents=True, exist_ok=False) + venv.create(directory / 'venv', with_pip=True) + python = directory / 'venv/bin/python3' + subprocess.run([str(python), '-I', '-m', 'pip', '--isolated', 'install', '--disable-pip-version-check', + '--cache-dir', str(directory / 'wheel-cache'), '--index-url', 'https://pypi.org/simple', '--only-binary=:all:', '--require-hashes', + '--no-deps', '-r', str(ROOT / 'requirements.lock')], check=True) + with urllib.request.urlopen(URL, timeout=30) as response: + data = response.read() + if hashlib.sha256(data).hexdigest() != SHA: + raise RuntimeError('Pinned o200k rank SHA256 mismatch') + cache = directory / 'cache' + cache.mkdir() + (cache / KEY).write_bytes(data) + (directory / 'setup.json').write_text(json.dumps({'python': sys.version, 'rank_url': URL, + 'rank_sha256': SHA, 'rank_bytes': len(data), + 'requirements_sha256': hashlib.sha256((ROOT / 'requirements.lock').read_bytes()).hexdigest()}, indent=2) + '\n') + print(f'Offline command: {python} -I -B {ROOT / "run.py"} --output NEW_DIRECTORY') + + +if __name__ == '__main__': + main() diff --git a/script/token-budget/test_accounting.py b/script/token-budget/test_accounting.py new file mode 100644 index 00000000..bbac172d --- /dev/null +++ b/script/token-budget/test_accounting.py @@ -0,0 +1,136 @@ +"""Offline accounting faults using copied real captures and private environment copies.""" +import contextlib +import importlib.util +import io +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parent +sys.path.insert(0, str(ROOT)) +import reference +import count + +class Reference(unittest.TestCase): + def test_public_canonical_protocol_golden(self): + raw = json.dumps({'messages': [{'role': 'user', 'content': 'héllo'}], 'model': 'transport-only', + 'temperature': .1, 'tools': []}, ensure_ascii=False) + self.assertEqual(reference.input_text(raw), '{"messages":[{"content":"héllo","role":"user"}],"tools":[]}') + self.assertEqual(reference.output_text({'kind': 'tool', 'name': 'read', 'arguments': {'z': 1, 'a': 'é'}}), + '{"arguments":{"a":"é","z":1},"name":"read"}') + self.assertEqual(reference.output_text({'kind': 'text', 'text': 'Done'}), 'Done') + with self.assertRaises(ValueError): + reference.input_text('{}') + + def test_pinned_known_string_roundtrip(self): + encoding = reference.load_encoding() + text = 'Hello, world! aictrl offline tokenizer QA.' + encoded = encoding.encode_ordinary(text) + self.assertEqual(len(encoded), 11) + self.assertEqual(encoding.decode(encoded), text) + + def test_actual_copied_environment_rejects_missing_wrong_rank_and_wrong_version(self): + environment = Path(sys.prefix).parent + self.assertTrue((environment / 'setup.json').is_file(), 'Run with the explicit setup-created venv') + script = 'import sys;sys.path.insert(0,sys.argv[1]);import reference;reference.load_encoding()' + for mode, diagnostic in [('missing', 'Missing or mismatched'), ('wrong', 'Missing or mismatched'), + ('version', 'Require pinned tiktoken')]: + with self.subTest(mode=mode), tempfile.TemporaryDirectory(prefix='budget-environment-') as directory: + clone = Path(directory) / 'environment' + shutil.copytree(environment, clone, symlinks=True, copy_function=shutil.copy2, + ignore=shutil.ignore_patterns('wheel-cache')) + rank = clone / 'cache' / reference.RANK_KEY + if mode == 'missing': + rank.unlink() + elif mode == 'wrong': + rank.write_bytes(b'wrong-rank-data') + else: + metadata = next((clone / 'venv/lib/python3.12/site-packages').glob('tiktoken-*.dist-info/METADATA')) + text = metadata.read_text() + self.assertIn('Version: 0.12.0', text) + metadata.write_text(text.replace('Version: 0.12.0', 'Version: 0.11.0', 1)) + p = subprocess.run([str(clone / 'venv/bin/python3'), '-I', '-B', '-c', script, str(ROOT)], + env={'PATH': '/usr/bin:/bin', 'HOME': str(clone / 'home')}, + stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=10) + self.assertNotEqual(p.returncode, 0) + self.assertIn(diagnostic, p.stderr) + +@unittest.skipUnless(os.environ.get('TOKEN_BUDGET_TEST_CAMPAIGN'), 'Set TOKEN_BUDGET_TEST_CAMPAIGN to actual completed capture') +class Captures(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix='budget-capture-fault-') + self.addCleanup(self.temp.cleanup) + self.directory = Path(self.temp.name) + self.original = Path(os.environ['TOKEN_BUDGET_TEST_CAMPAIGN']) + shutil.copy2(self.original / 'campaign.json', self.directory / 'campaign.json') + for source in self.original.glob('*-*'): + if not source.is_dir(): + continue + target = self.directory / source.name + target.mkdir() + for name in ['controller-result.json', 'result.json', 'network.jsonl', 'request-0.json']: + shutil.copy2(source / name, target / name) + self.case = self.directory / 'default-1' + + def analyze(self): + with contextlib.redirect_stdout(io.StringIO()): + return count.analyze(self.directory, observe=True) + + def body(self, transform): + file = self.case / 'request-0.json' + body = json.loads(file.read_text()) + transform(body) + file.write_text(json.dumps(body, ensure_ascii=False)) + + def test_real_captures_reproduce_and_user_text_is_outside_prefix(self): + first = self.analyze() + for file in self.directory.glob('*-*/request-0.json'): + body = json.loads(file.read_text()) + next(m for m in body['messages'] if m['role'] == 'user')['content'] = 'A different explicit user fixture ' * 100 + file.write_text(json.dumps(body)) + second = self.analyze() + self.assertEqual([r['prefix_sha256'] for r in first['rows']], [r['prefix_sha256'] for r in second['rows']]) + self.assertNotEqual([r['full_first_model_input_reference_tokens'] for r in first['rows']], + [r['full_first_model_input_reference_tokens'] for r in second['rows']]) + + def test_wrong_catalog_rejected(self): + self.body(lambda b: b['tools'][0]['function'].update(name='unknown-fixture-work')) + with self.assertRaisesRegex(ValueError, 'Wrong actual tool catalog'): + self.analyze() + + def test_raw_clock_or_cwd_drift_rejected(self): + self.body(lambda b: b['messages'][0].update(content=b['messages'][0]['content'].replace('Wed Oct 21 2026', 'Thu Oct 22 2026'))) + with self.assertRaisesRegex(ValueError, 'Uncontrolled raw environment'): + self.analyze() + + def test_nonidentical_tool_description_repetitions_rejected(self): + self.body(lambda b: b['tools'][0]['function'].update(description=b['tools'][0]['function']['description'] + ' Drift')) + with self.assertRaisesRegex(ValueError, 'differs between cold repetitions'): + self.analyze() + + def test_unaccounted_auxiliary_network_and_request_rejected(self): + file = self.case / 'network.jsonl' + with file.open('a') as stream: + stream.write(json.dumps({'origin': 'http://127.0.0.1:1', 'path': '/v1/models', 'allowed': True}) + '\n') + with self.assertRaisesRegex(ValueError, 'Unaccounted network'): + self.analyze() + + def test_deadline_record_cannot_be_a_negative_success(self): + file = self.case / 'result.json' + data = json.loads(file.read_text()); data['deadline'] = True; file.write_text(json.dumps(data)) + with self.assertRaisesRegex(ValueError, 'Invalid capture'): + self.analyze() + + def test_surviving_controller_group_rejected(self): + file = self.case / 'controller-result.json' + data = json.loads(file.read_text()); data['remaining_group_killed'] = True; file.write_text(json.dumps(data)) + with self.assertRaisesRegex(ValueError, 'Unsuccessful controller'): + self.analyze() + +if __name__ == '__main__': + unittest.main() diff --git a/script/token-budget/test_protocol.py b/script/token-budget/test_protocol.py new file mode 100644 index 00000000..e7de55e8 --- /dev/null +++ b/script/token-budget/test_protocol.py @@ -0,0 +1,260 @@ +"""Actual child/guard controls for the owned token-budget runner; no provider calls.""" +import ctypes +import fcntl +import importlib.util +import json +import os +from pathlib import Path +import shutil +import signal +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parent +BUN = shutil.which('bun') + +def module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + value = importlib.util.module_from_spec(spec) + spec.loader.exec_module(value) + return value + +RUN = module('budget_run', ROOT / 'run.py') + +class Protocol(unittest.TestCase): + @classmethod + def setUpClass(cls): + if ctypes.CDLL(None).prctl(36, 1, 0, 0, 0) != 0: + raise RuntimeError('Linux subreaper required for owned child controls') + + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix='token-budget-test-') + self.addCleanup(self.temp.cleanup) + self.directory = Path(self.temp.name) + self.env = {'PATH': os.pathsep.join([str(Path(BUN).parent), '/usr/bin', '/bin']), + 'HOME': str(self.directory / 'home'), 'TZ': 'UTC'} + + def test_clean_process_is_accepted(self): + result, stdout, stderr = RUN.execute([sys.executable, '-I', '-c', 'print("clean")'], self.directory, self.env, 2) + RUN.verify(result) + self.assertEqual(stdout.strip(), 'clean') + self.assertEqual(stderr, '') + self.assertFalse(result['remaining_group_killed']) + + def test_actual_deadline_is_never_a_negative_success(self): + result, _, _ = RUN.execute([sys.executable, '-I', '-c', 'import time;time.sleep(10)'], self.directory, self.env, .05) + self.assertTrue(result['timedOut']) + with self.assertRaisesRegex(RuntimeError, 'failed|deadline'): + RUN.verify(result) + with self.assertRaises(ProcessLookupError): + os.killpg(result['process_group'], 0) + + def test_surviving_group_rejected_even_after_parent_exit_zero(self): + source = ('import os,time\npid=os.fork()\nif pid==0:\n os.close(1);os.close(2);time.sleep(10);os._exit(0)\nprint(pid,flush=True)') + result, _, _ = RUN.execute([sys.executable, '-I', '-c', source], self.directory, self.env, 2) + self.assertEqual(result['exit'], 0) + self.assertTrue(result['remaining_group_killed']) + with self.assertRaisesRegex(RuntimeError, 'survived controller exit'): + RUN.verify(result) + with self.assertRaises(ProcessLookupError): + os.killpg(result['process_group'], 0) + + def test_setsid_child_with_inherited_output_is_owned_rejected_and_reaped(self): + source = """import os,time +read,write=os.pipe() +pid=os.fork() +if pid==0: + os.close(read);os.setsid();os.write(write,b'ready');os.close(write);time.sleep(10);os._exit(0) +os.close(write);os.read(read,5);os.close(read);print(pid,flush=True) +""" + result, stdout, _ = RUN.execute([sys.executable, '-I', '-c', source], self.directory, self.env, 2) + self.assertEqual(result['exit'], 0) + self.assertFalse(result['timedOut']) + self.assertTrue(result['remaining_group_killed']) + self.assertNotEqual(result['supervisor'], os.getpid()) + pid = int(stdout.strip()) + self.assertTrue(any(row['pid'] == pid and row['state'] == 'live-killed' + and row['owner'] == result['supervisor'] for row in result['descendant_cleanup'])) + with self.assertRaisesRegex(RuntimeError, 'survived controller exit'): + RUN.verify(result) + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + + def test_setsid_child_and_grandchild_are_repeatedly_adopted_and_reaped(self): + source = """import os,time,json +read,write=os.pipe() +pid=os.fork() +if pid==0: + os.close(read);os.setsid() + grand=os.fork() + if grand==0: + os.setsid();os.write(write,(str(os.getpid())+'\\n').encode());os.close(write);time.sleep(10);os._exit(0) + os.close(write);time.sleep(10);os._exit(0) +os.close(write) +with os.fdopen(read) as stream: grand=int(stream.readline()) +print(json.dumps([pid,grand]),flush=True) +""" + result, stdout, _ = RUN.execute([sys.executable, '-I', '-c', source], self.directory, self.env, 2) + self.assertFalse(result['timedOut']) + self.assertTrue(result['remaining_group_killed']) + pids = json.loads(stdout) + killed = {row['pid'] for row in result['descendant_cleanup'] if row['state'] == 'live-killed'} + self.assertTrue(set(pids) <= killed) + for pid in pids: + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + + def test_zombie_only_adoptee_does_not_reject_a_clean_capture(self): + source = """import os +pid=os.fork() +if pid==0: os._exit(0) +os.waitid(os.P_PID,pid,os.WEXITED|os.WNOWAIT) +print(pid,flush=True) +""" + result, stdout, _ = RUN.execute([sys.executable, '-I', '-c', source], self.directory, self.env, 2) + RUN.verify(result) + self.assertFalse(result['remaining_group_killed']) + pid = int(stdout.strip()) + self.assertIn({'pid': pid, 'state': 'reaped', 'owner': result['supervisor']}, result['descendant_cleanup']) + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + + def test_caller_unrelated_live_child_is_not_signalled_reaped_or_waited(self): + foreign = subprocess.Popen([sys.executable, '-I', '-c', 'import time;time.sleep(10)'], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + identity = Path(f'/proc/{foreign.pid}/stat').read_text().rsplit(')', 1)[1].split()[19] + result, stdout, _ = RUN.execute([sys.executable, '-I', '-c', 'print("owned")'], + self.directory, self.env, 2) + RUN.verify(result) + self.assertEqual(stdout.strip(), 'owned') + self.assertIsNone(foreign.poll()) + self.assertEqual(Path(f'/proc/{foreign.pid}/stat').read_text().rsplit(')', 1)[1].split()[19], identity) + self.assertNotIn(foreign.pid, [row['pid'] for row in result['descendant_cleanup']]) + finally: + foreign.kill() + foreign.wait(timeout=3) + + def test_result_pipe_drains_output_larger_than_pipe_capacity(self): + source = 'import sys;sys.stdout.write("λ"*262144);sys.stderr.write("x"*262144)' + result, stdout, stderr = RUN.execute([sys.executable, '-I', '-c', source], self.directory, self.env, 2) + RUN.verify(result) + self.assertEqual(stdout, 'λ' * 262144) + self.assertEqual(stderr, 'x' * 262144) + + def test_root_spawn_error_remains_actionable(self): + with self.assertRaises(FileNotFoundError): + RUN.execute([str(self.directory / 'missing-owned-command')], self.directory, self.env, 2) + + def test_cleanup_failure_preserves_primary_spawn_error_with_diagnostic(self): + with mock.patch.object(RUN, '_reap_descendants', side_effect=RuntimeError('owned cleanup probe failed')): + with self.assertRaises(FileNotFoundError) as failure: + RUN.execute([str(self.directory / 'missing-owned-command')], self.directory, self.env, 2) + self.assertIn('owned cleanup probe failed', '\n'.join(failure.exception.__notes__)) + + def test_cleanup_failure_rejects_an_otherwise_successful_capture(self): + with mock.patch.object(RUN, '_reap_descendants', side_effect=RuntimeError('owned cleanup probe failed')): + with self.assertRaisesRegex(RuntimeError, 'Owned capture cleanup failed'): + RUN.execute([sys.executable, '-I', '-c', 'print("clean")'], self.directory, self.env, 2) + + def test_private_evidence_directory_ignores_permissive_umask_and_refuses_overwrite(self): + previous = os.umask(0) + try: + output = RUN.private_directory(self.directory / 'evidence') + arm = RUN.private_directory(output / 'default-1') + finally: + os.umask(previous) + self.assertEqual(output.stat().st_mode & 0o777, 0o700) + self.assertEqual(arm.stat().st_mode & 0o777, 0o700) + marker = output / 'accepted.json' + marker.write_text('retained') + with self.assertRaises(FileExistsError): + RUN.private_directory(output) + self.assertEqual(marker.read_text(), 'retained') + + def test_reused_fixture_directory_rejects_shared_mode_and_symlinks(self): + shared = self.directory / 'shared' + shared.mkdir(mode=0o700) + shared.chmod(0o755) + with self.assertRaisesRegex(RuntimeError, 'owned private directory'): + RUN.private_directory(shared, reuse=True) + target = self.directory / 'owned' + RUN.private_directory(target) + link = self.directory / 'link' + link.symlink_to(target, target_is_directory=True) + with self.assertRaisesRegex(RuntimeError, 'never a symlink'): + RUN.private_directory(link, reuse=True) + with self.assertRaises(FileExistsError): + RUN.private_directory(link) + + def test_before_import_guard_rejects_network_preconnect_and_install(self): + guard = self.directory / 'guard.ts' + env = {**self.env, 'GUARD_MODULE': str(ROOT / 'guard.ts'), 'GUARD_FILE': str(guard), + 'SELECTOR_FIXTURE_GUARD_LOG': str(self.directory / 'network.jsonl'), + 'SELECTOR_FIXTURE_ORIGIN': 'http://127.0.0.1:1'} + Path(env['SELECTOR_FIXTURE_GUARD_LOG']).write_text('') + create = 'const {guard}=await import(process.env.GUARD_MODULE);await Bun.write(process.env.GUARD_FILE,guard)' + createfile = self.directory / 'create.ts' + createfile.write_text(create) + p = subprocess.run([BUN, 'run', '--no-install', '--no-env-file', str(createfile)], cwd=self.directory, + env=env, stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=3) + self.assertEqual(p.returncode, 0, p.stderr) + cases = [ + ('await fetch("https://example.invalid/offline-probe")', 'blocked nonlocal'), + ('fetch.preconnect("https://example.invalid")', 'forbids preconnect'), + ('Bun.spawn([process.execPath,"install","--help"])', 'forbids dependency installation'), + ] + for source, diagnostic in cases: + with self.subTest(diagnostic=diagnostic): + probe = self.directory / 'probe.ts' + probe.write_text(source) + p = subprocess.run([BUN, 'run', '--no-install', '--no-env-file', '--preload', str(guard), str(probe)], + cwd=self.directory, env=env, stdin=subprocess.DEVNULL, + capture_output=True, text=True, timeout=3) + self.assertNotEqual(p.returncode, 0) + self.assertIn(diagnostic, p.stderr) + ledger = [json.loads(line) for line in Path(env['SELECTOR_FIXTURE_GUARD_LOG']).read_text().splitlines()] + self.assertEqual(ledger, [{'origin': 'https://example.invalid', 'path': '/offline-probe', 'allowed': False}]) + + def test_fixed_workspace_lock_rejects_parallel_campaign_before_output(self): + RUN.private_directory(RUN.FIXTURE, reuse=True) + fd = os.open(RUN.FIXTURE / 'lock', os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + output = self.directory / 'must-not-exist' + with os.fdopen(fd, 'a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + p = subprocess.run([sys.executable, '-I', '-B', str(ROOT / 'run.py'), '--output', str(output)], + cwd=self.directory, env=self.env, stdin=subprocess.DEVNULL, + capture_output=True, text=True, timeout=3) + self.assertNotEqual(p.returncode, 0) + self.assertIn('Exclusive fixed workspace is already in use', p.stderr) + self.assertFalse(output.exists()) + + def test_clock_preload_is_fixed_before_imports(self): + probe = self.directory / 'clock.ts' + probe.write_text('console.log(JSON.stringify({iso:new Date().toISOString(),date:new Date().toDateString(),now:Date.now()}))') + p = subprocess.run([BUN, 'run', '--no-install', '--no-env-file', '--preload', str(ROOT / 'clock.js'), str(probe)], + cwd=self.directory, env=self.env, stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=3) + self.assertEqual(p.returncode, 0, p.stderr) + self.assertEqual(json.loads(p.stdout), {'iso': '2026-10-21T12:00:00.000Z', 'date': 'Wed Oct 21 2026', 'now': 1792584000000}) + +class ReleaseManifest(unittest.TestCase): + def test_missing_final_cleanup_is_explicit_incomplete_campaign(self): + counter = module('release_count', ROOT / 'release-count.py') + for checks in [None, [], [{'name': 'last-run', 'removed': {}, 'remaining': {}}], + [{'finalCampaignCheck': {}}], ['not-an-object']]: + with self.subTest(checks=checks), tempfile.TemporaryDirectory(prefix='release-incomplete-') as directory: + campaign = Path(directory) / 'campaign.json' + data = {'protocolVersion': 3, 'failures': [], 'REPS': 2} + if checks is not None: + data['cleanupChecks'] = checks + campaign.write_text(json.dumps(data)) + with self.assertRaisesRegex(ValueError, 'Incomplete release campaign: final cleanup evidence missing'): + counter.analyze(campaign) + self.assertFalse(campaign.with_name('release-comparison.json').exists()) + + +if __name__ == '__main__': + unittest.main()