From ddda43518c9ba5500ca1ab18539d02926c8c3088 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 21 Oct 2026 04:48:19 +0100 Subject: [PATCH 1/7] test: enforce headless token budgets and release comparisons --- .github/workflows/ci.yml | 24 + RELEASING.md | 41 +- script/token-budget/README.md | 27 + script/token-budget/budgets.json | 4 + script/token-budget/capture.ts | 144 +++++ script/token-budget/clock.js | 13 + script/token-budget/count.py | 91 ++++ script/token-budget/guard.ts | 25 + script/token-budget/reference.py | 63 +++ script/token-budget/release-count.py | 87 +++ script/token-budget/release-fixtures/LICENSE | 21 + .../token-budget/release-fixtures/README.md | 58 ++ .../release-fixtures/token-stub.ts | 498 ++++++++++++++++++ script/token-budget/release.ts | 457 ++++++++++++++++ script/token-budget/requirements.lock | 8 + script/token-budget/run.py | 132 +++++ script/token-budget/setup.py | 46 ++ script/token-budget/test_accounting.py | 136 +++++ script/token-budget/test_protocol.py | 115 ++++ 19 files changed, 1981 insertions(+), 9 deletions(-) create mode 100644 script/token-budget/README.md create mode 100644 script/token-budget/budgets.json create mode 100644 script/token-budget/capture.ts create mode 100644 script/token-budget/clock.js create mode 100644 script/token-budget/count.py create mode 100644 script/token-budget/guard.ts create mode 100644 script/token-budget/reference.py create mode 100644 script/token-budget/release-count.py create mode 100644 script/token-budget/release-fixtures/LICENSE create mode 100644 script/token-budget/release-fixtures/README.md create mode 100644 script/token-budget/release-fixtures/token-stub.ts create mode 100644 script/token-budget/release.ts create mode 100644 script/token-budget/requirements.lock create mode 100644 script/token-budget/run.py create mode 100644 script/token-budget/setup.py create mode 100644 script/token-budget/test_accounting.py create mode 100644 script/token-budget/test_protocol.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 483c6c5e..86a2e6dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,5 +37,29 @@ jobs: sudo apt-get install --no-install-recommends -y ripgrep rg --version + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + + # Setup may download pinned wheels/ranks. Capture and accounting are offline. + - name: Provision reference tokenizer + run: python3 -I -B script/token-budget/setup.py --directory "$RUNNER_TEMP/token-budget-environment" + + - name: Enforce actual SDK prefix token budgets + run: '"$RUNNER_TEMP/token-budget-environment/venv/bin/python3" -I -B script/token-budget/run.py --output "$RUNNER_TEMP/token-budget-evidence"' + + - name: Verify token-budget fault controls + run: '"$RUNNER_TEMP/token-budget-environment/venv/bin/python3" -I -B -m unittest discover -s script/token-budget -p "test_*.py" -v' + env: + TOKEN_BUDGET_TEST_CAMPAIGN: ${{ runner.temp }}/token-budget-evidence + + - name: Retain token-budget captures + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: token-budget-evidence + path: ${{ runner.temp }}/token-budget-evidence + if-no-files-found: warn + - name: Run Tests run: bun turbo test --concurrency=100% diff --git a/RELEASING.md b/RELEASING.md index d8d1e639..218fe148 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -10,6 +10,29 @@ bun test test/cli/run-mcp-discovery.test.ts test/mcp/discovery-recovery.test.ts test/cli/publish-workflow.test.ts test/cli/run-provider-finish.test.ts test/cli/run-signal-cancellation.test.ts test/cli/classify-session-error.test.ts test/session/idle.test.ts test/session/processor-idle.test.ts ``` + Before publication, run the separate current-versus-previous native release + comparison. Prepare and review explicit candidate and previous published binary + artifacts, their SHA256/version/provenance pins and one existing immutable + fixture image using [the manifest contract](script/token-budget/release-fixtures/README.md). + Provision the pinned Python counter in a fresh directory (requires Python3.12 + and network access during setup only), then capture and count offline: + + ```bash + python3.12 script/token-budget/setup.py --directory /absolute/new-counter-env + bun run --no-install --no-env-file script/token-budget/release.ts /absolute/release-inputs.json /absolute/new-release-evidence + /absolute/new-counter-env/venv/bin/python3 -I -B script/token-budget/release-count.py /absolute/new-release-evidence/campaign.json + ``` + + Run these commands from the repository root. Require valid current-default and + previous-default task runs, raw requests/responses and actual Read-result + evidence, verified versions/binary hashes, successful cleanup and + `release-comparison.json`. Review first-input and session input/output/total + deltas before publication and record any accepted regression with rationale. + Minimal arms are measured only when each actual binary advertises `--tools`; + an unsupported previous arm must be labeled UNSUPPORTED, never synthesized + through permission settings. This scripted reference-token gate is separate + from source CI token budgets and from provider billing or model-quality claims. + Wait for CI build, workspace typecheck and tests to pass and address reviews. 3. Merge the approved PR and record the resulting `main` commit SHA. Confirm @@ -43,15 +66,15 @@ publish workflow publishes each as a platform package. It then lists them in the wrapper's `optionalDependencies`: -| Package | `os` | `cpu` | `libc` | -| --- | --- | --- | --- | -| `@aictrl/cli-linux-x64-baseline` | linux | x64 | glibc | -| `@aictrl/cli-linux-x64-baseline-musl` | linux | x64 | musl | -| `@aictrl/cli-linux-arm64` | linux | arm64 | glibc | -| `@aictrl/cli-linux-arm64-musl` | linux | arm64 | musl | -| `@aictrl/cli-darwin-x64-baseline` | darwin | x64 | | -| `@aictrl/cli-darwin-arm64` | darwin | arm64 | | -| `@aictrl/cli-windows-x64-baseline` | win32 | x64 | | +| Package | `os` | `cpu` | `libc` | +| ------------------------------------- | ------ | ----- | ------ | +| `@aictrl/cli-linux-x64-baseline` | linux | x64 | glibc | +| `@aictrl/cli-linux-x64-baseline-musl` | linux | x64 | musl | +| `@aictrl/cli-linux-arm64` | linux | arm64 | glibc | +| `@aictrl/cli-linux-arm64-musl` | linux | arm64 | musl | +| `@aictrl/cli-darwin-x64-baseline` | darwin | x64 | | +| `@aictrl/cli-darwin-arm64` | darwin | arm64 | | +| `@aictrl/cli-windows-x64-baseline` | win32 | x64 | | npm installs only the package that matches the host. Every x64 package is the baseline (no AVX2) build: Bun ships the same binary for its baseline and AVX2 diff --git a/script/token-budget/README.md b/script/token-budget/README.md new file mode 100644 index 00000000..88db0714 --- /dev/null +++ b/script/token-budget/README.md @@ -0,0 +1,27 @@ +# Fixed SDK prefix token guard (Linux) + +This guard starts the public CLI twice per configuration with fresh HOME/XDG state and receives its actual OpenAI-compatible SDK request at a loopback endpoint. It verifies the complete nine-tool default catalog and the explicit `--tools read,bash,edit,write` four-tool catalog. It does not import tests, reconstruct production schemas, invoke a paid model, or replace the existing 15,000-character Qwen guard. + +Prerequisites: Linux x86_64, Python 3.12 with `venv`/`pip`, Bun from `package.json`, real `rg` and Git on PATH, and the repository's installed frozen Bun dependencies and built SDK. Missing prerequisites fail explicitly. + +```sh +# Online setup only: creates a NEW owned directory, pinned official PyPI wheels, +# and checksum-verified OpenAI o200k rank data. It never installs JS packages. +python3.12 -I -B script/token-budget/setup.py --directory /tmp/my-token-environment + +# Offline actual CLI capture and accounting; output must be a NEW directory. +/tmp/my-token-environment/venv/bin/python3 -I -B script/token-budget/run.py \ + --output /tmp/my-token-evidence +``` + +The guard uses an exclusive nonblocking lock in the owner-only `/tmp/aictrl-token-budget` directory. Concurrent invocations fail before changing the fixed workspace. Each cold child runs in the literal `/tmp/aictrl-token-budget/workspace`, newly initialized as a Git repository, with UTC and Date fixed before production imports to 21 October 2026. This controls all three raw cwd occurrences, including the two in Bash descriptions/schema. No text normalization is applied. Existing unowned, shared, or symlink fixture directories are rejected. The directory/lock remain for safe reuse; raw captures remain in the requested evidence directory. + +Only loopback fixture HTTP is allowed by the before-import fetch/preconnect/install guard reused from the selector fixture. Child environments are explicitly allowlisted, without provider credentials or inherited project configuration. The real CLI has a mandatory 15-second deadline; the outer owned process group has 25 seconds and rejects surviving descendants even if its parent exits successfully. These fixture controls are not an OS network sandbox. + +`count.py` verifies the pinned tokenizer version, exact rank SHA256, raw environment, catalog, single request and clean process/network evidence. `reference.py` denies socket/DNS use during accounting. The enforced metric is `o200k_base` encoding of compact sorted Unicode JSON containing actual SDK system-role messages, serialized tools, and instructions when present. The fixture user text is excluded. The full first model input is separately reported using `instructions/messages/input/tools/response_format/text`; transport controls are excluded. System/tool diagnostic counts use separate tokenizations and are not additive BPE contributions. + +`budgets.json` contains two fixed explicit ceilings. Raising either is a reviewed source edit requiring raw evidence and an explanation. `--observe` records an evaluation without enforcing ceilings and is never used by CI. `--source` selects an isolated source copy for QA source-growth controls; CI always uses the repository entry. The default two cold repetitions must produce identical canonical prefix bytes for each arm. + +CI provisions the Python runtime, lock and rank data before capture; capture/accounting do not download. Raw requests, canonical prefixes, network/exit/deadline logs and the complete reference report are retained as `token-budget-evidence`, including when enforcement fails. These counts are reference overhead measurements, not provider billing, cache savings, quality or performance estimates. Release comparisons have a separate current-versus-previous protocol; these two selector arms do not substitute for that requirement. + +Initial portable observation: default **3,441** prefix tokens and selected **2,001**, at the controlled path above (15,453 and 8,973 canonical characters). Both raw request bodies and canonical prefixes were byte-identical across two cold captures per arm. Initial ceilings equal those observed counts with **zero headroom**: deterministic prompt/tool growth should require an explicit reviewed explanation rather than consume a hidden allowance. Full first model input was separately 3,457/2,017. The longer earlier evaluation cwd produced different counts and was not used as a ceiling. diff --git a/script/token-budget/budgets.json b/script/token-budget/budgets.json new file mode 100644 index 00000000..7763cec9 --- /dev/null +++ b/script/token-budget/budgets.json @@ -0,0 +1,4 @@ +{ + "default": 3441, + "four": 2001 +} diff --git a/script/token-budget/capture.ts b/script/token-budget/capture.ts new file mode 100644 index 00000000..8f17815a --- /dev/null +++ b/script/token-budget/capture.ts @@ -0,0 +1,144 @@ +import path from "node:path" +import fs from "node:fs/promises" +import { guard } from "./guard" +const out = process.env.TOKEN_BUDGET_OUTPUT +const work = process.env.TOKEN_BUDGET_WORKSPACE +const source = process.env.TOKEN_BUDGET_SOURCE +if (!out || !work || !source) throw new Error("Explicit output/workspace/source required") +const binpath = [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter) +const rg = Bun.which("rg", { PATH: binpath }) +if (!rg) throw new Error("Real ripgrep prerequisite missing") +const requests: string[] = [] +const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + idleTimeout: 0, + async fetch(request) { + if (new URL(request.url).pathname !== "/v1/chat/completions") throw new Error("Unexpected endpoint " + request.url) + if (request.method !== "POST") throw new Error("Expected POST") + requests.push(await request.text()) + const chunks = [ + { + id: "eval", + object: "chat.completion.chunk", + choices: [{ index: 0, delta: { role: "assistant", content: "Fixture complete" } }], + }, + { id: "eval", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }, + ] + return new Response( + chunks + .map((chunk) => "data: " + JSON.stringify(chunk)) + .concat("data: [DONE]") + .join("\n\n") + "\n\n", + { headers: { "content-type": "text/event-stream" } }, + ) + }, +}) +await fs.mkdir(out, { recursive: true }) +await Bun.write(path.join(out, "guard.ts"), guard) +await Bun.write(path.join(out, "network.jsonl"), "") +const config = { + enabled_providers: ["fixture"], + provider: { + fixture: { + npm: "@ai-sdk/openai-compatible", + options: { apiKey: "fixture", baseURL: server.url.origin + "/v1" }, + models: { fixture: { name: "fixture", tool_call: true, limit: { context: 200000, output: 1000 } } }, + }, + }, + agent: { title: { disable: true } }, + permission: { "*": "allow" }, +} +const command = [ + process.execPath, + "run", + "--no-install", + "--no-env-file", + "--preload", + path.join(import.meta.dir, "clock.js"), + "--preload", + path.join(out, "guard.ts"), + "--conditions=browser", + source, + "run", + "--model", + "fixture/fixture", + "--agent", + "build", + "--format", + "json", + ...(process.env.TOKEN_BUDGET_ARM === "four" ? ["--tools", "read,bash,edit,write"] : []), + "Say Fixture complete without tools.", +] +const env = { + PATH: binpath, + SHELL: "/bin/bash", + TZ: "UTC", + HOME: path.join(out, "home"), + AICTRL_TEST_HOME: path.join(out, "home"), + AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(out, "managed"), + XDG_DATA_HOME: path.join(out, "data"), + XDG_CONFIG_HOME: path.join(out, "config"), + XDG_STATE_HOME: path.join(out, "state"), + XDG_CACHE_HOME: path.join(out, "cache"), + BUN_RUNTIME_TRANSPILER_CACHE_PATH: path.join(out, "bun-cache"), + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + AICTRL_DISABLE_PROJECT_CONFIG: "true", + AICTRL_DISABLE_DEFAULT_PLUGINS: "true", + AICTRL_DISABLE_AUTOCOMPACT: "true", + AICTRL_MODEL_STREAM_IDLE_TIMEOUT_MS: "3000", + AICTRL_CONFIG_CONTENT: JSON.stringify(config), + SELECTOR_FIXTURE_ORIGIN: server.url.origin, + SELECTOR_FIXTURE_GUARD_LOG: path.join(out, "network.jsonl"), +} +const child = Bun.spawn(command, { cwd: work, env, stdin: "ignore", stdout: "pipe", stderr: "pipe" }) +let deadline = false +const timer = setTimeout(() => { + deadline = true + child.kill("SIGKILL") +}, 15000) +try { + const [stdout, stderr, exit] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]) + clearTimeout(timer) + await Bun.write(path.join(out, "stdout.jsonl"), stdout) + await Bun.write(path.join(out, "stderr.log"), stderr) + for (const [index, body] of requests.entries()) await Bun.write(path.join(out, `request-${index}.json`), body) + const network = (await Bun.file(path.join(out, "network.jsonl")).text()) + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)) + const result = { exit, deadline, command, environment: env, rg, requests: requests.length, network } + await Bun.write(path.join(out, "result.json"), JSON.stringify(result, null, 2) + "\n") + if (deadline || exit !== 0 || requests.length !== 1 || network.some((row) => !row.allowed)) + throw new Error( + "Capture failed: " + JSON.stringify({ exit, deadline, requests: requests.length, network }) + stderr.slice(-2000), + ) + const events = stdout + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)) + if ( + events + .filter((event) => event.type === "text") + .map((event) => event.part?.text) + .join("") !== "Fixture complete" + ) + throw new Error("Exact assistant response missing") + if ( + events.filter((event) => event.type === "invocation_complete" && event.status === "completed").length !== 1 || + events.some((event) => event.type === "error" || event.type === "tool_use") + ) + throw new Error("Unexpected invocation lifecycle") +} catch (error) { + console.error(String(error)) + process.exitCode = 1 +} finally { + clearTimeout(timer) + child.kill("SIGKILL") + server.stop(true) +} diff --git a/script/token-budget/clock.js b/script/token-budget/clock.js new file mode 100644 index 00000000..f1c11845 --- /dev/null +++ b/script/token-budget/clock.js @@ -0,0 +1,13 @@ +const epoch = 1792584000000 // 2026-10-21T12:00:00.000Z +const NativeDate = Date +globalThis.Date = new Proxy(NativeDate, { + construct(target, args, next) { + return Reflect.construct(target, args.length ? args : [epoch], next) + }, + apply() { + return new NativeDate(epoch).toString() + }, + get(target, key, receiver) { + return key === "now" ? () => epoch : Reflect.get(target, key, receiver) + }, +}) diff --git a/script/token-budget/count.py b/script/token-budget/count.py new file mode 100644 index 00000000..13a8dc38 --- /dev/null +++ b/script/token-budget/count.py @@ -0,0 +1,91 @@ +"""Count actual SDK captures; no production schema reassembly or normalization.""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from reference import canonical, input_text, load_encoding, RANK_SHA256 + +EXPECTED = {'default': ['bash', 'edit', 'glob', 'grep', 'read', 'skill', 'task', 'webfetch', 'write'], + 'four': ['bash', 'edit', 'read', 'write']} +ENVIRONMENT = "\n Working directory: /tmp/aictrl-token-budget/workspace\n Is directory a git repo: yes\n Platform: linux\n Today's date: Wed Oct 21 2026\n" + + +def analyze(output, observe=False): + encoding = load_encoding() + count = lambda value: len(encoding.encode_ordinary(canonical(value))) + campaign = json.loads((output / 'campaign.json').read_text()) + runs = campaign['runs'] + if campaign['workspace'] != '/tmp/aictrl-token-budget/workspace' or campaign['raw_normalization'] is not False: + raise ValueError('Wrong raw fixture conditions') + repeats = campaign['repeats'] + if not isinstance(repeats, int) or repeats < 2 or len(runs) != 2 * repeats: + raise ValueError('Require complete repeated two-arm capture') + rows = [] + for arm, names in EXPECTED.items(): + for repeat in range(1, repeats + 1): + out = output / f'{arm}-{repeat}' + controller = json.loads((out / 'controller-result.json').read_text()) + if controller not in runs or controller['exit'] or controller['timedOut'] or controller['remaining_group_killed']: + raise ValueError('Unsuccessful controller or surviving process group') + run = json.loads((out / 'result.json').read_text()) + if run['exit'] or run['deadline'] or run['requests'] != 1 or any(not x['allowed'] for x in run['network']): + raise ValueError('Invalid capture exit/deadline/request/network evidence') + ledger = [json.loads(line) for line in (out / 'network.jsonl').read_text().splitlines() if line] + if ledger != run['network'] or len(ledger) != 1 or ledger[0]['path'] != '/v1/chat/completions': + raise ValueError('Unaccounted network ledger or auxiliary request') + if len(list(out.glob('request-*.json'))) != 1: + raise ValueError('Unaccounted request capture') + raw = (out / 'request-0.json').read_text() + body = json.loads(raw) + if body.get('model') != 'fixture': + raise ValueError('Wrong fixture model') + tools = body['tools'] + if sorted(tool['function']['name'] for tool in tools) != names: + raise ValueError('Wrong actual tool catalog: ' + arm) + systems = [m for m in body['messages'] if m['role'] == 'system'] + users = [m for m in body['messages'] if m['role'] == 'user'] + if len(systems) != 1 or len(users) != 1 or len(body['messages']) != 2: + raise ValueError('Unexpected single-turn message roles') + text = systems[0]['content'] + if 'You are aictrl' not in text: + raise ValueError('Base prompt absent') + if re.findall(r'.*?', text, re.S) != [ENVIRONMENT]: + raise ValueError('Uncontrolled raw environment') + prefix = {'messages': systems, 'tools': tools} + if 'instructions' in body: + prefix['instructions'] = body['instructions'] + payload = canonical(prefix) + (out / 'prefix.canonical.json').write_text(payload) + rows.append({'arm': arm, 'repeat': repeat, 'raw_body_sha256': hashlib.sha256(raw.encode()).hexdigest(), + 'prefix_sha256': hashlib.sha256(payload.encode()).hexdigest(), 'prefix_chars': len(payload), + 'prefix_reference_tokens': count(prefix), + 'full_first_model_input_reference_tokens': len(encoding.encode_ordinary(input_text(raw))), + 'system_separate_tokens': count({'messages': systems}), 'tools_separate_tokens': count({'tools': tools}), + 'tool_names': names}) + for arm in EXPECTED: + if len({r['prefix_sha256'] for r in rows if r['arm'] == arm}) != 1: + raise ValueError('Raw prefix differs between cold repetitions: ' + arm) + budgets = None if observe else json.loads((Path(__file__).parent / 'budgets.json').read_text()) + violations = [] if observe else [f"{r['arm']}: {r['prefix_reference_tokens']} > {budgets[r['arm']]}" for r in rows + if r['prefix_reference_tokens'] > budgets[r['arm']]] + report = {'metric': 'raw SDK prefix o200k_base reference tokens; not provider billing/cache/quality/speed', + 'raw_normalization': False, 'tokenizer': 'tiktoken0.12.0/o200k_base', 'rank_sha256': RANK_SHA256, + 'piece_warning': 'System/tool counts encode separate projections and are not additive BPE contributions.', + 'budgets': budgets, 'violations': violations, 'rows': rows} + (output / 'reference-counts.json').write_text(json.dumps(report, indent=2) + '\n') + print(json.dumps(report, indent=2)) + if violations: + raise RuntimeError('Fixed prefix budget exceeded; review source change or explicit budget edit: ' + '; '.join(violations)) + return report + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('output', type=Path) + parser.add_argument('--observe', action='store_true') + args = parser.parse_args() + analyze(args.output, args.observe) diff --git a/script/token-budget/guard.ts b/script/token-budget/guard.ts new file mode 100644 index 00000000..d7f70ac1 --- /dev/null +++ b/script/token-budget/guard.ts @@ -0,0 +1,25 @@ +const guard = [ + 'import { appendFileSync } from "node:fs"', + "const original = globalThis.fetch", + "const spawn = Bun.spawn.bind(Bun)", + "const checked = (...args) => {", + " const command = Array.isArray(args[0]) ? args[0] : args[0]?.cmd", + " if (command && /(?:^|\\/)bun$/.test(String(command[0])) && ['add','install'].includes(command[1])) throw new Error('Fixture forbids dependency installation')", + " return spawn(...args)", + "}", + "Bun.spawn = checked", + "globalThis.fetch = Object.assign(async (input, init) => {", + " const url = new URL(input instanceof Request ? input.url : String(input))", + " if (process.env.SELECTOR_FIXTURE_CODEX === 'true' && url.origin === 'https://chatgpt.com' && url.pathname === '/backend-api/codex/responses') {", + ' appendFileSync(process.env.SELECTOR_FIXTURE_GUARD_LOG,JSON.stringify({origin:url.origin,path:url.pathname,allowed:true,intercepted:true})+"\\n")', + ' appendFileSync(process.env.SELECTOR_FIXTURE_CODEX_LOG,JSON.stringify(JSON.parse(String(init?.body)))+"\\n")', + " const events=[{type:'response.created',response:{id:'resp-fixture',created_at:1,model:'gpt-5.2',service_tier:null}},{type:'response.output_item.added',output_index:0,item:{type:'message',id:'item-fixture'}},{type:'response.output_text.delta',item_id:'item-fixture',delta:'done',logprobs:null},{type:'response.output_item.done',output_index:0,item:{type:'message',id:'item-fixture'}},{type:'response.completed',response:{incomplete_details:null,usage:{input_tokens:1,input_tokens_details:null,output_tokens:1,output_tokens_details:null},service_tier:null}}]", + " return new Response(events.map(event=>'data: '+JSON.stringify(event)).concat('data: [DONE]').join('\\n\\n')+'\\n\\n',{headers:{'content-type':'text/event-stream'}})", + " }", + " const allowed = url.origin === process.env.SELECTOR_FIXTURE_ORIGIN", + ' appendFileSync(process.env.SELECTOR_FIXTURE_GUARD_LOG, JSON.stringify({ origin: url.origin, path: url.pathname, allowed }) + "\\n")', + ' if (!allowed) throw new Error("Selector fixture blocked nonlocal fetch: " + url.origin)', + " return original(input, init)", + '}, { preconnect: () => { throw new Error("Selector fixture forbids preconnect") } })', +].join("\n") +export { guard } diff --git a/script/token-budget/reference.py b/script/token-budget/reference.py new file mode 100644 index 00000000..05839ab8 --- /dev/null +++ b/script/token-budget/reference.py @@ -0,0 +1,63 @@ +"""Pinned offline reference accounting shared by CI and release comparisons.""" +import hashlib +import importlib.metadata +import json +from pathlib import Path +import os +import socket +import sys + +INPUT_FIELDS = ('instructions', 'messages', 'input', 'tools', 'response_format', 'text') +RANK_SHA256 = '446a9538cb6c348e3516120d7c08b09f57c36495e2acfffe59a5bf8b0cfb1a2d' +RANK_KEY = hashlib.sha1(b'https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken').hexdigest() + + +def canonical(value): + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':')) + + +def input_text(body): + payload = json.loads(body) + if not isinstance(payload, dict): + raise ValueError('Request body must be an object') + selected = {key: payload[key] for key in INPUT_FIELDS if key in payload} + if not any(key in selected for key in ('instructions', 'messages', 'input')): + raise ValueError('No model input found') + return canonical(selected) + + +def output_text(response): + if response['kind'] == 'text': + return response['text'] + if response['kind'] == 'tool': + return canonical({'name': response['name'], 'arguments': response['arguments']}) + raise ValueError('Unknown response kind: ' + str(response.get('kind'))) + + +def load_encoding(): + if sys.version_info[:2] != (3, 12): + raise RuntimeError('Require provisioned Python 3.12') + if importlib.metadata.version('tiktoken') != '0.12.0': + raise RuntimeError('Require pinned tiktoken 0.12.0') + cache = Path(sys.prefix).parent / 'cache' + rank = cache / RANK_KEY + if not rank.is_file() or hashlib.sha256(rank.read_bytes()).hexdigest() != RANK_SHA256: + raise RuntimeError('Missing or mismatched pinned o200k rank data; rerun explicit setup') + attempts = [] + + def deny(*args, **kwargs): + attempts.append(repr(args)) + raise RuntimeError('Offline reference counter forbids network') + + socket.socket.connect = deny + socket.socket.connect_ex = deny + socket.create_connection = deny + socket.getaddrinfo = deny + os.environ['TIKTOKEN_CACHE_DIR'] = str(cache) + import tiktoken + encoding = tiktoken.get_encoding('o200k_base') + if attempts: + raise RuntimeError('Unexpected tokenizer network attempt') + if len(encoding.encode_ordinary('hello world')) != 2: + raise RuntimeError('Known-string tokenizer parity failed') + return encoding diff --git a/script/token-budget/release-count.py b/script/token-budget/release-count.py new file mode 100644 index 00000000..2cdcd19f --- /dev/null +++ b/script/token-budget/release-count.py @@ -0,0 +1,87 @@ +"""Account the explicit release comparison with the shared pinned offline counter.""" +import gzip +import hashlib +import json +from pathlib import Path +import statistics +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from reference import input_text, output_text, load_encoding + + +def analyze(campaign_path): + campaign = json.loads(campaign_path.read_text()) + target = campaign_path.with_name('release-comparison.json') + if target.exists(): + raise ValueError('Refuse overwriting accounting results') + if campaign['protocolVersion'] != 3 or campaign['failures'] or campaign['REPS'] < 2: + raise ValueError('Incomplete or failed release campaign') + final = campaign['cleanupChecks'][-1]['finalCampaignCheck'] + if final['exit'] != 0 or final['timedOut'] or final['stdout'].strip(): + raise ValueError('Final campaign container cleanup not verified') + encoding = load_encoding() + arms = {} + for arm in campaign['HARNESSES']: + sessions = [] + runs = campaign['raw']['short'][arm] + if len(runs) != campaign['REPS']: + raise ValueError('Incomplete arm: ' + arm) + for run in runs: + if not run['valid'] or run['exit'] or run['timedOut'] or run['mainRequests'] != 4: + raise ValueError('Invalid capture: ' + run['runId']) + log = campaign_path.parent / 'campaign-logs' / (run['runId'] + '.requests-with-bodies.jsonl.gz') + records = [json.loads(line) for line in gzip.decompress(log.read_bytes()).decode().splitlines()] + counted = [] + for r in records: + if r['kind'] not in ('main', 'aux'): + continue + body = r['body'].encode() + if len(body) != r['bytes'] or hashlib.sha256(body).hexdigest() != r['bodySha256']: + raise ValueError('Raw body hash/byte mismatch') + response = r['responseBody'].encode() + if len(response) != r['responseDelivery']['bodyBytes']: + raise ValueError('Raw SSE response byte mismatch') + counted.append({'kind': r['kind'], 'step': r['step'], + 'request_sha256': hashlib.sha256(body).hexdigest(), + 'response_sha256': hashlib.sha256(response).hexdigest(), + 'input': len(encoding.encode_ordinary(input_text(r['body']))), + 'output': len(encoding.encode_ordinary(output_text(r['response'])))}) + main = [r for r in records if r['kind'] == 'main'] + if len(main) != 4 or [r['step'] for r in main] != [0, 1, 2, 3]: + raise ValueError('Expected exactly ordered scripted steps') + for i in range(3): + call = main[i]['response'] + result = next((r for r in main[i+1]['toolResults'] if r['callId'] == f'call_{i}'), None) + if call['kind'] != 'tool' or call['name'] != 'read' or call['arguments']['filePath'] != f'/workspace/notes-{i}.txt' or not result or not result['complete'] or result['matchedLines'] != 20 or any(result.get(k) for k in ['errorNotice','truncationNotice','nativeSuppressed','nativeGuardrailRefusal']): + raise ValueError('Missing actual exact20line ordered Read result') + if len(main[-1]['toolResults']) != 3 or main[-1]['response']['text'] != 'Done.': + raise ValueError('Missing final completion') + if len(counted) != run['mainRequests'] + run['auxRequests']: + raise ValueError('Request count mismatch') + incoming = sum(r['input'] for r in counted) + outgoing = sum(r['output'] for r in counted) + sessions.append({'runId': run['runId'], 'first_input': next(r['input'] for r in counted if r['kind']=='main'), + 'session_input': incoming, 'session_output': outgoing, 'session_total': incoming+outgoing, + 'capture_sha256': hashlib.sha256(log.read_bytes()).hexdigest(), 'requests': counted}) + arms[arm] = {'sessions': sessions, 'median': {k: statistics.median(r[k] for r in sessions) + for k in ('first_input','session_input','session_output','session_total')}} + if not {'current-default','previous-default'} <= arms.keys(): + raise ValueError('Both actual default versions required') + unsupported = {name+'-selected': 'UNSUPPORTED: actual run --help lacks --tools; no permission/config substitute' + for name, supported in campaign['support'].items() if not supported} + deltas = {k: arms['current-default']['median'][k] - arms['previous-default']['median'][k] + for k in ('first_input','session_input','session_output','session_total')} + result = {'measurement': 'offline o200k_base reference tokens; tiktoken0.12.0; no billing/cache/quality/speed claim', + 'campaign_sha256': hashlib.sha256(campaign_path.read_bytes()).hexdigest(), + 'accounting_source_sha256': hashlib.sha256(Path(__file__).with_name('reference.py').read_bytes()).hexdigest(), + 'arms': arms, 'unsupported': unsupported, 'current_minus_previous_default': deltas, + 'gate': 'Complete valid task/provenance/cleanup comparison required; numerical release deltas require reviewer acceptance, not an automatic no-increase policy'} + target.write_text(json.dumps(result, indent=2)+'\n') + print(json.dumps({k:v['median'] for k,v in arms.items()}, indent=2)) + + +if __name__ == '__main__': + if len(sys.argv) != 2: + raise SystemExit('Usage: OWNED_ENV/venv/bin/python3 -I -B script/token-budget/release-count.py CAMPAIGN.json') + analyze(Path(sys.argv[1])) diff --git a/script/token-budget/release-fixtures/LICENSE b/script/token-budget/release-fixtures/LICENSE new file mode 100644 index 00000000..f44de99e --- /dev/null +++ b/script/token-budget/release-fixtures/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 aictrl + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/script/token-budget/release-fixtures/README.md b/script/token-budget/release-fixtures/README.md new file mode 100644 index 00000000..b5b23b58 --- /dev/null +++ b/script/token-budget/release-fixtures/README.md @@ -0,0 +1,58 @@ +The scripted protocol is adapted from the MIT-licensed +[aictrl public benchmark](https://github.com/aictrl-dev/skill-md-research/blob/cc47b1e5f5277de19c222f7216081110571c5504/experiments/headless-cli-benchmark/harness/token-stub.ts). +The vendored copy adds raw SSE response capture. `LICENSE` retains its notice. + +Release comparisons use explicit local native binaries and an explicit existing +Linux amd64 fixture image. The image must provide Node, a fresh empty runner HOME, +UID10001 and `/workspace` containing the benchmark's 55 exact fixtures. The +controller verifies every file hash before a task. It never pulls or builds an +image, downloads an artifact, installs a dependency, or authenticates a provider. +Artifacts must be obtained and their publication/build provenance reviewed first. +`previous.provenance` is a pinned published npm wrapper/platform inspection +manifest containing `npmWrapperVersion`, `publishedPackage.version`, and +`installedPackages.platformPackages` binary version/hash. `current.provenance` +is a pinned completed build manifest with `complete`, `version`, `binarySha256`. +These records bind identity; reviewers still validate their authenticity and the +candidate's source/build relationship. + +Input manifest (paths resolve relative to the manifest): + +```json +{ + "fixtureImage": "sha256:<64 hex digits>", + "repeats": 2, + "current": { + "path": "current-aictrl", + "sha256": "<64 hex digits>", + "version": "", + "provenance": { "path": "current-build.json", "sha256": "<64 hex digits>" } + }, + "previous": { + "path": "previous-aictrl", + "sha256": "<64 hex digits>", + "version": "", + "provenance": { "path": "previous-npm-inspection.json", "sha256": "<64 hex digits>" } + } +} +``` + +The same fixture image, read-only binary bind, fresh HOME, UID10001, 2 CPU, +2 GiB memory, permissive tool permissions, non-GPT `fixture/fixture` model and +scripted short task apply to both versions. Host networking permits the loopback +stub; this is not an OS egress sandbox. The controller receives an allowlisted +process environment without provider credentials. No real inference occurs. +`--tools read,bash,edit,write` adds a minimal arm only when the binary's actual +`run --help` advertises `--tools`. Previous releases without it are explicitly +unsupported; permission configuration never substitutes for schema selection. + +Each session must contain four ordered model requests, three actual exact +20-line Read results and native Done with exit0. Raw request JSON, raw SSE +responses, stdout, stderr, binary/help/fixture preflight and owned-container +cleanup remain in the new output directory, including failures. The shared +pinned offline `reference.py` counts canonical model-facing fields and all +request input/response output, including repeated history and auxiliary calls. +The comparison records first input, session input/output/total and default +current-minus-previous deltas. These are reference tokens, not provider billing, +cache, quality, reasoning or performance measurements. Review numerical release +deltas explicitly; the release gate does not silently accept an increase or +reuse CI budget thresholds as a cross-version claim. diff --git a/script/token-budget/release-fixtures/token-stub.ts b/script/token-budget/release-fixtures/token-stub.ts new file mode 100644 index 00000000..bb62a9cd --- /dev/null +++ b/script/token-budget/release-fixtures/token-stub.ts @@ -0,0 +1,498 @@ +// Token-capture snapshot: adds output argument evidence only. +/** + * Scripted model stub shared by every harness. It speaks two wire APIs on the same port: + * POST /v1/chat/completions (OpenAI chat completions, streaming) aictrl, opencode, pi + * POST /v1/responses (OpenAI Responses API, streaming) codex + * Standard OpenAI model discovery is supported and logged separately. Other backend + * probes are logged and answered 404, so stray calls show up in the results. + * + * The next turn is keyed by the latest returned scripted call ID, so retries cannot + * shift it and pruning older results does not restart the loop. History regression fails. The tool to call is + * picked from the request's own tool list: `read` when the harness has one, otherwise its + * shell tool with `cat`. Argument names come from the tool's JSON schema. + * + * A request is "main" when it carries tools and the BENCH marker from the task prompt. + * Anything else (title generation and the like) is "aux": it gets a short text reply and + * does not advance the script. + */ + +export type Scenario = "trivial" | "short" | "long" | "version" +export type Req = { + at: number + epoch: number + path: string + api: "chat" | "responses" | "other" + kind: "main" | "aux" | "discovery" | "other" + step: number + bytes: number + codeUnits: number + toolResults?: FixtureEvidence[] + error?: string + response?: { + kind: "tool" | "text" + callId?: string + name?: string + arguments?: Record + text?: string + } + responseDelivery?: { + stage: "planned" | "body-created" | "handler-returning" + plannedAt: number + plannedEpoch: number + bodyCreatedAt?: number + bodyCreatedEpoch?: number + handlerReturningAt?: number + handlerReturningEpoch?: number + bodyBytes?: number + completionId?: string + completionCreated?: number + clientAbortedAt?: number + clientAbortedEpoch?: number + } + responseBody?: string + body?: string +} + +export const fixturePath = (scenario: Scenario, index: number) => + `/workspace/${scenario === "long" ? "big" : "notes"}-${index}.txt` +export const MARKER = "BENCHTASK" +export const PROTOCOL_VERSION = 3 +const CHAT_NONCE = crypto.randomUUID().replaceAll("-", "") +let chatCompletionSequence = 0 + +export class Stub { + scenario: Scenario = "trivial" + longSteps = 50 + latency: number + log: Req[] = [] + keepBodies = false + server: ReturnType + + constructor(latency: number) { + this.latency = latency + this.server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + idleTimeout: 0, + fetch: async (req) => { + try { + return await this.handle(req) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + console.error(`Stub rejected request: ${message}`) + this.log.push({ + at: performance.now(), + epoch: Date.now(), + path: new URL(req.url).pathname, + api: "other", + kind: "other", + step: -1, + bytes: 0, + codeUnits: 0, + error: message, + }) + return Response.json({ error: { message, type: "benchmark_validation_error" } }, { status: 400 }) + } + }, + }) + } + + get port() { + return this.server.port + } + + reset(scenario: Scenario) { + this.scenario = scenario + this.log = [] + } + + private async handle(req: Request) { + const at = performance.now() + const epoch = Date.now() + const path = new URL(req.url).pathname + const body = req.method === "POST" ? await req.text() : "" + const api = path.endsWith("/chat/completions") ? "chat" : path.endsWith("/responses") ? "responses" : "other" + if (req.method === "GET" && ["/v1/models", "/models", "/v1/models/fixture", "/models/fixture"].includes(path)) { + this.log.push({ + at, + epoch, + path: `GET ${path}`, + api: "other", + kind: "discovery", + step: -1, + bytes: 0, + codeUnits: 0, + }) + const model = { + id: "fixture", + object: "model", + created: 1, + owned_by: "benchmark", + context_window: 200_000, + context_length: 200_000, + max_output_tokens: 1000, + } + return Response.json(path.endsWith("/fixture") ? model : { object: "list", data: [model] }) + } + if (api === "other") { + this.log.push({ + at, + epoch, + path: `${req.method} ${path}`, + api, + kind: "other", + step: -1, + bytes: Buffer.byteLength(body, "utf8"), + codeUnits: body.length, + }) + return new Response("not found", { status: 404 }) + } + const json = JSON.parse(body) + const tools = toolList(json, api) + const main = tools.length > 0 && body.includes(MARKER) + const resultEvidence = fixtureEvidence(json, api, this.scenario) + const done = scriptProgress(resultEvidence) + const furthest = Math.max(0, ...this.log.filter((r) => r.kind === "main").map((r) => r.step)) + if (main && done < furthest) throw new Error(`Tool history regressed from scripted step ${furthest} to ${done}`) + const turn = main ? this.next(done, tools) : { text: "Bench title" } + const firstMain = main && !this.log.some((r) => r.kind === "main") + const evidence: Req = { + at, + epoch, + path, + api, + kind: main ? "main" : "aux", + step: done, + bytes: Buffer.byteLength(body, "utf8"), + codeUnits: body.length, + toolResults: resultEvidence, + response: turn.call + ? { kind: "tool", callId: turn.call.id, name: turn.call.name, arguments: turn.call.args } + : { kind: "text", text: turn.text }, + responseDelivery: { stage: "planned", plannedAt: performance.now(), plannedEpoch: Date.now() }, + body: this.keepBodies || firstMain ? body : undefined, + } + this.log.push(evidence) + const delivery = evidence.responseDelivery! + req.signal.addEventListener( + "abort", + () => { + delivery.clientAbortedAt = performance.now() + delivery.clientAbortedEpoch = Date.now() + }, + { once: true }, + ) + await Bun.sleep(this.latency) + const completion = + api === "chat" + ? { id: `chatcmpl-bench-${CHAT_NONCE}-${++chatCompletionSequence}`, created: Math.floor(Date.now() / 1000) } + : undefined + const sse = api === "chat" ? chatSSE(turn, completion!) : responsesSSE(turn, json) + if (this.keepBodies) evidence.responseBody = sse + delivery.stage = "body-created" + delivery.bodyCreatedAt = performance.now() + delivery.bodyCreatedEpoch = Date.now() + delivery.bodyBytes = Buffer.byteLength(sse, "utf8") + if (completion) { + delivery.completionId = completion.id + delivery.completionCreated = completion.created + } + const response = new Response(sse, { headers: { "content-type": "text/event-stream" } }) + // This is the fetch handler's return boundary, not proof of network delivery or client receipt. + delivery.stage = "handler-returning" + delivery.handlerReturningAt = performance.now() + delivery.handlerReturningEpoch = Date.now() + return response + } + + private next(done: number, tools: Tool[]): Turn { + const wanted = this.scenario === "short" ? 3 : this.scenario === "long" ? this.longSteps : 0 + if (done >= wanted) return { text: "Done." } + return { call: fixtureCall(tools, fixturePath(this.scenario, done), done) } + } +} + +// ---- tool selection --------------------------------------------------------------------- +type Tool = { name: string; params: any; kind: "function" | "other" } +type Call = { id: string; name: string; args: Record } +type Turn = { text?: string; call?: Call } + +function toolList(json: any, api: "chat" | "responses"): Tool[] { + const raw: any[] = json.tools ?? [] + return raw.map((t) => + api === "chat" + ? { name: t.function?.name, params: t.function?.parameters, kind: "function" as const } + : { + name: t.name ?? t.type, + params: t.parameters, + kind: t.type === "function" ? ("function" as const) : ("other" as const), + }, + ) +} + +const READ_NAMES = ["read", "read_file", "view"] +const SHELL_NAMES = [ + "developer__shell", + "terminal", + "bash", + "shell", + "exec_command", + "shell_command", + "run_shell_command", +] + +/** A call that reads `file`: the harness's read tool if it has one, else `cat` through its shell tool. */ +export function fixtureCall(tools: Tool[], file: string, n: number): Call { + const fn = tools.filter((t) => t.kind === "function") + const read = fn.find((t) => READ_NAMES.includes(t.name)) + if (read) { + const props = read.params?.properties ?? {} + const key = ["filePath", "path", "file_path"].find((k) => k in props) ?? "path" + return { + id: `call_${n}`, + name: read.name, + args: fillRequired(read, { [key]: file, ...(read.name === "read_file" ? { offset: 1, limit: 2000 } : {}) }), + } + } + const shell = SHELL_NAMES.map((name) => fn.find((t) => t.name === name)).find(Boolean) + if (!shell) throw new Error(`no read or shell tool in: ${tools.map((t) => t.name).join(",")}`) + const props = shell.params?.properties ?? {} + const cmd = `cat ${file}` + if ("cmd" in props) return { id: `call_${n}`, name: shell.name, args: fillRequired(shell, { cmd }) } + if (props.command?.type === "array") + return { id: `call_${n}`, name: shell.name, args: fillRequired(shell, { command: ["bash", "-lc", cmd] }) } + return { id: `call_${n}`, name: shell.name, args: fillRequired(shell, { command: cmd }) } +} + +/** Plausible values for required arguments the script does not set (e.g. a bash `description`). */ +function fillRequired(tool: Tool, args: Record) { + const props = tool.params?.properties ?? {} + for (const key of tool.params?.required ?? []) { + if (key in args) continue + const type = props[key]?.type + args[key] = type === "number" || type === "integer" ? 1 : type === "boolean" ? false : "Read the fixture file" + } + return args +} + +/** Latest scripted call ID advances the loop even when a harness prunes older results. + * Identical request retries yield the same next call. Unknown IDs fail explicitly. + */ +export function scriptProgress(results: FixtureEvidence[]): number { + if (!results.length) return 0 + const indices = results + .map((r) => /^call_(\d+)(?:_|$)/.exec(r.callId)) + .filter(Boolean) + .map((m) => Number(m![1])) + if (!indices.length) throw new Error("Tool results lost all scripted call IDs; cannot advance benchmark safely") + return Math.max(...indices) + 1 +} + +// ---- wire formats ----------------------------------------------------------------------- +const USAGE = { prompt: 100, completion: 10 } + +function chatSSE(turn: Turn, completion: { id: string; created: number }) { + const delta = turn.call + ? { + tool_calls: [ + { + index: 0, + id: turn.call.id, + type: "function", + function: { name: turn.call.name, arguments: JSON.stringify(turn.call.args) }, + }, + ], + } + : { content: turn.text } + const chunks = [ + { choices: [{ index: 0, delta: { role: "assistant", ...delta }, finish_reason: null }] }, + { + choices: [{ index: 0, delta: {}, finish_reason: turn.call ? "tool_calls" : "stop" }], + usage: { + prompt_tokens: USAGE.prompt, + completion_tokens: USAGE.completion, + total_tokens: USAGE.prompt + USAGE.completion, + }, + }, + ] + return ( + chunks + .map( + (c) => + `data: ${JSON.stringify({ id: completion.id, object: "chat.completion.chunk", created: completion.created, model: "fixture", ...c })}\n\n`, + ) + .join("") + "data: [DONE]\n\n" + ) +} + +let responseSeq = 0 +function responsesSSE(turn: Turn, json: any) { + const id = `resp_${++responseSeq}` + const base = { + id, + object: "response", + created_at: 1, + model: json.model ?? "fixture", + status: "in_progress", + output: [] as unknown[], + } + const item = turn.call + ? { + type: "function_call", + id: `fc_${responseSeq}`, + call_id: turn.call.id + `_${responseSeq}`, + name: turn.call.name, + arguments: JSON.stringify(turn.call.args), + status: "completed", + } + : { + type: "message", + id: `msg_${responseSeq}`, + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: turn.text, annotations: [] }], + } + const usage = { + input_tokens: USAGE.prompt, + input_tokens_details: { cached_tokens: 0 }, + output_tokens: USAGE.completion, + output_tokens_details: { reasoning_tokens: 0 }, + total_tokens: USAGE.prompt + USAGE.completion, + } + const events: [string, unknown][] = [ + ["response.created", { type: "response.created", response: base }], + [ + "response.output_item.added", + { type: "response.output_item.added", output_index: 0, item: { ...item, status: "in_progress" } }, + ], + ] + if (!turn.call) + events.push([ + "response.output_text.delta", + { type: "response.output_text.delta", item_id: item.id, output_index: 0, content_index: 0, delta: turn.text }, + ]) + events.push( + ["response.output_item.done", { type: "response.output_item.done", output_index: 0, item }], + [ + "response.completed", + { type: "response.completed", response: { ...base, status: "completed", output: [item], usage } }, + ], + ) + return events.map(([event, data]) => `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`).join("") +} + +/** Privacy-safe evidence: retain only lengths, fixture coverage and hashes, never provider prompts. */ +export type FixtureEvidence = { + callId: string + bytes: number + codeUnits: number + sha256: string + expectedLines: number + matchedLines: number + firstMarker: boolean + lastMarker: boolean + complete: boolean + truncationNotice: boolean + errorNotice: boolean + nativeSuppressed: boolean + nativeGuardrailRefusal: boolean +} +export function fixtureLines(scenario: Scenario): string[] { + return scenario === "long" + ? Array.from( + { length: 800 }, + (_, i) => `line ${String(i + 1).padStart(4, "0")} lorem ipsum dolor sit amet consectetur adipiscing`, + ) + : Array.from({ length: 20 }, () => "alpha beta gamma") +} +function resultText(value: any): string { + if (typeof value === "string") return value + if (Array.isArray(value)) return value.map(resultText).join("\n") + if (value && typeof value === "object") { + if (typeof value.text === "string") return value.text + if (value.content !== undefined) return resultText(value.content) + } + return value === undefined ? "" : JSON.stringify(value) +} +export function fixtureEvidence(json: any, api: "chat" | "responses", scenario: Scenario): FixtureEvidence[] { + const items = + api === "chat" + ? (json.messages ?? []).filter((m: any) => m.role === "tool") + : (Array.isArray(json.input) ? json.input : []).filter((m: any) => m.type?.endsWith("_call_output")) + const expected = fixtureLines(scenario) + return items.map((m: any, index: number) => { + const content = resultText(api === "chat" ? m.content : m.output) + const flags = nativeResultFlags(content) + // Native tools can add line numbers/XML envelopes. Require every source line in order, + // including 20 occurrences of the repeated short fixture, without requiring identical wrappers. + let cursor = 0, + matchedLines = 0 + for (const line of expected) { + const at = content.indexOf(line, cursor) + if (at < 0) break + matchedLines++ + cursor = at + line.length + } + return { + callId: String(m.tool_call_id ?? m.call_id ?? index), + bytes: Buffer.byteLength(content, "utf8"), + codeUnits: content.length, + sha256: new Bun.CryptoHasher("sha256").update(content).digest("hex"), + expectedLines: expected.length, + matchedLines, + firstMarker: content.includes(expected[0]), + lastMarker: content.includes(expected.at(-1)!), + complete: matchedLines === expected.length, + truncationNotice: flags.truncated, + errorNotice: + flags.error || /(?:^|\n)(?:error:|traceback|permission denied|cat:.*(?:No such|denied))/i.test(content), + nativeSuppressed: flags.suppressed, + nativeGuardrailRefusal: flags.guardrail, + } + }) +} + +/** Native tools often JSON-encode their result inside OpenAI's string content. + * Respect explicit booleans: the field `truncated: false` is not a notice. + */ +function nativeResultFlags(content: string) { + const flags = { truncated: false, error: false, suppressed: false, guardrail: false } + const humanTruncation = (s: string) => + /(?:\btruncated\b|output.{0,30}(?:limit|cap)|(?:lines|characters|bytes) omitted)/i.test(s) + function walk(value: any) { + if (typeof value === "string") { + try { + walk(JSON.parse(value)) + } catch { + flags.truncated ||= humanTruncation(value) + } + return + } + if (Array.isArray(value)) { + value.forEach(walk) + return + } + if (!value || typeof value !== "object") return + for (const [key, item] of Object.entries(value)) { + if (/^truncat/i.test(key)) { + flags.truncated ||= + item !== false && item !== null && item !== 0 && item !== "" && !(Array.isArray(item) && !item.length) + continue + } + if (key === "guardrail_refusal" && item === true) { + flags.guardrail = true + flags.error = true + } + if ( + (key === "is_error" && item === true) || + (key === "error" && !!item) || + (key === "status" && ["error", "blocked", "refused", "failed"].includes(String(item))) + ) + flags.error = true + if ((key === "content_returned" && item === false) || (key === "dedup" && item === true)) flags.suppressed = true + walk(item) + } + } + walk(content) + return flags +} diff --git a/script/token-budget/release.ts b/script/token-budget/release.ts new file mode 100644 index 00000000..d2a0606b --- /dev/null +++ b/script/token-budget/release.ts @@ -0,0 +1,457 @@ +/** Explicit release binary comparison. No paid provider or image build; capture only. */ +import fs from "node:fs" +import path from "node:path" +import os from "node:os" +import { gzipSync } from "node:zlib" +import { MARKER, PROTOCOL_VERSION, Stub } from "./release-fixtures/token-stub" + +type Binary = { path: string; sha256: string; version: string; provenance: { path: string; sha256: string } } +type Inputs = { fixtureImage: string; current: Binary; previous: Binary; repeats: number } +const SHA = /^[a-f0-9]{64}$/ +const digest = (b: string | Uint8Array) => new Bun.CryptoHasher("sha256").update(b).digest("hex") +const [inputArg, outputArg] = process.argv.slice(2) +if (!inputArg || !outputArg || process.argv.length !== 4) + throw new Error("Usage: bun script/token-budget/release.ts INPUT.json NEW-OUTPUT-DIRECTORY") +const inputPath = path.resolve(inputArg) +const inputBytes = fs.readFileSync(inputPath) +const input = JSON.parse(inputBytes.toString()) as Inputs +if (!/^sha256:[a-f0-9]{64}$/.test(input.fixtureImage) || !Number.isInteger(input.repeats) || input.repeats < 2) + throw new Error("Pinned fixture image and repeats >=2 required") +for (const name of ["current", "previous"] as const) { + const bin = input[name] + if ( + !bin || + !SHA.test(bin.sha256) || + typeof bin.version !== "string" || + !bin.version || + !bin.provenance || + !SHA.test(bin.provenance.sha256) + ) + throw new Error(`Explicit ${name} binary/version/hash/provenance required`) + bin.path = path.resolve(path.dirname(inputPath), bin.path) + if ( + fs.lstatSync(bin.path).isSymbolicLink() || + !fs.statSync(bin.path).isFile() || + digest(fs.readFileSync(bin.path)) !== bin.sha256 + ) + throw new Error(`${name} binary hash/file mismatch`) + const bytes = fs.readFileSync(path.resolve(path.dirname(inputPath), bin.provenance.path)) + if (digest(bytes) !== bin.provenance.sha256) throw new Error(`${name} provenance hash mismatch`) + const provenance = JSON.parse(bytes.toString()) + const published = provenance.installedPackages?.platformPackages?.find( + (item: any) => item.binarySha256 === bin.sha256 && item.version === bin.version, + ) + if ( + name === "current" && + (provenance.complete !== true || provenance.binarySha256 !== bin.sha256 || provenance.version !== bin.version) + ) + throw new Error("Current build provenance must bind complete reviewed build/version/binary") + if ( + name === "previous" && + (!published || provenance.publishedPackage?.version !== bin.version || provenance.npmWrapperVersion !== bin.version) + ) + throw new Error("Previous provenance must bind actual published npm wrapper/platform binary/version") +} +if (input.current.sha256 === input.previous.sha256) + throw new Error( + "Current and previous binaries must have distinct SHA256 hashes; self-comparison is not a release comparison", + ) +const output = path.resolve(outputArg) +fs.mkdirSync(output, { recursive: false }) // refuse existing campaign, including failed runs +const logs = path.join(output, "campaign-logs") +fs.mkdirSync(logs, { mode: 0o700 }) +fs.writeFileSync(path.join(output, "original-inputs.json"), inputBytes, { mode: 0o600 }) +for (const name of ["current", "previous"] as const) { + const bytes = fs.readFileSync(path.resolve(path.dirname(inputPath), input[name].provenance.path)) + fs.writeFileSync(path.join(output, `${name}-provenance.json`), bytes, { mode: 0o600 }) + input[name].provenance.path = path.join(output, `${name}-provenance.json`) +} +fs.writeFileSync(path.join(output, "effective-inputs.json"), JSON.stringify(input, null, 2) + "\n", { mode: 0o600 }) +const hostHome = fs.mkdtempSync(path.join(os.tmpdir(), "release248-docker-")) +const hostEnv = { PATH: "/usr/local/bin:/usr/bin:/bin", HOME: hostHome } +const TIMEOUT = 180_000 +let ownedContainer: string | undefined +let proc: ReturnType | undefined +const campaignId = `release248-${process.pid}-${Date.now()}` +const failures: unknown[] = [] +const cleanupChecks: unknown[] = [] +const arms: string[] = ["current-default", "previous-default"] +const support: Record = {} +const raw: Record> = { short: Object.fromEntries(arms.map((x) => [x, []])) } +const protocolSources = Object.fromEntries( + ["release-fixtures/token-stub.ts", "release.ts", "release-count.py", "reference.py"].map((name) => [ + name, + digest(fs.readFileSync(path.join(import.meta.dir, name))), + ]), +) +const metadata = { + purpose: "explicit current versus previous release reference-token capture; no performance comparison", + network: "host network; loopback scripted stub only is intended, but OS egress is not restricted", + model: "fixture/fixture (non-GPT), no real inference", + prompt: `${MARKER} short: read notes-0.txt, notes-1.txt and notes-2.txt once each in order, then reply Done.`, + permission: { "*": "allow" }, + selector: "--tools read,bash,edit,write, only if actual run help supports it", + inputManifestSha256: digest(inputBytes), + protocolSources, + limitations: [ + "Not provider billing/cache/quality evidence", + "Raw inert requests include repeated history and auxiliary calls", + "Timing is diagnostic only; never compare against historical campaigns", + ], +} +let stub: Stub | undefined +function checkpoint() { + fs.writeFileSync( + path.join(output, "campaign.json"), + JSON.stringify( + { + schemaVersion: 2, + protocolVersion: PROTOCOL_VERSION, + HARNESSES: arms, + REPS: input.repeats, + LONG_REPS: 0, + WARMUP: 0, + LATENCY: 50, + CPUS: "2", + MEMORY: "2g", + TIMEOUT_MS: TIMEOUT, + binaries: input, + support, + campaignMetadata: metadata, + raw, + failures, + cleanupChecks, + }, + null, + 2, + ) + "\n", + ) +} +async function docker(args: string[]) { + const child = Bun.spawn(["docker", ...args], { env: hostEnv, stdin: "ignore", stdout: "pipe", stderr: "pipe" }) + let timeout = false + const timer = setTimeout(() => { + timeout = true + child.kill("SIGKILL") + }, TIMEOUT) + try { + const [out, err, code] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]) + if (code || timeout) { + await cleanup() + throw new Error(`Docker preflight failed (${code}, timeout=${timeout}): ${err}`) + } + return out + } finally { + clearTimeout(timer) + } +} +async function cleanupCommand(args: string[]) { + const child = Bun.spawn(["docker", ...args], { env: hostEnv, stdin: "ignore", stdout: "pipe", stderr: "pipe" }) + let timedOut = false + const timer = setTimeout(() => { + timedOut = true + child.kill("SIGKILL") + }, 15_000) + try { + const [stdout, stderr, exit] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]) + return { stdout, stderr, exit, timedOut } + } finally { + clearTimeout(timer) + } +} +async function cleanup() { + try { + if (ownedContainer) { + const name = ownedContainer + const removed = await cleanupCommand(["rm", "-f", name]) + const remaining = await cleanupCommand(["ps", "-aq", "--filter", `name=^/${name}$`]) + cleanupChecks.push({ name, removed, remaining }) + if ( + removed.timedOut || + remaining.timedOut || + remaining.exit !== 0 || + remaining.stdout.trim() || + (removed.exit !== 0 && !removed.stderr.includes("No such container")) + ) + throw new Error(`Unverified container cleanup: ${name}`) + } + } finally { + if (proc && proc.exitCode === null) proc.kill("SIGKILL") + } +} +for (const signal of ["SIGINT", "SIGTERM"] as const) + process.on(signal, () => { + void cleanup().finally(() => process.exit(128 + (signal === "SIGINT" ? 2 : 15))) + }) +try { + const image = JSON.parse(await docker(["image", "inspect", input.fixtureImage]))[0] + if ( + image.Id !== input.fixtureImage || + image.Architecture !== "amd64" || + image.Os !== "linux" || + image.Config.Env.some((entry: string) => !/^(PATH|NODE_VERSION|YARN_VERSION|HOME)=/.test(entry)) + ) + throw new Error("Fixture image identity/platform mismatch") + for (const name of ["current", "previous"] as const) { + const binary = input[name] + const script = `const fs=require('fs'),cp=require('child_process'),crypto=require('crypto'); const binary='/release/aictrl'; console.log(JSON.stringify({uid:process.getuid(),home:fs.readdirSync(process.env.HOME),binarySha256:crypto.createHash('sha256').update(fs.readFileSync(binary)).digest('hex'),version:cp.execFileSync(binary,['--version'],{encoding:'utf8'}).trim(),help:cp.execFileSync(binary,['run','--help'],{encoding:'utf8'}),fixtures:Object.fromEntries(['notes.txt','big.txt',...Array.from({length:3},(_,i)=>'notes-'+i+'.txt'),...Array.from({length:50},(_,i)=>'big-'+i+'.txt')].map(name=>{const b=fs.readFileSync('/workspace/'+name);return[name,{bytes:b.length,sha256:crypto.createHash('sha256').update(b).digest('hex')}]}))}));` + ownedContainer = `${campaignId}-preflight-${name}` + const preflight = JSON.parse( + await docker([ + "run", + "--rm", + "--pull=never", + "--name", + ownedContainer, + "--label", + `aictrl.harness-benchmark=${campaignId}`, + "--network", + "none", + "--user", + "10001:10001", + "--mount", + `type=bind,src=${binary.path},dst=/release/aictrl,readonly`, + "--entrypoint", + "node", + input.fixtureImage, + "-e", + script, + ]), + ) + await cleanup() + ownedContainer = undefined + fs.writeFileSync(path.join(output, `${name}-preflight.json`), JSON.stringify(preflight, null, 2) + "\n") + if ( + preflight.uid !== 10001 || + preflight.home.length || + preflight.binarySha256 !== binary.sha256 || + preflight.version !== binary.version + ) + throw new Error(`${name} binary/UID/HOME/version mismatch`) + for (const [filename, item] of Object.entries(preflight.fixtures) as [ + string, + { bytes: number; sha256: string }, + ][]) { + const short = filename.startsWith("notes") + if ( + item.bytes !== (short ? 340 : 48000) || + item.sha256 !== + (short + ? "268a0ba51362b6181510ff8e8cea2804328d526cb07a97ce8fe56d995f5c64a0" + : "1c756a4da138e2789f27bccbd48c08dcf99bcbf51bd27f7a4442b13b28b912db") + ) + throw new Error(`Fixture mismatch:${filename}`) + } + support[name] = /^\s*--tools\s/m.test(preflight.help) + if (support[name]) { + arms.push(`${name}-selected`) + raw.short[`${name}-selected`] = [] + } + } + stub = new Stub(50) + stub.keepBodies = true + for (let rep = 0; rep < input.repeats; rep++) { + for (const arm of rep % 2 ? [...arms].reverse() : arms) { + stub.reset("short") + const runId = `${String(rep + 1).padStart(2, "0")}-${arm}-short` + ownedContainer = `${campaignId}-${runId}` + const config = { + provider: { + fixture: { + npm: "@ai-sdk/openai-compatible", + options: { apiKey: "fixture", baseURL: `http://127.0.0.1:${stub.port}/v1` }, + models: { fixture: { name: "fixture", tool_call: true, limit: { context: 200_000, output: 1000 } } }, + }, + }, + permission: metadata.permission, + } + const binary = input[arm.startsWith("current-") ? "current" : "previous"] + const cliArgv = [ + "/release/aictrl", + "run", + "--format", + "json", + "--model", + "fixture/fixture", + ...(arm.endsWith("-selected") ? ["--tools", "read,bash,edit,write"] : []), + metadata.prompt, + ] + const argv = [ + "docker", + "run", + "--rm", + "--pull=never", + "--name", + ownedContainer, + "--label", + `aictrl.harness-benchmark=${campaignId}`, + "--network", + "host", + "--user", + "10001:10001", + "--cpus", + "2", + "--memory", + "2g", + "--memory-swap", + "2g", + "-e", + `AICTRL_CONFIG_CONTENT=${JSON.stringify(config)}`, + "--mount", + `type=bind,src=${binary.path},dst=/release/aictrl,readonly`, + "--entrypoint", + "/release/aictrl", + input.fixtureImage, + ...cliArgv.slice(1), + ] + fs.writeFileSync( + path.join(logs, `${runId}.argv.json`), + JSON.stringify({ argv, cliArgv, config, inputManifestSha256: digest(inputBytes) }, null, 2) + "\n", + ) + let timeout = false + let cleanupFailure: string | undefined + const child = Bun.spawn(argv, { env: hostEnv, stdin: "ignore", stdout: "pipe", stderr: "pipe" }) + proc = child + const timer = setTimeout(() => { + timeout = true + void cleanup().catch((error) => { + cleanupFailure = String(error) + }) + }, TIMEOUT) + let stdout = "", + stderr = "", + code: number | null = null + try { + ;[stdout, stderr, code] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]) + } finally { + clearTimeout(timer) + try { + await cleanup() + } catch (error) { + cleanupFailure = String(error) + } + if (!cleanupFailure) ownedContainer = undefined + proc = undefined + } + const requests = [...stub.log] + fs.writeFileSync(path.join(logs, `${runId}.stdout.jsonl`), stdout) + fs.writeFileSync(path.join(logs, `${runId}.stderr.log`), stderr) + fs.writeFileSync( + path.join(logs, `${runId}.requests-with-bodies.jsonl.gz`), + gzipSync( + requests + .map((q) => JSON.stringify({ ...q, bodySha256: q.body === undefined ? null : digest(q.body) })) + .join("\n") + "\n", + ), + { mode: 0o600 }, + ) + fs.writeFileSync( + path.join(logs, `${runId}.requests.jsonl`), + requests + .map(({ body, ...q }) => JSON.stringify({ ...q, bodySha256: body === undefined ? null : digest(body) })) + .join("\n") + "\n", + ) + const errors: string[] = [] + if (cleanupFailure) errors.push(cleanupFailure) + const events: any[] = [] + for (const line of stdout.split("\n").filter(Boolean)) { + try { + events.push(JSON.parse(line)) + } catch { + errors.push("Invalid native NDJSON") + } + } + const main = requests.filter((q) => q.kind === "main") + const tools = main.map((q) => JSON.parse(q.body!).tools.map((t: any) => t.function?.name ?? t.name)) as string[][] + for (const names of tools) { + if (names.some((n) => typeof n !== "string") || new Set(names).size !== names.length) + errors.push("Malformed/duplicate schema names") + if (arm.endsWith("-selected") && [...names].sort().join(",") !== "bash,edit,read,write") + errors.push("Selected schema does not contain exactly four work tools") + if (arm.endsWith("-default") && !names.includes("read")) errors.push("Default catalog lacks scripted Read tool") + } + if (main.length !== 4 || main.map((q) => q.step).join(",") !== "0,1,2,3") + errors.push("Expected four main requests / scripted steps0..3") + for (let i = 0; i < 3; i++) { + const call = main[i]?.response + const evidence = main[i + 1]?.toolResults?.find((q) => q.callId === `call_${i}`) + if ( + call?.kind !== "tool" || + call.name !== "read" || + call.arguments?.filePath !== `/workspace/notes-${i}.txt` || + !evidence?.complete || + evidence.matchedLines !== 20 || + evidence.errorNotice || + evidence.truncationNotice || + evidence.nativeSuppressed || + evidence.nativeGuardrailRefusal + ) + errors.push(`Read${i} not exactly validated`) + } + if ( + main.at(-1)?.toolResults?.length !== 3 || + main.at(-1)?.response?.text !== "Done." || + !events.some((e) => e.type === "text" && /^Done\.?$/.test((e.text ?? e.part?.text ?? "").trim())) + ) + errors.push("Missing final three reads / native Done") + if (requests.some((q) => q.kind === "other")) errors.push("Unexpected provider endpoint request") + if ( + events.some((e) => e.type === "error" || e.is_error || e.type === "invocation_failed") || + requests.some((q) => q.error) + ) + errors.push("Native/stub error") + if (code !== 0 || timeout) errors.push(`Exit${code}, timeout=${timeout}`) + const sample = { + runId, + valid: errors.length === 0, + mainRequests: main.length, + auxRequests: requests.filter((q) => q.kind === "aux").length, + discoveryRequests: requests.filter((q) => q.kind === "discovery").length, + otherRequests: requests.filter((q) => q.kind === "other").map((q) => q.path), + toolNames: tools, + exit: code, + timedOut: timeout, + errors, + } + raw.short[arm].push(sample) + if (errors.length) failures.push({ arm, rep, ...sample }) + checkpoint() + if (errors.length) throw new Error(`Capture ${runId} failed: ${errors.join("; ")}`) + } + } +} catch (error) { + failures.push({ campaignError: error instanceof Error ? error.message : String(error) }) + checkpoint() + throw error +} finally { + try { + await cleanup() + const remaining = await cleanupCommand(["ps", "-aq", "--filter", `label=aictrl.harness-benchmark=${campaignId}`]) + cleanupChecks.push({ finalCampaignCheck: remaining }) + if (remaining.timedOut || remaining.exit !== 0 || remaining.stdout.trim()) + throw new Error("Campaign-owned containers remain or cleanup state could not be inspected") + checkpoint() + } catch (error) { + failures.push({ cleanupError: String(error) }) + checkpoint() + throw error + } finally { + stub?.server.stop(true) + fs.rmSync(hostHome, { recursive: true, force: true }) + } +} +console.log( + `Captured ${input.repeats} sessions per supported arm; containers verified absent in ${output}; run release-count.py next`, +) diff --git a/script/token-budget/requirements.lock b/script/token-budget/requirements.lock new file mode 100644 index 00000000..2ed1de35 --- /dev/null +++ b/script/token-budget/requirements.lock @@ -0,0 +1,8 @@ +# Python 3.12 Linux x86_64; wheels only, official PyPI. Tokenizer stays fixed; support libraries use patched official releases. +tiktoken==0.12.0 --hash=sha256:edde1ec917dfd21c1f2f8046b86348b0f54a2c0547f68149d8600859598769ad +regex==2026.1.15 --hash=sha256:c32bef3e7aeee75746748643667668ef941d28b003bfc89994ecf09a10f7a1b5 +requests==2.34.2 --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 +charset-normalizer==3.4.4 --hash=sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86 +idna==3.20 --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c +urllib3==2.8.0 --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 +certifi==2026.10.19 --hash=sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b diff --git a/script/token-budget/run.py b/script/token-budget/run.py new file mode 100644 index 00000000..49b49d57 --- /dev/null +++ b/script/token-budget/run.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Linux-only public CLI SDK capture; no downloads or model traffic.""" +import argparse +import ctypes +import fcntl +import json +import os +from pathlib import Path +import shutil +import signal +import stat +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parent +FIXTURE = Path('/tmp/aictrl-token-budget') +WORKSPACE = FIXTURE / 'workspace' + + +def execute(command, cwd, env, limit=25): + child = subprocess.Popen(command, cwd=cwd, env=env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) + timed = False + try: + stdout, stderr = child.communicate(timeout=limit) + except subprocess.TimeoutExpired: + timed = True + os.killpg(child.pid, signal.SIGKILL) + stdout, stderr = child.communicate(timeout=5) + surviving = False + try: + os.killpg(child.pid, 0) + surviving = True + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + # As a Linux subreaper, reap orphaned members after terminating our owned group. + end = time.monotonic() + 2 + while True: + try: + pid, _ = os.waitpid(-child.pid, os.WNOHANG) + if not pid and time.monotonic() < end: + time.sleep(0.01) + continue + if not pid: + break + except ChildProcessError: + break + try: + os.killpg(child.pid, 0) + raise RuntimeError('Owned process group remains after cleanup') + except ProcessLookupError: + pass + return {'exit': child.returncode, 'timedOut': timed, 'remaining_group_killed': surviving, + 'process_group': child.pid, 'command': command}, stdout, stderr + + +def verify(result): + if result['remaining_group_killed']: + raise RuntimeError('Owned process group survived controller exit; capture rejected after cleanup') + if result['exit'] or result['timedOut']: + raise RuntimeError('Capture failed or exceeded deadline: ' + json.dumps(result)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', type=Path, required=True, help='New evidence directory; never overwritten') + parser.add_argument('--repeats', type=int, default=2) + parser.add_argument('--source', type=Path, default=ROOT.parents[1] / 'packages/cli/src/index.ts', + help='Explicit alternate source entry for isolated source-growth controls; CI uses repository entry') + parser.add_argument('--observe', action='store_true', help='Report observations without enforcing budgets (evaluation only)') + args = parser.parse_args() + if sys.platform != 'linux' or sys.version_info[:2] != (3, 12): + raise RuntimeError('Require Linux and provisioned Python 3.12') + if args.repeats < 2: + raise ValueError('At least two cold captures per arm required') + bun = shutil.which('bun') + if not bun or not shutil.which('rg'): + raise RuntimeError('Require real bun and ripgrep on PATH; install prerequisites before offline capture') + if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0: + raise OSError('Unable to establish Linux child subreaper') + try: + FIXTURE.mkdir(mode=0o700) + except FileExistsError: + info = FIXTURE.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077: + raise RuntimeError('Fixed fixture must be an owned private directory, never a symlink') + fd = os.open(FIXTURE / 'lock', os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, 'a') as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise RuntimeError('Exclusive fixed workspace is already in use') + output = args.output.resolve() + if output.is_relative_to(FIXTURE): + raise ValueError('Evidence output must be outside the fixed fixture directory') + output.mkdir(parents=True, exist_ok=False) + records = [] + for arm in ['default', 'four']: + for repeat in range(1, args.repeats + 1): + out = output / f'{arm}-{repeat}' + out.mkdir() + if WORKSPACE.is_symlink(): + raise RuntimeError('Refusing a symlink fixed workspace') + if WORKSPACE.exists(): + shutil.rmtree(WORKSPACE) + WORKSPACE.mkdir() + env = {'PATH': os.pathsep.join([str(Path(bun).parent), '/usr/local/bin', '/usr/bin', '/bin']), + 'HOME': str(out / 'controller-home'), 'XDG_CACHE_HOME': str(out / 'controller-cache'), + 'BUN_RUNTIME_TRANSPILER_CACHE_PATH': str(out / 'controller-bun-cache'), + 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null', 'TZ': 'UTC', + 'TOKEN_BUDGET_OUTPUT': str(out), 'TOKEN_BUDGET_WORKSPACE': str(WORKSPACE), + 'TOKEN_BUDGET_SOURCE': str(args.source.resolve(strict=True)), + 'TOKEN_BUDGET_ARM': arm} + subprocess.run(['git', '-c', 'init.defaultBranch=fixture', 'init', '--quiet', str(WORKSPACE)], + env=env, check=True, stdin=subprocess.DEVNULL) + result, stdout, stderr = execute([bun, 'run', '--no-install', '--no-env-file', str(ROOT / 'capture.ts')], ROOT, env) + result.update(arm=arm, repeat=repeat) + (out / 'controller.log').write_text(stdout + stderr) + (out / 'controller-result.json').write_text(json.dumps(result, indent=2) + '\n') + verify(result) + records.append(result) + (output / 'campaign.json').write_text(json.dumps({'runs': records, 'workspace': str(WORKSPACE), + 'raw_normalization': False, 'repeats': args.repeats, 'source_entry': str(args.source.resolve())}, indent=2) + '\n') + subprocess.run([sys.executable, '-I', '-B', str(ROOT / 'count.py'), str(output), + *(['--observe'] if args.observe else [])], check=True, + env={'PATH': env['PATH'], 'HOME': str(output / 'counter-home'), 'TZ': 'UTC'}) + + +if __name__ == '__main__': + main() diff --git a/script/token-budget/setup.py b/script/token-budget/setup.py new file mode 100644 index 00000000..112113f4 --- /dev/null +++ b/script/token-budget/setup.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Online SETUP ONLY: owned Python environment and checksum-verified rank data.""" +import argparse +import hashlib +import json +from pathlib import Path +import platform +import subprocess +import sys +import urllib.request +import venv + +ROOT = Path(__file__).resolve().parent +URL = 'https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken' +SHA = '446a9538cb6c348e3516120d7c08b09f57c36495e2acfffe59a5bf8b0cfb1a2d' +KEY = hashlib.sha1(URL.encode()).hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--directory', type=Path, required=True, help='New owned environment directory') + args = parser.parse_args() + if sys.platform != 'linux' or platform.machine() != 'x86_64' or sys.version_info[:2] != (3, 12): + raise RuntimeError('Lock targets Python 3.12 Linux x86_64') + directory = args.directory.resolve() + directory.mkdir(parents=True, exist_ok=False) + venv.create(directory / 'venv', with_pip=True) + python = directory / 'venv/bin/python3' + subprocess.run([str(python), '-I', '-m', 'pip', '--isolated', 'install', '--disable-pip-version-check', + '--cache-dir', str(directory / 'wheel-cache'), '--index-url', 'https://pypi.org/simple', '--only-binary=:all:', '--require-hashes', + '--no-deps', '-r', str(ROOT / 'requirements.lock')], check=True) + with urllib.request.urlopen(URL, timeout=30) as response: + data = response.read() + if hashlib.sha256(data).hexdigest() != SHA: + raise RuntimeError('Pinned o200k rank SHA256 mismatch') + cache = directory / 'cache' + cache.mkdir() + (cache / KEY).write_bytes(data) + (directory / 'setup.json').write_text(json.dumps({'python': sys.version, 'rank_url': URL, + 'rank_sha256': SHA, 'rank_bytes': len(data), + 'requirements_sha256': hashlib.sha256((ROOT / 'requirements.lock').read_bytes()).hexdigest()}, indent=2) + '\n') + print(f'Offline command: {python} -I -B {ROOT / "run.py"} --output NEW_DIRECTORY') + + +if __name__ == '__main__': + main() diff --git a/script/token-budget/test_accounting.py b/script/token-budget/test_accounting.py new file mode 100644 index 00000000..bbac172d --- /dev/null +++ b/script/token-budget/test_accounting.py @@ -0,0 +1,136 @@ +"""Offline accounting faults using copied real captures and private environment copies.""" +import contextlib +import importlib.util +import io +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parent +sys.path.insert(0, str(ROOT)) +import reference +import count + +class Reference(unittest.TestCase): + def test_public_canonical_protocol_golden(self): + raw = json.dumps({'messages': [{'role': 'user', 'content': 'héllo'}], 'model': 'transport-only', + 'temperature': .1, 'tools': []}, ensure_ascii=False) + self.assertEqual(reference.input_text(raw), '{"messages":[{"content":"héllo","role":"user"}],"tools":[]}') + self.assertEqual(reference.output_text({'kind': 'tool', 'name': 'read', 'arguments': {'z': 1, 'a': 'é'}}), + '{"arguments":{"a":"é","z":1},"name":"read"}') + self.assertEqual(reference.output_text({'kind': 'text', 'text': 'Done'}), 'Done') + with self.assertRaises(ValueError): + reference.input_text('{}') + + def test_pinned_known_string_roundtrip(self): + encoding = reference.load_encoding() + text = 'Hello, world! aictrl offline tokenizer QA.' + encoded = encoding.encode_ordinary(text) + self.assertEqual(len(encoded), 11) + self.assertEqual(encoding.decode(encoded), text) + + def test_actual_copied_environment_rejects_missing_wrong_rank_and_wrong_version(self): + environment = Path(sys.prefix).parent + self.assertTrue((environment / 'setup.json').is_file(), 'Run with the explicit setup-created venv') + script = 'import sys;sys.path.insert(0,sys.argv[1]);import reference;reference.load_encoding()' + for mode, diagnostic in [('missing', 'Missing or mismatched'), ('wrong', 'Missing or mismatched'), + ('version', 'Require pinned tiktoken')]: + with self.subTest(mode=mode), tempfile.TemporaryDirectory(prefix='budget-environment-') as directory: + clone = Path(directory) / 'environment' + shutil.copytree(environment, clone, symlinks=True, copy_function=shutil.copy2, + ignore=shutil.ignore_patterns('wheel-cache')) + rank = clone / 'cache' / reference.RANK_KEY + if mode == 'missing': + rank.unlink() + elif mode == 'wrong': + rank.write_bytes(b'wrong-rank-data') + else: + metadata = next((clone / 'venv/lib/python3.12/site-packages').glob('tiktoken-*.dist-info/METADATA')) + text = metadata.read_text() + self.assertIn('Version: 0.12.0', text) + metadata.write_text(text.replace('Version: 0.12.0', 'Version: 0.11.0', 1)) + p = subprocess.run([str(clone / 'venv/bin/python3'), '-I', '-B', '-c', script, str(ROOT)], + env={'PATH': '/usr/bin:/bin', 'HOME': str(clone / 'home')}, + stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=10) + self.assertNotEqual(p.returncode, 0) + self.assertIn(diagnostic, p.stderr) + +@unittest.skipUnless(os.environ.get('TOKEN_BUDGET_TEST_CAMPAIGN'), 'Set TOKEN_BUDGET_TEST_CAMPAIGN to actual completed capture') +class Captures(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix='budget-capture-fault-') + self.addCleanup(self.temp.cleanup) + self.directory = Path(self.temp.name) + self.original = Path(os.environ['TOKEN_BUDGET_TEST_CAMPAIGN']) + shutil.copy2(self.original / 'campaign.json', self.directory / 'campaign.json') + for source in self.original.glob('*-*'): + if not source.is_dir(): + continue + target = self.directory / source.name + target.mkdir() + for name in ['controller-result.json', 'result.json', 'network.jsonl', 'request-0.json']: + shutil.copy2(source / name, target / name) + self.case = self.directory / 'default-1' + + def analyze(self): + with contextlib.redirect_stdout(io.StringIO()): + return count.analyze(self.directory, observe=True) + + def body(self, transform): + file = self.case / 'request-0.json' + body = json.loads(file.read_text()) + transform(body) + file.write_text(json.dumps(body, ensure_ascii=False)) + + def test_real_captures_reproduce_and_user_text_is_outside_prefix(self): + first = self.analyze() + for file in self.directory.glob('*-*/request-0.json'): + body = json.loads(file.read_text()) + next(m for m in body['messages'] if m['role'] == 'user')['content'] = 'A different explicit user fixture ' * 100 + file.write_text(json.dumps(body)) + second = self.analyze() + self.assertEqual([r['prefix_sha256'] for r in first['rows']], [r['prefix_sha256'] for r in second['rows']]) + self.assertNotEqual([r['full_first_model_input_reference_tokens'] for r in first['rows']], + [r['full_first_model_input_reference_tokens'] for r in second['rows']]) + + def test_wrong_catalog_rejected(self): + self.body(lambda b: b['tools'][0]['function'].update(name='unknown-fixture-work')) + with self.assertRaisesRegex(ValueError, 'Wrong actual tool catalog'): + self.analyze() + + def test_raw_clock_or_cwd_drift_rejected(self): + self.body(lambda b: b['messages'][0].update(content=b['messages'][0]['content'].replace('Wed Oct 21 2026', 'Thu Oct 22 2026'))) + with self.assertRaisesRegex(ValueError, 'Uncontrolled raw environment'): + self.analyze() + + def test_nonidentical_tool_description_repetitions_rejected(self): + self.body(lambda b: b['tools'][0]['function'].update(description=b['tools'][0]['function']['description'] + ' Drift')) + with self.assertRaisesRegex(ValueError, 'differs between cold repetitions'): + self.analyze() + + def test_unaccounted_auxiliary_network_and_request_rejected(self): + file = self.case / 'network.jsonl' + with file.open('a') as stream: + stream.write(json.dumps({'origin': 'http://127.0.0.1:1', 'path': '/v1/models', 'allowed': True}) + '\n') + with self.assertRaisesRegex(ValueError, 'Unaccounted network'): + self.analyze() + + def test_deadline_record_cannot_be_a_negative_success(self): + file = self.case / 'result.json' + data = json.loads(file.read_text()); data['deadline'] = True; file.write_text(json.dumps(data)) + with self.assertRaisesRegex(ValueError, 'Invalid capture'): + self.analyze() + + def test_surviving_controller_group_rejected(self): + file = self.case / 'controller-result.json' + data = json.loads(file.read_text()); data['remaining_group_killed'] = True; file.write_text(json.dumps(data)) + with self.assertRaisesRegex(ValueError, 'Unsuccessful controller'): + self.analyze() + +if __name__ == '__main__': + unittest.main() diff --git a/script/token-budget/test_protocol.py b/script/token-budget/test_protocol.py new file mode 100644 index 00000000..a5199930 --- /dev/null +++ b/script/token-budget/test_protocol.py @@ -0,0 +1,115 @@ +"""Actual child/guard controls for the owned token-budget runner; no provider calls.""" +import ctypes +import fcntl +import importlib.util +import json +import os +from pathlib import Path +import shutil +import signal +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parent +BUN = shutil.which('bun') + +def module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + value = importlib.util.module_from_spec(spec) + spec.loader.exec_module(value) + return value + +RUN = module('budget_run', ROOT / 'run.py') + +class Protocol(unittest.TestCase): + @classmethod + def setUpClass(cls): + if ctypes.CDLL(None).prctl(36, 1, 0, 0, 0) != 0: + raise RuntimeError('Linux subreaper required for owned child controls') + + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix='token-budget-test-') + self.addCleanup(self.temp.cleanup) + self.directory = Path(self.temp.name) + self.env = {'PATH': os.pathsep.join([str(Path(BUN).parent), '/usr/bin', '/bin']), + 'HOME': str(self.directory / 'home'), 'TZ': 'UTC'} + + def test_clean_process_is_accepted(self): + result, stdout, stderr = RUN.execute([sys.executable, '-I', '-c', 'print("clean")'], self.directory, self.env, 2) + RUN.verify(result) + self.assertEqual(stdout.strip(), 'clean') + self.assertEqual(stderr, '') + self.assertFalse(result['remaining_group_killed']) + + def test_actual_deadline_is_never_a_negative_success(self): + result, _, _ = RUN.execute([sys.executable, '-I', '-c', 'import time;time.sleep(10)'], self.directory, self.env, .05) + self.assertTrue(result['timedOut']) + with self.assertRaisesRegex(RuntimeError, 'failed|deadline'): + RUN.verify(result) + with self.assertRaises(ProcessLookupError): + os.killpg(result['process_group'], 0) + + def test_surviving_group_rejected_even_after_parent_exit_zero(self): + source = ('import os,time\npid=os.fork()\nif pid==0:\n os.close(1);os.close(2);time.sleep(10);os._exit(0)\nprint(pid,flush=True)') + result, _, _ = RUN.execute([sys.executable, '-I', '-c', source], self.directory, self.env, 2) + self.assertEqual(result['exit'], 0) + self.assertTrue(result['remaining_group_killed']) + with self.assertRaisesRegex(RuntimeError, 'survived controller exit'): + RUN.verify(result) + with self.assertRaises(ProcessLookupError): + os.killpg(result['process_group'], 0) + + def test_before_import_guard_rejects_network_preconnect_and_install(self): + guard = self.directory / 'guard.ts' + env = {**self.env, 'GUARD_MODULE': str(ROOT / 'guard.ts'), 'GUARD_FILE': str(guard), + 'SELECTOR_FIXTURE_GUARD_LOG': str(self.directory / 'network.jsonl'), + 'SELECTOR_FIXTURE_ORIGIN': 'http://127.0.0.1:1'} + Path(env['SELECTOR_FIXTURE_GUARD_LOG']).write_text('') + create = 'const {guard}=await import(process.env.GUARD_MODULE);await Bun.write(process.env.GUARD_FILE,guard)' + createfile = self.directory / 'create.ts' + createfile.write_text(create) + p = subprocess.run([BUN, 'run', '--no-install', '--no-env-file', str(createfile)], cwd=self.directory, + env=env, stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=3) + self.assertEqual(p.returncode, 0, p.stderr) + cases = [ + ('await fetch("https://example.invalid/offline-probe")', 'blocked nonlocal'), + ('fetch.preconnect("https://example.invalid")', 'forbids preconnect'), + ('Bun.spawn([process.execPath,"install","--help"])', 'forbids dependency installation'), + ] + for source, diagnostic in cases: + with self.subTest(diagnostic=diagnostic): + probe = self.directory / 'probe.ts' + probe.write_text(source) + p = subprocess.run([BUN, 'run', '--no-install', '--no-env-file', '--preload', str(guard), str(probe)], + cwd=self.directory, env=env, stdin=subprocess.DEVNULL, + capture_output=True, text=True, timeout=3) + self.assertNotEqual(p.returncode, 0) + self.assertIn(diagnostic, p.stderr) + ledger = [json.loads(line) for line in Path(env['SELECTOR_FIXTURE_GUARD_LOG']).read_text().splitlines()] + self.assertEqual(ledger, [{'origin': 'https://example.invalid', 'path': '/offline-probe', 'allowed': False}]) + + def test_fixed_workspace_lock_rejects_parallel_campaign_before_output(self): + RUN.FIXTURE.mkdir(mode=0o700, exist_ok=True) + fd = os.open(RUN.FIXTURE / 'lock', os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + output = self.directory / 'must-not-exist' + with os.fdopen(fd, 'a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + p = subprocess.run([sys.executable, '-I', '-B', str(ROOT / 'run.py'), '--output', str(output)], + cwd=self.directory, env=self.env, stdin=subprocess.DEVNULL, + capture_output=True, text=True, timeout=3) + self.assertNotEqual(p.returncode, 0) + self.assertIn('Exclusive fixed workspace is already in use', p.stderr) + self.assertFalse(output.exists()) + + def test_clock_preload_is_fixed_before_imports(self): + probe = self.directory / 'clock.ts' + probe.write_text('console.log(JSON.stringify({iso:new Date().toISOString(),date:new Date().toDateString(),now:Date.now()}))') + p = subprocess.run([BUN, 'run', '--no-install', '--no-env-file', '--preload', str(ROOT / 'clock.js'), str(probe)], + cwd=self.directory, env=self.env, stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=3) + self.assertEqual(p.returncode, 0, p.stderr) + self.assertEqual(json.loads(p.stdout), {'iso': '2026-10-21T12:00:00.000Z', 'date': 'Wed Oct 21 2026', 'now': 1792584000000}) + +if __name__ == '__main__': + unittest.main() From 91f913fba2802967d6bc7589a2a7d8f1ddb5175c Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 21 Oct 2026 07:24:20 +0100 Subject: [PATCH 2/7] ci: verify the integrated headless CLI candidate --- .github/workflows/released-smoke.yml | 51 ++ EVENTS.md | 42 +- README.md | 70 ++- examples/released-smoke/README.md | 31 ++ examples/released-smoke/fixture.txt | 1 + examples/released-smoke/smoke.mjs | 395 ++++++++++++++ examples/released-smoke/workflow.example.yml | 44 ++ packages/cli/README.md | 21 +- packages/cli/src/cli/cmd/generate.ts | 3 + packages/cli/src/cli/cmd/run.input.ts | 11 + packages/cli/src/cli/cmd/run.ts | 54 +- packages/cli/src/command/template.ts | 57 ++ packages/cli/src/session/instruction.ts | 5 +- packages/cli/src/session/prompt.ts | 71 +-- .../cli/fixture/command-arguments-legacy.ts | 18 + .../cli/fixture/command-template-capture.ts | 255 +++++++++ .../cli/fixture/command-template-handler.ts | 18 + .../cli/fixture/command-template-plugin.ts | 13 + .../test/cli/fixture/integrated-capture.ts | 345 ++++++++++++ .../cli/fixture/integrated-observer-plugin.ts | 16 + packages/cli/test/cli/fixture/stdin-guard.ts | 32 ++ .../test/cli/run-command-arguments.test.ts | 435 +++++++++++++++ .../cli/test/cli/run-command-template.test.ts | 412 +++++++++++++++ .../test/cli/run-integrated-headless.test.ts | 218 ++++++++ packages/cli/test/cli/run-stdin.test.ts | 499 ++++++++++++++++++ .../cli/run-tool-selection-attach.test.ts | 2 +- .../cli/test/cli/run-user-message.test.ts | 10 +- .../session/instruction-isolation.test.ts | 157 ++++++ packages/sdk/script/build.ts | 11 + packages/sdk/src/gen/types.gen.ts | 2 +- packages/sdk/src/v2/gen/types.gen.ts | 67 +++ 31 files changed, 3249 insertions(+), 117 deletions(-) create mode 100644 .github/workflows/released-smoke.yml create mode 100644 examples/released-smoke/README.md create mode 100644 examples/released-smoke/fixture.txt create mode 100644 examples/released-smoke/smoke.mjs create mode 100644 examples/released-smoke/workflow.example.yml create mode 100644 packages/cli/src/cli/cmd/run.input.ts create mode 100644 packages/cli/src/command/template.ts create mode 100644 packages/cli/test/cli/fixture/command-arguments-legacy.ts create mode 100644 packages/cli/test/cli/fixture/command-template-capture.ts create mode 100644 packages/cli/test/cli/fixture/command-template-handler.ts create mode 100644 packages/cli/test/cli/fixture/command-template-plugin.ts create mode 100644 packages/cli/test/cli/fixture/integrated-capture.ts create mode 100644 packages/cli/test/cli/fixture/integrated-observer-plugin.ts create mode 100644 packages/cli/test/cli/fixture/stdin-guard.ts create mode 100644 packages/cli/test/cli/run-command-arguments.test.ts create mode 100644 packages/cli/test/cli/run-command-template.test.ts create mode 100644 packages/cli/test/cli/run-integrated-headless.test.ts create mode 100644 packages/cli/test/cli/run-stdin.test.ts create mode 100644 packages/cli/test/session/instruction-isolation.test.ts diff --git a/.github/workflows/released-smoke.yml b/.github/workflows/released-smoke.yml new file mode 100644 index 00000000..dd790b4c --- /dev/null +++ b/.github/workflows/released-smoke.yml @@ -0,0 +1,51 @@ +name: Released aictrl consumer-contract smoke + +# Maintainer-owned branches only; never execute a pull request checkout here. +on: + push: + branches: + - main + - ci/released-consumer-smoke-20261021 + paths: + - .github/workflows/released-smoke.yml + - examples/released-smoke/** + workflow_dispatch: + +permissions: + contents: read + +jobs: + smoke: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + env: + RECIPE_DIR: examples/released-smoke + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 20.20.2 + - name: Install pinned public release without package scripts + shell: bash + run: | + npm install --prefix "$RUNNER_TEMP/aictrl-owned-cli" \ + --ignore-scripts --no-audit --no-fund \ + @aictrl/cli@0.5.4 @aictrl/cli-linux-x64-baseline@0.5.4 + node -e 'const fs=require("node:fs"); const root=process.env.RUNNER_TEMP+"/aictrl-owned-cli/node_modules/@aictrl/"; for(const name of ["cli","cli-linux-x64-baseline"]){const p=JSON.parse(fs.readFileSync(root+name+"/package.json","utf8")); if(p.version!=="0.5.4") throw new Error("Unexpected package version"); console.log(p.name+"@"+p.version)}' + - name: Run owned fixture against released native artifact + shell: bash + run: | + node "$RECIPE_DIR/smoke.mjs" \ + --binary "$RUNNER_TEMP/aictrl-owned-cli/node_modules/@aictrl/cli-linux-x64-baseline/bin/aictrl" \ + --expected-sha256 f12a54259ae0d77d85fd6d0cfefe1dcc742b95a617ca71e48f4fdfb2575182ee \ + --evidence-dir "$RUNNER_TEMP/aictrl-owned-evidence" + - name: Retain only inert fixture evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: aictrl-owned-fixture-evidence + path: ${{ runner.temp }}/aictrl-owned-evidence + if-no-files-found: error + retention-days: 7 diff --git a/EVENTS.md b/EVENTS.md index 3d77b97a..78dc5ec9 100644 --- a/EVENTS.md +++ b/EVENTS.md @@ -21,7 +21,9 @@ The invocation envelope covers accepted `run --format json` executions from the ### `invocation_start` -Emitted once, before piped stdin is read and before run validation or bootstrap begins. +Emitted once, before any selected stdin is read and before run validation or bootstrap begins. Stdin is selected by a standalone `-` message operand (including after `--`), or by no message operands with non-terminal stdin. A supplied ordinary prompt does not read stdin. Repeated markers cause one read and are removed before message and derived-title assembly. An explicitly selected stream still waits for EOF; no stdin deadline is implied by this event. + +Missing required option values and unknown options fail in the argument parser, before the handler starts. They exit with a parser error and do not emit invocation events. Failures after the handler starts retain the invocation lifecycle below. ```json { @@ -103,7 +105,7 @@ Emitted once when the session begins. ### `tool_catalog` -Emitted once per session, immediately after `session_start` and before the first model turn (present even for zero-turn or immediately-failing runs). Lists every tool and skill that was resolved for the session so consumers can structurally verify tool exposure without string-matching the model's prose. +Emitted once per session, immediately after `session_start` and before the first model turn (present even for zero-turn or immediately-failing runs). Reports builtin registry and resolved MCP tool inventory, plus in-scope skills. This session-level snapshot supports inventory diagnostics; it is not the effective tool list for each model request. ```json { @@ -111,8 +113,8 @@ Emitted once per session, immediately after `session_start` and before the first "sessionID": "ses_…", "timestamp": 1741500000000, "tools": [ - { "name": "record_finding", "source": "mcp", "server": "aictrl" }, - { "name": "record_review_completed", "source": "mcp", "server": "aictrl" }, + { "name": "aictrl_record_finding", "source": "mcp", "server": "aictrl" }, + { "name": "aictrl_record_review_completed", "source": "mcp", "server": "aictrl" }, { "name": "bash", "source": "builtin" }, { "name": "read", "source": "builtin" } ], @@ -123,13 +125,13 @@ Emitted once per session, immediately after `session_start` and before the first } ``` -#### `tools[]` — builtin and MCP tools resolved for this session +#### `tools[]` — builtin registry and resolved MCP inventory for this session -- **`name`** (string, **required**) — the tool/function name as exposed to the model. Mirrors upstream `ToolListItem.id`. For MCP tools this is `{serverName}_{toolName}` (both sanitised to `[a-zA-Z0-9_-]`). +- **`name`** (string, **required**) — the tool/function identifier used when a tool is dispatched. Mirrors upstream `ToolListItem.id`. For MCP tools this is `{serverName}_{toolName}` (both sanitised to `[a-zA-Z0-9_-]`). The identifier does not guarantee inclusion in a model request. - **`source`** (string, **required**) — `"builtin"` for tools built into the CLI; `"mcp"` for tools provided by a connected MCP server. - **`server`** (string, optional) — the MCP server (client name from config) that provides this tool. Present only when `source` is `"mcp"`. -`description` and `parameters` are intentionally omitted to keep the event lean. The primary consumer only needs `name` + `source` to verify tool exposure. +`description` and `parameters` are intentionally omitted to keep the event lean. `name` + `source` identify inventory entries and their provenance; they do not prove dispatch exposure. #### `skills[]` — skill packs resolved for this session (separate from `tools[]`) @@ -138,9 +140,9 @@ Only skills in scope for the run are listed; see [Skill Events](#skill-events). - **`name`** (string, **required**) — skill name from `SKILL.md` frontmatter. - **`version`** (string or null, **required**) — version string from the `version` field in `SKILL.md` frontmatter. `null` when no version is declared. -> **Use case:** The server-side completion gate (aictrl-dev/aictrl #3216) uses `tools[]` to verify that `record_finding` and `record_review_completed` were actually in the model's function list at dispatch time, and `skills[]` to record which skill version produced the review. Detects the "silent success" failure mode where a missing MCP server lets a run complete green without ever having the review tools available. +> **Use case:** A server-side completion gate (aictrl-dev/aictrl #3216) can use `tools[]` to detect missing expected inventory and `skills[]` to record resolved skill versions. To verify that `aictrl_record_finding` and `aictrl_record_review_completed` were offered on a particular turn, it must inspect that actual model request's dispatched function list after filtering. Inventory presence alone proves neither dispatch nor a successful review-tool call. > -> **Note on builtin tool filtering:** The `source: "builtin"` entries in `tools[]` reflect the _instance-level superset_ registered in `ToolRegistry`. Per-model filters (e.g. `apply_patch` only on gpt-\*, `codesearch`/`websearch` only for aictrl provider) are applied at dispatch time inside `resolveTools` and are **not** reflected here. Consumers should treat the presence of a builtin tool name as "registered and potentially available", not as "guaranteed to appear in the model's function list". The strong structural guarantee (tool present ↔ tool in dispatch list) applies only to `source: "mcp"` entries. +> **Inventory versus dispatch:** Builtin entries reflect the _instance-level superset_ registered in `ToolRegistry`; model-specific builtin filters are applied when resolving each request. MCP entries come from connected servers' resolved tool catalogs. Both sources can be excluded by effective work-tool selection, permissions or request-specific disabling. The shared MCP naming helper guarantees identifier consistency, not inclusion in the dispatched function list. ### `session_complete` @@ -250,6 +252,10 @@ Emitted once for each primary-session user message, after all of its parts are s - `parts` (array, **required**) — the message's parts exactly as stored, in the order they were written. Parts the CLI derived from the input have `synthetic: true`; for example, the text a `-f` file or MCP resource expands to, or the compaction "continue" text. A compaction request has a single `compaction` part. Parts are not truncated: a `file` part can carry a `data:` URL, so consumers must accept arbitrarily large lines, as for `reasoning`. - `sequenceNum` (number, **required**) — the same per-session counter as `text`, `reasoning` and `tool_use`. A `user_message` is emitted before any event of the assistant turn it triggers, so its `sequenceNum` is lower than theirs. +Ordinary prompt text joins normalized message operands with spaces without adding quote or backslash encoding; selected stdin is appended after one actual newline. Command templates receive exact structured operands and the selected stream as one final positional data operand. The highest numbered placeholder joins its remaining operands with spaces. Raw `$ARGUMENTS`, plugin arguments and command-executed arguments use the textual projection `argv.join(" ")` plus a newline and exact selected stdin. Existing string SDK inputs retain their legacy lexer. Literal dollar replacement tokens are not interpreted as JavaScript replacement instructions. These data-fidelity rules do not establish attached receiver support. + +Only shell spans present in the original resolved command template execute. Argument values, selected stdin, fallback text and captured shell output are never rescanned for placeholders or shell spans. In original shell spans, numbered placeholders bind scalar data using the original template's global highest-position suffix rule, while `$ARGUMENTS` binds the decoded operand vector. Bun supplies unquoted vector elements as separate arguments and quoted vectors as one space-joined scalar; missing numbered values bind empty scalars. This intentionally replaces raw shell-source interpolation while keeping the existing template-source authority, trusted operators and explicit interpreters. It is independent of model tool permissions and is not a sandbox. The expanded template keeps its outer whitespace trim before user-message parts are resolved. + The StructuredOutput corrective reminder is not a stored user message. It is added to the next model request only, so it emits no `user_message`; `structured_output_rejected` reports each corrective turn instead. Child-session (subagent) user messages are not emitted, matching `text`. ### `message_complete` @@ -670,15 +676,15 @@ can contain property names submitted by the model (for example, in map or to 64 characters; paths also have an overall length bound. A result file that cannot be used emits one fixed diagnostic with `path: ""`: -| `keyword` | `message` | -|---|---| -| `file` | `working directory unavailable` | -| `file` | `file not found` | -| `file` | `path must stay inside the working directory` | -| `file` | `file not readable` | -| `file` | `path must be a regular file` | -| `file` | `file must not exceed 2097152 bytes` | -| `parse` | `file must contain complete, valid JSON` | +| `keyword` | `message` | +| --------- | --------------------------------------------- | +| `file` | `working directory unavailable` | +| `file` | `file not found` | +| `file` | `path must stay inside the working directory` | +| `file` | `file not readable` | +| `file` | `path must be a regular file` | +| `file` | `file must not exceed 2097152 bytes` | +| `parse` | `file must contain complete, valid JSON` | A file that parses but fails the schema emits Ajv diagnostics as above. Feedback for a rejected file goes to the model as the StructuredOutput tool error and diff --git a/README.md b/README.md index a60ce76f..adccc41b 100644 --- a/README.md +++ b/README.md @@ -26,14 +26,14 @@ Interactive coding agents are built for a person at a terminal: in automation th Engineers running terminal-first coding agents in CI and batch workflows repeatedly hit the same failure modes: -| What breaks in automation | How `aictrl run` handles it | -| :--- | :--- | -| **Silent hangs** (`claude-code-action#865`, `goose#11164`) | **No terminal UI or interactive prompts.** Any unpermitted tool request is rejected immediately; optional stream idle timeouts stop stalled model calls. | -| **False `exit 0` on failure** (`claude-code#79500`, `goose#11909`, `gemini-cli#1088`) | **Distinct exit codes:** `0` accepted result, `1` provider/runtime/timeout error, `2` configuration error, `3` missing or schema-invalid result, `130`/`143` on `SIGINT`/`SIGTERM`. | -| **Unparseable prose or broken JSON** (`codex#19816`, `claude-code#18536`, `gemini-cli#4665`) | **Enforced JSON Schema (`--output-schema`)** with bounded corrective turns and atomic file output (`--output-result`). No prose fallback. | -| **`--dangerously-skip-permissions` in CI** | **Declarative per-tool allow/deny rules.** Tools denied for all patterns are stripped from the model request entirely, saving tokens and shrinking the attack surface. Subagents inherit caller denials. | -| **Cost & token blind spots** (`claude-code#8245`, `codex#13733`) | **608-token system prompt** and a [versioned NDJSON event stream](EVENTS.md) on stdout reporting per-call token usage, cost, tool timing, and permission decisions. | -| **Background auto-updates & telemetry** (`claude-code-action#1290`) | **Zero telemetry, no auto-update, built-in model catalog.** Outbound traffic goes only to the model provider, MCP servers, and web tools you configure. | +| What breaks in automation | How `aictrl run` handles it | +| :------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Silent hangs** (`claude-code-action#865`, `goose#11164`) | **No terminal UI or interactive prompts.** Any unpermitted tool request is rejected immediately; optional stream idle timeouts stop stalled model calls. | +| **False `exit 0` on failure** (`claude-code#79500`, `goose#11909`, `gemini-cli#1088`) | **Distinct exit codes:** `0` accepted result, `1` provider/runtime/timeout error, `2` configuration error, `3` missing or schema-invalid result, `130`/`143` on `SIGINT`/`SIGTERM`. | +| **Unparseable prose or broken JSON** (`codex#19816`, `claude-code#18536`, `gemini-cli#4665`) | **Enforced JSON Schema (`--output-schema`)** with bounded corrective turns and atomic file output (`--output-result`). No prose fallback. | +| **`--dangerously-skip-permissions` in CI** | **Declarative per-tool allow/deny rules.** Tools denied for all patterns are stripped from the model request entirely, saving tokens and shrinking the attack surface. Subagents inherit caller denials. | +| **Cost & token blind spots** (`claude-code#8245`, `codex#13733`) | **608-token system prompt** and a [versioned NDJSON event stream](EVENTS.md) on stdout reporting per-call token usage, cost, tool timing, and permission decisions. | +| **Background auto-updates & telemetry** (`claude-code-action#1290`) | **Zero telemetry, no auto-update, built-in model catalog.** Outbound traffic goes only to the model provider, MCP servers, and web tools you configure. | ## Install @@ -55,10 +55,34 @@ aictrl run --format json \ --output-result findings.json \ "Review the changes on this branch and report security or reliability bugs" -# Pipe input from stdin -git diff origin/main...HEAD | aictrl run --model openai/gpt-5 "Summarize breaking API changes" +# Explicitly append stdin to a prompt +git diff origin/main...HEAD | aictrl run --model openai/gpt-5 "Summarize breaking API changes" - ``` +Stdin is read only when a standalone `-` is a message operand, or when no message operands are supplied and stdin is not a terminal. A supplied prompt proceeds without reading an unrelated open pipe. Existing pipelines that combine a prompt and piped context must add `-` as shown above. + +The marker can appear before or after `--`; repeated markers still read stdin once. Markers are removed from the model message, command arguments and derived session title. `--` ends option parsing, so `run -- -` also selects stdin. An operand such as `"literal -"` remains ordinary text. An explicit empty operand (`run ""`) does not select implicit stdin. Selected stdin is read to EOF and appended after one actual newline. Ordinary prompt operands are joined with spaces without adding artificial quotes or backslash escapes; quotes, backslashes and whitespace inside an operand remain its text. + +Option values belong to their options: `--model=-` or `--command=-` does not select stdin, and scalar options also accept a separated hyphen value. Use `--file=-` to attach a file literally named `-`; `--file -` is missing its required file value. Multiple file values and repeated `--skill` flags retain their existing behavior. Required option values are checked by the argument parser; explicit `--title ""` remains supported. + +An explicit marker reads stdin even from a terminal. Any selected stream can wait for EOF indefinitely; this change adds no stdin timeout. Callers must close selected input or apply their own supervisor deadline. + +### Command argument data + +`--command` supplies structured arguments as `{ argv: string[], stdin?: string }`. Each normalized operand retains its exact value, including empty text, embedded quotes, tabs, newlines and Windows paths. An explicitly selected stream, even an empty stream, is preserved separately and becomes one final positional data operand. The highest numbered `$N` placeholder still consumes the remaining operands joined with spaces; missing positions remain empty. + +`$ARGUMENTS`, command plugins and the executed-command event receive textual arguments derived as `argv.join(" ")` plus one newline and the exact stream when stdin was selected. Commands without argument placeholders append this text after two newlines. This projection preserves text but does not encode operand boundaries. Session titles keep their existing separate derivation. Legacy SDK callers may continue passing an `arguments` string; its existing quote-tokenization and positional rules remain unchanged. Raw `$ARGUMENTS` insertion now treats dollar replacement tokens as literal data. + +Attached command execution requires a receiver that accepts this structured argument contract. This headless checkout does not include that HTTP receiver or capability negotiation, so attached compatibility is not established by the local implementation. Do not retry a rejected structured request by converting it to a legacy string: that can lose argument boundaries. Command-template shell preprocessing follows the separate authority and data-binding rules below. + +### Command-template shell expansion + +Only `!` followed by a backtick-delimited shell span already present in the resolved command template executes. Operands, selected stdin, appended fallback text and shell output are inserted as data; they cannot create additional shell spans or argument placeholders. Configured commands, skill commands and MCP-provided command templates keep their existing template authority. Treat executable templates from any of these sources as code you choose to trust. This preprocessing runs independently of model tool permissions; denying the model's Bash tool does not disable an original template's shell span. + +Inside an original shell span, `$N` is one scalar data value. The highest numbered placeholder in the **whole original template**, including prose and shell spans, still consumes the remaining operands joined with spaces. Missing positions bind an empty scalar. `$ARGUMENTS` binds the decoded operand vector, including a selected stream as one final element. Unquoted vectors supply separate shell arguments; inside single or double quotes Bun binds the vector as one space-joined scalar. An empty unquoted vector supplies no argument; an explicitly empty element or missing numbered scalar remains an empty data argument. Quoting an empty vector binds an empty scalar. Outside shell spans, `$ARGUMENTS` continues using the textual projection described above. + +These binding rules intentionally replace raw argument-to-shell-source interpolation. Trusted template operators, pipelines, substitutions and explicit interpreters remain available. An author may deliberately invoke an interpreter or `eval` that reinterprets data; binding is not a sandbox and cannot promise that arbitrary trusted programs preserve their inputs. Shell spans retain concurrent execution, captured stdout, nonthrowing exit handling and the existing execution-error text. The complete expanded template retains its final outer whitespace trim. + ### 2. Drop-in GitHub Actions PR review (split-secret pattern) Save as `.github/workflows/aictrl-review.yml`. No step holds both the model API key and the GitHub token, and the agent runs with read-only tools (`read`, `glob`, `grep` only—no `bash`, no `write`, no repo config or `AGENTS.md` instructions from the pull request): @@ -145,19 +169,19 @@ Notes on this example: We benchmarked six coding-agent CLIs (`aictrl 0.5.4`, `pi 1.3.0`, `Goose 1.55.0`, `Codex 0.163.0`, `OpenCode 2.1.0`, and `Hermes 0.22.2`) in fresh 2-CPU Docker containers against a deterministic 50 ms local stub model to isolate harness overhead from remote LLM latency. Full runner, raw JSON captures, and methodology are open in [`aictrl-dev/skill-md-research`](https://github.com/aictrl-dev/skill-md-research/tree/main/experiments/headless-cli-benchmark) and analyzed in the [benchmark post](https://aictrl.dev/blog/headless-coding-agent-cli-benchmark?utm_source=github&utm_medium=readme&utm_campaign=headless-readme). -| Metric (lower is better, except coverage) | **aictrl 0.5.4** | pi 1.3.0 | Goose 1.55.0 | Codex 0.163.0 | OpenCode 2.1.0 | Hermes 0.22.2 | -| :--- | ---: | ---: | ---: | ---: | ---: | ---: | -| **First response, median** (ms) | 579 | 522 | **392** (1st) | 471 (2nd) | 926 | 5,521 | -| **50-read session time, median** (s) | **4.27** (2nd) | **3.98** (1st) | 7.22 | 5.57 | 6.03 | 12.76 | -| **Harness time per turn, median** (ms) | **12** (2nd) | **9** (1st) | 72 | 45 | 32 | 64 | -| **Peak cgroup memory, median** (MiB) | **207** (2nd) | **154** (1st) | 247 | 132\* | 424 | 344 | -| **Container image increment** (MiB) | **113** (2nd) | **108** (1st) | 292 | 410 | 818 | 388 | -| **System prompt** (`o200k_base` tokens) | 608 | 601 (2nd) | **563** (1st) | 4,047 | 1,857 | 1,053 | -| **First request total** (tokens, default tools) | 3,442 (9 tools) | **1,276** (4 tools) | 1,577 (6 tools) | 8,178 (9 tools) | 4,903 (12 tools) | 3,391 (8 tools) | -| **Short task, 4 calls** (tokens, default tools) | 15,209 (9,805 w/ 4 tools) | **6,074** (1st) | 7,281 (2nd) | 34,078 | 21,489 | 15,562 | -| **NDJSON event stream coverage** (25 facts) | **100%** | **100%** | 68% | 40% | 80% | 72% | - -*\*Codex's shell reader truncates each 800-line file to its first and last ~80 lines, holding ~1/5th of the file in context. `pi`, `Goose`, and `Hermes` ran without a subagent tool in the tested profile; restricting `aictrl` to `pi`'s 4 tools via permission config drops its first request from 3,442 to 2,091 tokens.* +| Metric (lower is better, except coverage) | **aictrl 0.5.4** | pi 1.3.0 | Goose 1.55.0 | Codex 0.163.0 | OpenCode 2.1.0 | Hermes 0.22.2 | +| :---------------------------------------------- | ------------------------: | ------------------: | --------------: | --------------: | ---------------: | --------------: | +| **First response, median** (ms) | 579 | 522 | **392** (1st) | 471 (2nd) | 926 | 5,521 | +| **50-read session time, median** (s) | **4.27** (2nd) | **3.98** (1st) | 7.22 | 5.57 | 6.03 | 12.76 | +| **Harness time per turn, median** (ms) | **12** (2nd) | **9** (1st) | 72 | 45 | 32 | 64 | +| **Peak cgroup memory, median** (MiB) | **207** (2nd) | **154** (1st) | 247 | 132\* | 424 | 344 | +| **Container image increment** (MiB) | **113** (2nd) | **108** (1st) | 292 | 410 | 818 | 388 | +| **System prompt** (`o200k_base` tokens) | 608 | 601 (2nd) | **563** (1st) | 4,047 | 1,857 | 1,053 | +| **First request total** (tokens, default tools) | 3,442 (9 tools) | **1,276** (4 tools) | 1,577 (6 tools) | 8,178 (9 tools) | 4,903 (12 tools) | 3,391 (8 tools) | +| **Short task, 4 calls** (tokens, default tools) | 15,209 (9,805 w/ 4 tools) | **6,074** (1st) | 7,281 (2nd) | 34,078 | 21,489 | 15,562 | +| **NDJSON event stream coverage** (25 facts) | **100%** | **100%** | 68% | 40% | 80% | 72% | + +_\*Codex's shell reader truncates each 800-line file to its first and last ~80 lines, holding ~1/5th of the file in context. `pi`, `Goose`, and `Hermes` ran without a subagent tool in the tested profile; restricting `aictrl` to `pi`'s 4 tools via permission config drops its first request from 3,442 to 2,091 tokens._ ## Core Contracts for Automation diff --git a/examples/released-smoke/README.md b/examples/released-smoke/README.md new file mode 100644 index 00000000..8a2e2af7 --- /dev/null +++ b/examples/released-smoke/README.md @@ -0,0 +1,31 @@ +# Released CLI consumer-contract smoke + +A credential-free companion to the [real-provider PR review recipe](../../README.md#2-drop-in-github-actions-pr-review-split-secret-pattern). This runs published **0.5.4**, rather than building repository source. A scripted loopback provider requests a real file read and checks the returned bytes. It tests CLI integration, JSON events and failure handling; it does not measure model coding quality or establish safety for untrusted checkouts. + +## Run locally + +Use Node **20.20.2** and the published Linux x64 baseline native package. Package setup needs network access; package scripts are disabled: + +```bash +CLI_DIR=$(mktemp -d) +npm install --prefix "$CLI_DIR" --ignore-scripts --no-audit --no-fund \ + @aictrl/cli@0.5.4 @aictrl/cli-linux-x64-baseline@0.5.4 +node examples/released-smoke/smoke.mjs \ + --binary "$CLI_DIR/node_modules/@aictrl/cli-linux-x64-baseline/bin/aictrl" \ + --expected-sha256 f12a54259ae0d77d85fd6d0cfefe1dcc742b95a617ca71e48f4fdfb2575182ee \ + --evidence-dir "$CLI_DIR/evidence" +``` + +The runner verifies the native hash/version, two provider requests, an effective wire catalog containing only `read`, the completed read result, schema-version-1 NDJSON, successful terminal events and exit 0. A second invocation supplies invalid root schema `[]`: it must exit 2 with `OUTPUT_SCHEMA_CONFIG` before any additional model request. This is schema rejection, not successful structured output. + +The CLI gets closed stdin, fresh HOME/XDG/workspace directories and an explicit environment allowlist without host credentials. Each subprocess has a 30-second deadline; the whole run has 60 seconds. Cleanup terminates POSIX process groups, escalates to SIGKILL after five seconds, and removes temporary state. A successful run prints `"status": "pass"`; a nonzero exit is failure. Evidence contains only the fixed inert fixture, public prompts and a placeholder key. Remove `CLI_DIR` after inspecting its evidence. + +## GitHub Actions + +The [active workflow](../../.github/workflows/released-smoke.yml) runs on changes to this directory or itself on `main` and the dedicated maintainer branch, plus manual dispatch. It has a five-minute job timeout, immutable action revisions, pinned Node/npm package versions and no user-managed secrets. `contents: read` follows the pinned checkout action's [recommended permissions](https://github.com/actions/checkout/blob/d23441a48e516b6c34aea4fa41551a30e30af803/README.md#recommended-permissions); credentials are not persisted or forwarded to the CLI. The [manual-only example](workflow.example.yml) can be copied to another trusted maintained branch with the recipe path preserved. + +Neither workflow runs on `pull_request` or `pull_request_target`. The npm setup step uses network access; the CI job does not enforce no-egress isolation. The local transport binds only an ephemeral loopback port. Environment and tool restrictions are not an operating-system sandbox. + +Released instruction discovery is imperfect ([#250](https://github.com/aictrl-dev/cli/issues/250)), so the runner rejects ancestor `AGENTS.md` and `.opencode/AGENTS.md` files before launching the CLI. Closed stdin avoids [#210](https://github.com/aictrl-dev/cli/issues/210). Ambient shell/config execution concerns remain [#245](https://github.com/aictrl-dev/cli/issues/245); use only this owned fixture, not an unknown PR checkout. + +The runner passed local acceptance against the pinned released artifact. GitHub Actions execution must be checked separately; adding the workflow alone is not evidence of a successful CI run. diff --git a/examples/released-smoke/fixture.txt b/examples/released-smoke/fixture.txt new file mode 100644 index 00000000..102b9454 --- /dev/null +++ b/examples/released-smoke/fixture.txt @@ -0,0 +1 @@ +CI_FIXTURE_BYTES_20261021=owned-and-inert diff --git a/examples/released-smoke/smoke.mjs b/examples/released-smoke/smoke.mjs new file mode 100644 index 00000000..a546e11d --- /dev/null +++ b/examples/released-smoke/smoke.mjs @@ -0,0 +1,395 @@ +#!/usr/bin/env node +import assert from "node:assert/strict" +import { createHash } from "node:crypto" +import { spawn } from "node:child_process" +import { createServer } from "node:http" +import { mkdtemp, mkdir, readFile, writeFile, rm, lstat } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { fileURLToPath } from "node:url" + +const recipe = path.dirname(fileURLToPath(import.meta.url)) +const args = process.argv.slice(2) +const options = new Map() +for (let i = 0; i < args.length; i += 2) { + assert(["--binary", "--evidence-dir", "--expected-sha256"].includes(args[i]), `Unknown option ${args[i]}`) + assert(args[i + 1] && !options.has(args[i]), `Missing or duplicate option ${args[i]}`) + options.set(args[i], args[i + 1]) +} +const binary = options.get("--binary") +assert(binary && path.isAbsolute(binary), "Supply --binary with the absolute released native binary path") +assert(process.platform !== "win32", "This recipe requires POSIX process groups") +const evidence = options.get("--evidence-dir") +if (evidence) assert(path.isAbsolute(evidence), "--evidence-dir must be absolute") +const sha256 = (value) => createHash("sha256").update(value).digest("hex") +const marker = "CI_OWNED_FIXTURE_V1" +const callID = "owned-fixture-read-1" +const answer = "CI_FIXTURE_OK" +const groups = new Set() +const requests = [] +const protocolErrors = [] +const runs = [] +const controller = new AbortController() +let root +let server +let overallTimer +let pendingFailure +const report = { + recipe: marker, + expectedVersion: "0.5.4", + sourceCommit: "6c661f4cf3e3d555a0f290bc22e071c6de4bd19b", + startedUTC: new Date().toISOString(), + nodeVersion: process.version, + binary, + status: "running", + runs, + requests, + protocolErrors, +} +const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) +function signalGroup(pid, signal) { + try { + process.kill(-pid, signal) + return true + } catch (error) { + if (error.code === "ESRCH") return false + throw error + } +} +function abort(reason) { + if (controller.signal.aborted) return + controller.abort(new Error(reason)) + for (const pid of groups) signalGroup(pid, "SIGTERM") +} +const interrupted = (signal) => abort(`Runner received ${signal}`) +const onINT = () => interrupted("SIGINT") +const onTERM = () => interrupted("SIGTERM") +process.on("SIGINT", onINT) +process.on("SIGTERM", onTERM) + +async function cleanupGroup(pid) { + if (!signalGroup(pid, "SIGTERM")) return + const until = Date.now() + 5000 + while (Date.now() < until) { + if (!signalGroup(pid, 0)) return + await pause(50) + } + signalGroup(pid, "SIGKILL") + const killedUntil = Date.now() + 1000 + while (Date.now() < killedUntil) { + if (!signalGroup(pid, 0)) return + await pause(50) + } + throw new Error(`Process group ${pid} still exists after SIGKILL`) +} + +async function run(label, argv, env, cwd) { + controller.signal.throwIfAborted() + const record = { label, argv, startedUTC: new Date().toISOString(), stdout: "", stderr: "" } + runs.push(record) + const child = spawn(binary, argv, { cwd, env, detached: true, stdio: ["ignore", "pipe", "pipe"] }) + const pid = child.pid + if (pid) groups.add(pid) + let deadline = false + let killTimer + const term = () => { + if (!pid) return + signalGroup(pid, "SIGTERM") + killTimer ??= setTimeout(() => signalGroup(pid, "SIGKILL"), 5000) + } + const timer = setTimeout(() => { + deadline = true + term() + }, 30_000) + controller.signal.addEventListener("abort", term, { once: true }) + const collect = (name, data) => { + record[name] += data.toString("utf8") + if (Buffer.byteLength(record[name]) > 2 * 1024 * 1024) { + abort(`${label} ${name} exceeds the fixture's 2 MiB capture bound`) + } + } + child.stdout.setEncoding("utf8") + child.stderr.setEncoding("utf8") + child.stdout.on("data", (data) => collect("stdout", data)) + child.stderr.on("data", (data) => collect("stderr", data)) + try { + const result = await new Promise((resolve, reject) => { + child.once("error", reject) + child.once("close", (code, signal) => resolve({ code, signal })) + }) + Object.assign(record, result, { finishedUTC: new Date().toISOString(), deadline }) + if (deadline) throw new Error(`${label} exceeded its external 30-second deadline`) + controller.signal.throwIfAborted() + return record + } finally { + clearTimeout(timer) + clearTimeout(killTimer) + controller.signal.removeEventListener("abort", term) + if (pid) { + await cleanupGroup(pid) + groups.delete(pid) + } + } +} + +function events(run) { + assert(run.stdout.trim(), `${run.label} emitted no stdout`) + return run.stdout + .trim() + .split(/\r?\n/) + .map((line, i) => { + let value + try { + value = JSON.parse(line) + } catch { + throw new Error(`${run.label} stdout line ${i + 1} is not JSON`) + } + assert.equal(value.schemaVersion, "1", `${run.label} event schema version`) + return value + }) +} +function one(list, type) { + const found = list.filter((event) => event.type === type) + assert.equal(found.length, 1, `Expected exactly one ${type}`) + return found[0] +} +function stream(response, call, text) { + const base = { id: `fixture-${requests.length}`, object: "chat.completion.chunk", created: 1, model: "fixture" } + const delta = call + ? { + tool_calls: [ + { index: 0, id: callID, type: "function", function: { name: "read", arguments: JSON.stringify(call) } }, + ], + } + : { content: text } + const chunks = [ + { ...base, choices: [{ index: 0, delta: { role: "assistant", ...delta }, finish_reason: null }] }, + { + ...base, + choices: [{ index: 0, delta: {}, finish_reason: call ? "tool_calls" : "stop" }], + usage: { prompt_tokens: 100, completion_tokens: 10, total_tokens: 110 }, + }, + ] + response.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }) + response.end(chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n") +} + +try { + report.sourceHashes = {} + for (const name of ["smoke.mjs", "fixture.txt", "README.md", "workflow.example.yml"]) + report.sourceHashes[name] = sha256(await readFile(path.join(recipe, name))) + report.binarySha256 = sha256(await readFile(binary)) + if (options.has("--expected-sha256")) + assert.equal(report.binarySha256, options.get("--expected-sha256"), "Native artifact SHA-256") + const fixture = await readFile(path.join(recipe, "fixture.txt"), "utf8") + assert.equal(fixture, "CI_FIXTURE_BYTES_20261021=owned-and-inert\n", "Owned fixture bytes") + root = await mkdtemp(path.join(tmpdir(), "aictrl-owned-ci-")) + const workspace = path.join(root, "workspace") + const dirs = Object.fromEntries( + ["home", "data", "config", "cache", "state"].map((name) => [name, path.join(root, name)]), + ) + await Promise.all([workspace, ...Object.values(dirs)].map((dir) => mkdir(dir))) + // v0.5.4 still discovers nested instruction files despite the config flag. + for (let directory = workspace; ; directory = path.dirname(directory)) { + for (const name of ["AGENTS.md", ".opencode/AGENTS.md"]) { + const instruction = path.join(directory, name) + const absent = await lstat(instruction).then( + () => false, + (error) => { + if (error.code === "ENOENT") return true + throw error + }, + ) + assert(absent, `Unexpected ancestor instruction file: ${instruction}`) + } + if (directory === path.dirname(directory)) break + } + const file = path.join(workspace, "fixture.txt") + await writeFile(file, fixture) + await writeFile(path.join(workspace, "invalid.schema.json"), "[]\n") + overallTimer = setTimeout(() => abort("Recipe exceeded its external 60-second deadline"), 60_000) + server = createServer(async (request, response) => { + const record = { method: request.method, path: request.url, atUTC: new Date().toISOString() } + requests.push(record) + try { + assert.equal(request.method, "POST", "Only model POST requests are expected") + assert.equal(request.url, "/v1/chat/completions", "Only the loopback chat endpoint is expected") + let raw = "" + for await (const part of request) { + raw += part.toString("utf8") + assert(Buffer.byteLength(raw) <= 2 * 1024 * 1024, "Request exceeds 2 MiB") + } + const body = JSON.parse(raw) + record.body = body + assert.equal(body.model, "fixture") + assert.equal(body.stream, true) + assert(Array.isArray(body.messages)) + assert( + body.messages.some((message) => message.role === "user" && JSON.stringify(message.content).includes(marker)), + "Unexpected auxiliary/model request", + ) + assert.deepEqual(body.tools.map((tool) => tool.function.name).sort(), ["read"], "Wire tool allowlist") + if (requests.length === 1) { + assert(!body.messages.some((message) => message.role === "tool")) + record.response = "read" + stream(response, { filePath: file }) + return + } + assert.equal(requests.length, 2, "Only two model turns are scripted") + const result = body.messages.find((message) => message.role === "tool" && message.tool_call_id === callID) + assert(result, "Second model request lacks the real read result") + assert.equal(typeof result.content, "string") + assert(result.content.includes(fixture.trimEnd()), "Fixture bytes missing from actual tool output") + record.verifiedReadResult = true + record.response = answer + stream(response, undefined, answer) + } catch (error) { + record.error = error.message + protocolErrors.push(error.message) + console.error(`[fixture transport] ${error.message}`) + if (!response.headersSent) response.writeHead(400, { "content-type": "application/json" }) + response.end(JSON.stringify({ error: { message: error.message, type: "fixture_protocol_error" } })) + } + }) + await new Promise((resolve, reject) => { + server.once("error", reject) + server.listen(0, "127.0.0.1", resolve) + }) + const config = { + enabled_providers: ["fixture"], + provider: { + fixture: { + npm: "@ai-sdk/openai-compatible", + options: { apiKey: "fixture-not-a-credential", baseURL: `http://127.0.0.1:${server.address().port}/v1` }, + models: { fixture: { name: "fixture", tool_call: true, limit: { context: 32000, output: 1000 } } }, + }, + }, + permission: { "*": "deny", read: "allow" }, + skills: { allow: [] }, + agent: { build: { steps: 3 } }, + } + const env = { + PATH: "/usr/local/bin:/usr/bin:/bin", + HOME: dirs.home, + XDG_DATA_HOME: dirs.data, + XDG_CONFIG_HOME: dirs.config, + XDG_CACHE_HOME: dirs.cache, + XDG_STATE_HOME: dirs.state, + AICTRL_CONFIG_CONTENT: JSON.stringify(config), + AICTRL_DISABLE_PROJECT_CONFIG: "true", + AICTRL_MODEL_STREAM_IDLE_TIMEOUT_MS: "5000", + NO_COLOR: "1", + TERM: "dumb", + } + report.childEnvKeys = Object.keys(env).sort() + report.config = config + const version = await run("version", ["--version"], env, workspace) + assert.equal(version.code, 0) + assert.equal(version.stdout.trim(), "0.5.4", "Released native version") + assert.equal(requests.length, 0, "Version command must not call the model") + const prompt = `${marker}: Read fixture.txt, then report the fixture result.` + const positive = await run( + "read-success", + ["run", "--format", "json", "--model", "fixture/fixture", "--title", "Owned fixture CI smoke", prompt], + env, + workspace, + ) + assert.equal(positive.code, 0, `Positive command failed: ${positive.stderr}`) + assert.equal(positive.signal, null) + assert.deepEqual(protocolErrors, []) + assert.equal(requests.length, 2) + assert.equal(requests[1].verifiedReadResult, true) + const output = events(positive) + one(output, "invocation_start") + one(output, "session_start") + assert.equal(one(output, "invocation_complete").status, "completed") + assert.equal(one(output, "session_complete").error, null) + assert( + !output.some((event) => ["error", "session_error", "invocation_error", "permission_rejected"].includes(event.type)), + ) + const read = one(output, "tool_use").part + assert.equal(read.tool, "read") + assert.equal(read.state.status, "completed") + assert(read.state.output.includes(fixture.trimEnd())) + assert.equal( + output + .filter((event) => event.type === "text") + .map((event) => event.part.text) + .join("\n") + .trim(), + answer, + ) + const before = requests.length + const negative = await run( + "invalid-schema", + ["run", "--format", "json", "--model", "fixture/fixture", "--output-schema", "invalid.schema.json", prompt], + env, + workspace, + ) + assert.equal(negative.code, 2) + assert.equal(negative.signal, null) + const rejected = events(negative) + assert.equal(one(rejected, "invocation_error").code, "OUTPUT_SCHEMA_CONFIG") + assert.equal(one(rejected, "invocation_complete").status, "error") + assert(!rejected.some((event) => event.type === "session_start")) + assert.equal(requests.length, before, "Invalid schema must not issue any model request") + report.assertions = { + version: true, + artifactHashRecorded: true, + twoWireRequests: true, + realReadResult: true, + readOnlyWireCatalog: true, + validNDJSON: true, + successExitAndTerminalEvents: true, + invalidSchemaExit2: true, + invalidSchemaNoModelCalls: true, + } + report.status = "pass" +} catch (error) { + pendingFailure = error + report.status = "fail" + report.error = { message: error.message, stack: error.stack } +} finally { + clearTimeout(overallTimer) + if (server?.listening) { + server.closeAllConnections() + await new Promise((resolve) => server.close(resolve)) + } + report.cleanupErrors = [] + for (const pid of groups) { + try { + await cleanupGroup(pid) + groups.delete(pid) + } catch (error) { + pendingFailure ??= error + report.status = "fail" + report.cleanupErrors.push(error.message) + } + } + let removed = !root + try { + if (root) await rm(root, { recursive: true, force: true }) + removed = true + } catch (error) { + pendingFailure ??= error + report.status = "fail" + report.cleanupErrors.push(error.message) + } + report.cleanup = { processGroupsRemaining: groups.size, temporaryRootRemoved: removed } + process.removeListener("SIGINT", onINT) + process.removeListener("SIGTERM", onTERM) + report.finishedUTC = new Date().toISOString() + if (evidence) { + await mkdir(evidence, { recursive: true }) + await writeFile(path.join(evidence, "report.json"), JSON.stringify(report, null, 2) + "\n") + for (const record of runs) { + await writeFile(path.join(evidence, `${record.label}.stdout`), record.stdout) + await writeFile(path.join(evidence, `${record.label}.stderr`), record.stderr) + } + } + console.log(JSON.stringify(report, null, 2)) +} +if (pendingFailure) { + console.error(`[owned CLI smoke] ${pendingFailure.message}`) + process.exitCode = 1 +} diff --git a/examples/released-smoke/workflow.example.yml b/examples/released-smoke/workflow.example.yml new file mode 100644 index 00000000..2edfbdd1 --- /dev/null +++ b/examples/released-smoke/workflow.example.yml @@ -0,0 +1,44 @@ +name: Released aictrl consumer-contract smoke + +# Inactive companion example: copy to .github/workflows on a trusted branch. +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + smoke: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + env: + RECIPE_DIR: examples/released-smoke + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 20.20.2 + - name: Install pinned public release without package scripts + shell: bash + run: | + npm install --prefix "$RUNNER_TEMP/aictrl-owned-cli" \ + --ignore-scripts --no-audit --no-fund \ + @aictrl/cli@0.5.4 @aictrl/cli-linux-x64-baseline@0.5.4 + node -e 'const fs=require("node:fs"); const root=process.env.RUNNER_TEMP+"/aictrl-owned-cli/node_modules/@aictrl/"; for(const name of ["cli","cli-linux-x64-baseline"]){const p=JSON.parse(fs.readFileSync(root+name+"/package.json","utf8")); if(p.version!=="0.5.4") throw new Error("Unexpected package version"); console.log(p.name+"@"+p.version)}' + - name: Run owned fixture against released native artifact + shell: bash + run: | + node "$RECIPE_DIR/smoke.mjs" \ + --binary "$RUNNER_TEMP/aictrl-owned-cli/node_modules/@aictrl/cli-linux-x64-baseline/bin/aictrl" \ + --expected-sha256 f12a54259ae0d77d85fd6d0cfefe1dcc742b95a617ca71e48f4fdfb2575182ee \ + --evidence-dir "$RUNNER_TEMP/aictrl-owned-evidence" + - name: Retain only inert fixture evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: aictrl-owned-fixture-evidence + path: ${{ runner.temp }}/aictrl-owned-evidence + if-no-files-found: error + retention-days: 7 diff --git a/packages/cli/README.md b/packages/cli/README.md index 812bb0f9..ea88b557 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -102,7 +102,7 @@ Aictrl is built to be used in non-interactive environments. ### Stdin Piping ```bash -cat diff.txt | aictrl run "review this code change" +cat diff.txt | aictrl run "review this code change" - ``` ### JSON Events @@ -275,6 +275,25 @@ If your agents should commit or open PRs, supply your own procedure through a skill, `AGENTS.md` or the `instructions` config, and run the CLI in an isolated environment with credentials scoped to what the task needs. +## Project instruction isolation + +Set `AICTRL_DISABLE_PROJECT_CONFIG=true` to disable automatic project instruction +discovery as well as project configuration. This excludes root `AGENTS.md`, +`.opencode/AGENTS.md`, and nested `AGENTS.md` files that would otherwise be +attached when a tool reads a project file. + +Trusted global instruction files (`~/.config/aictrl/AGENTS.md` and +`AICTRL_CONFIG_DIR/AGENTS.md`) and explicit `instructions` entries in caller +configuration remain available. Relative caller instructions resolve from +`AICTRL_CONFIG_DIR` when project configuration is disabled; without that directory, +relative entries are skipped. Absolute paths and URLs remain explicit inputs. + +This controls automatic instruction discovery. Project files and tool output +remain untrusted task data, so CI jobs still need scoped credentials, configured +permissions, and process isolation. Keep any existing instruction-file deletion +workaround until the installed release includes this behavior and has been +verified. + ## Local Development ```bash diff --git a/packages/cli/src/cli/cmd/generate.ts b/packages/cli/src/cli/cmd/generate.ts index 9a9063ab..1177d1de 100644 --- a/packages/cli/src/cli/cmd/generate.ts +++ b/packages/cli/src/cli/cmd/generate.ts @@ -3,6 +3,7 @@ import z from "zod" import { Config } from "@/config/config" import { MessageV2 } from "@/session/message-v2" import { Session } from "@/session" +import { SessionPrompt } from "@/session/prompt" import { ProviderTermination } from "@/provider/termination" export const GenerateCommand = { @@ -10,6 +11,8 @@ export const GenerateCommand = { handler: async () => { // Keep OpenAPI component identities separate from legacy Zod metadata. const registry = z.registry<{ id: string }>() + registry.add(SessionPrompt.CommandArguments, { id: "CommandArguments" }) + registry.add(SessionPrompt.CommandInput, { id: "CommandInput" }) registry.add(MessageV2.StepFinishPart, { id: "StepFinishPart" }) registry.add(ProviderTermination.Info, { id: "ProviderTermination" }) registry.add(MessageV2.Format, { id: "OutputFormat" }) diff --git a/packages/cli/src/cli/cmd/run.input.ts b/packages/cli/src/cli/cmd/run.input.ts new file mode 100644 index 00000000..e6077644 --- /dev/null +++ b/packages/cli/src/cli/cmd/run.input.ts @@ -0,0 +1,11 @@ +/** Normalize parser-owned message operands before titling or reading stdin. */ +export function runInput(operands: readonly string[], tty: boolean | undefined) { + const words = operands.filter((word) => word !== "-") + return { + argv: words, + prompt: words.join(" "), + message: words.join(" "), + // An explicit empty operand is still a supplied message; do not wait for stdin. + stdin: operands.includes("-") || (operands.length === 0 && !tty), + } +} diff --git a/packages/cli/src/cli/cmd/run.ts b/packages/cli/src/cli/cmd/run.ts index 319d7b85..cb28761b 100644 --- a/packages/cli/src/cli/cmd/run.ts +++ b/packages/cli/src/cli/cmd/run.ts @@ -49,6 +49,7 @@ import { Shutdown } from "../shutdown" import { Stdout } from "../stdout" import { attempt, signals, type Signals } from "../signals" import { createRunInvocation } from "./run.invocation" +import { runInput } from "./run.input" type ToolProps = { input: Tool.InferParameters @@ -314,9 +315,9 @@ function truncatedTitle(text: string) { */ export function sessionTitle(input: { title?: string - /** The positional words as typed, before they are re-quoted for the model. */ + /** The normalized positional text used to derive the title. */ prompt: string - /** The message sent to the model: the re-quoted prompt plus any stdin. */ + /** The message sent to the model: normalized prompt text plus any selected stdin. */ message: string command?: string }) { @@ -364,17 +365,18 @@ async function sessionPermissions(sessionID: string, agent: string | undefined, } export const RunCommand = cmd({ - command: "run [message..]", + command: "run", describe: "run aictrl with a message", builder: (yargs: Argv) => { return ( yargs - .positional("message", { - describe: "message to send", - type: "string", - array: true, - default: [], - }) + // A variadic positional is reparsed by yargs and loses bare "-" operands. + // Keep the parser's operands in this leaf while still rejecting unknown options. + .usage("$0 run [message..]") + .strict(false) + .strictOptions() + .strictCommands(false) + .parserConfiguration({ "populate--": true, "parse-positional-numbers": false }) .option("tools", { type: "string", nargs: 1, @@ -384,6 +386,7 @@ export const RunCommand = cmd({ .option("command", { describe: "the command to run, use message for args", type: "string", + requiresArg: true, }) .option("continue", { alias: ["c"], @@ -394,6 +397,7 @@ export const RunCommand = cmd({ alias: ["s"], describe: "session id to continue", type: "string", + requiresArg: true, }) .option("fork", { describe: "fork the session before continuing (requires --continue or --session)", @@ -406,45 +410,58 @@ export const RunCommand = cmd({ }) .option("model", { type: "string", + requiresArg: true, alias: ["m"], describe: "model to use in the format of provider/model", }) .option("agent", { type: "string", + requiresArg: true, describe: "agent to use", }) .option("format", { type: "string", + requiresArg: true, choices: ["default", "json"], default: "default", describe: "format: default (formatted) or json (raw JSON events)", }) .option("output-schema", { type: "string", + requiresArg: true, describe: "JSON Schema file for the final result (object root); the agent writes the result to a JSON file in the working directory and passes its path to StructuredOutput", }) - .option("output-schema-retries", { type: "number", describe: "additional corrective turns (0-10, default: 2)" }) + .option("output-schema-retries", { + type: "number", + requiresArg: true, + describe: "additional corrective turns (0-10, default: 2)", + }) .option("output-result", { type: "string", + requiresArg: true, describe: "atomically write only the validated JSON result to this file", }) .option("file", { alias: ["f"], type: "string", + requiresArg: true, array: true, describe: "file(s) to attach to message", }) .option("title", { type: "string", + requiresArg: true, describe: "title for the session (defaults to the truncated prompt, or / for command runs)", }) .option("attach", { type: "string", + requiresArg: true, describe: "attach to a running aictrl server (e.g., http://localhost:4096)", }) .option("dir", { type: "string", + requiresArg: true, describe: "directory to run in, path on remote server if attaching", }) .option("port", { @@ -453,6 +470,7 @@ export const RunCommand = cmd({ }) .option("variant", { type: "string", + requiresArg: true, describe: "model variant (provider-specific reasoning effort, e.g., high, max, minimal)", }) .option("thinking", { @@ -503,9 +521,9 @@ export const RunCommand = cmd({ await fail("--share was removed: session sharing is no longer supported", "INVOCATION_INVALID_ARGUMENTS") } - let message = [...args.message, ...(args["--"] || [])] - .map((arg) => (arg.includes(" ") ? `"${arg.replace(/"/g, '\\"')}"` : arg)) - .join(" ") + const normalized = runInput([...args._.slice(1), ...(args["--"] ?? [])].map(String), process.stdin.isTTY) + let message = normalized.message + let stdin: string | undefined const directory = await (async () => { if (!args.dir) return undefined @@ -553,9 +571,10 @@ export const RunCommand = cmd({ } } - if (!process.stdin.isTTY) { + if (normalized.stdin) { invocation.phase("stdin") - message += "\n" + (await invocation.guard(() => Bun.stdin.text())) + stdin = await invocation.guard(() => Bun.stdin.text()) + message += "\n" + stdin invocation.phase("validation") } @@ -601,8 +620,7 @@ export const RunCommand = cmd({ if (baseID) return baseID - const prompt = [...args.message, ...(args["--"] || [])].join(" ") - const title = sessionTitle({ title: args.title, prompt, message, command: args.command }) + const title = sessionTitle({ title: args.title, prompt: normalized.prompt, message, command: args.command }) const result = await sdk.session.create(input(undefined, { title })) return result.data?.id } @@ -1159,7 +1177,7 @@ export const RunCommand = cmd({ toolAllowlist, model: args.model, command: args.command, - arguments: message, + arguments: { argv: normalized.argv, ...(stdin !== undefined ? { stdin } : {}) }, variant: args.variant, ...(schema ? { format: schema.format } : {}), } diff --git a/packages/cli/src/command/template.ts b/packages/cli/src/command/template.ts new file mode 100644 index 00000000..08737847 --- /dev/null +++ b/packages/cli/src/command/template.ts @@ -0,0 +1,57 @@ +import { $ } from "bun" +import { ConfigMarkdown } from "../config/markdown" + +/** Render caller data once; only shell spans in the original command template execute. */ +export async function renderCommandTemplate(template: string, text: string, argv: readonly string[]) { + const placeholders = [...template.matchAll(/\$(\d+)/g)] + const last = placeholders.reduce((maximum, match) => Math.max(maximum, Number(match[1])), 0) + const numbered = (index: string) => { + const position = Number(index) + const offset = position - 1 + if (offset >= argv.length) return "" + if (position === last) return argv.slice(offset).join(" ") + return String(argv[offset]) + } + const prose = (source: string) => + source.replaceAll(/\$(ARGUMENTS|\d+)/g, (_, index) => (index === "ARGUMENTS" ? text : numbered(index))) + + // Capture structure before inserting argv, stdin, fallback text or shell output. + const spans = ConfigMarkdown.shell(template) + const segments: string[] = [] + const scripts: { strings: TemplateStringsArray; values: (string | string[])[] }[] = [] + let cursor = 0 + for (const span of spans) { + segments.push(prose(template.slice(cursor, span.index))) + cursor = span.index + span[0].length + const fragments: string[] = [] + const values: (string | string[])[] = [] + let start = 0 + for (const match of span[1].matchAll(/\$(ARGUMENTS|\d+)/g)) { + fragments.push(span[1].slice(start, match.index)) + values.push(match[1] === "ARGUMENTS" ? [...argv] : numbered(match[1])) + start = match.index + match[0].length + } + fragments.push(span[1].slice(start)) + const strings: TemplateStringsArray = Object.freeze( + Object.assign(fragments, { raw: Object.freeze([...fragments]) }), + ) + scripts.push({ strings, values }) + } + segments.push(prose(template.slice(cursor))) + + const results = await Promise.all( + scripts.map(async (script) => { + try { + return await $(script.strings, ...script.values) + .quiet() + .nothrow() + .text() + } catch (error) { + return `Error executing command: ${error instanceof Error ? error.message : String(error)}` + } + }), + ) + const rendered = segments.map((segment, index) => segment + (index < results.length ? results[index] : "")).join("") + const fallback = placeholders.length === 0 && !template.includes("$ARGUMENTS") && text.trim() ? "\n\n" + text : "" + return (rendered + fallback).trim() +} diff --git a/packages/cli/src/session/instruction.ts b/packages/cli/src/session/instruction.ts index e1708489..3c05d773 100644 --- a/packages/cli/src/session/instruction.ts +++ b/packages/cli/src/session/instruction.ts @@ -19,7 +19,9 @@ function globalFiles() { files.push(path.join(Flag.AICTRL_CONFIG_DIR, "AGENTS.md")) } files.push(path.join(Global.Path.config, "AGENTS.md")) - files.push(path.join(Instance.directory, ".opencode", "AGENTS.md")) + if (!Flag.AICTRL_DISABLE_PROJECT_CONFIG) { + files.push(path.join(Instance.directory, ".opencode", "AGENTS.md")) + } return files } @@ -181,6 +183,7 @@ export namespace InstructionPrompt { } export async function resolve(messages: MessageV2.WithParts[], filepath: string, messageID: string) { + if (Flag.AICTRL_DISABLE_PROJECT_CONFIG) return [] const system = await systemPaths() const already = loaded(messages) const results: { filepath: string; content: string }[] = [] diff --git a/packages/cli/src/session/prompt.ts b/packages/cli/src/session/prompt.ts index 299b822d..f0343a6c 100644 --- a/packages/cli/src/session/prompt.ts +++ b/packages/cli/src/session/prompt.ts @@ -31,7 +31,8 @@ import { ReadTool } from "../tool/read" import { FileTime } from "../file/time" import { ulid } from "ulid" import { Command } from "../command" -import { $, fileURLToPath, pathToFileURL } from "bun" +import { renderCommandTemplate } from "../command/template" +import { fileURLToPath, pathToFileURL } from "bun" import { ConfigMarkdown } from "../config/markdown" import { NamedError } from "@aictrl/util/error" import { fn } from "@/util/fn" @@ -1619,12 +1620,21 @@ export namespace SessionPrompt { return { message: { info, parts }, catalog } } + export const CommandArguments = z.union([ + z.string(), + z.object({ + argv: z.array(z.string()), + stdin: z.string().optional(), + }), + ]) + export type CommandArguments = z.infer + export const CommandInput = z.object({ messageID: Identifier.schema("message").optional(), sessionID: Identifier.schema("session"), agent: z.string().optional(), model: z.string().optional(), - arguments: z.string(), + arguments: CommandArguments, command: z.string(), toolAllowlist: ToolSelection.Names.optional(), format: MessageV2.Format.optional(), @@ -1643,10 +1653,8 @@ export namespace SessionPrompt { .optional(), }) export type CommandInput = z.infer - const bashRegex = /!`([^`]+)`/g // Match [Image N] as single token, quoted strings, or non-space sequences const argsRegex = /(?:\[Image\s+\d+\]|"[^"]*"|'[^']*'|[^\s"']+)/gi - const placeholderRegex = /\$(\d+)/g const quoteTrimRegex = /^["']|["']$/g /** * Regular expression to match @ file references in text @@ -1665,50 +1673,15 @@ export namespace SessionPrompt { const command = await Command.get(input.command) const agentName = command.agent ?? input.agent ?? (await Agent.defaultAgent()) - const raw = input.arguments.match(argsRegex) ?? [] - const args = raw.map((arg) => arg.replace(quoteTrimRegex, "")) - - const templateCommand = await command.template - - const placeholders = templateCommand.match(placeholderRegex) ?? [] - let last = 0 - for (const item of placeholders) { - const value = Number(item.slice(1)) - if (value > last) last = value - } - - // Let the final placeholder swallow any extra arguments so prompts read naturally - const withArgs = templateCommand.replaceAll(placeholderRegex, (_, index) => { - const position = Number(index) - const argIndex = position - 1 - if (argIndex >= args.length) return "" - if (position === last) return args.slice(argIndex).join(" ") - return args[argIndex] - }) - const usesArgumentsPlaceholder = templateCommand.includes("$ARGUMENTS") - let template = withArgs.replaceAll("$ARGUMENTS", input.arguments) + const value = input.arguments + const text = + typeof value === "string" ? value : value.argv.join(" ") + (value.stdin !== undefined ? "\n" + value.stdin : "") + const args = + typeof value === "string" + ? (value.match(argsRegex) ?? []).map((arg) => arg.replace(quoteTrimRegex, "")) + : [...value.argv, ...(value.stdin !== undefined ? [value.stdin] : [])] - // If command doesn't explicitly handle arguments (no $N or $ARGUMENTS placeholders) - // but user provided arguments, append them to the template - if (placeholders.length === 0 && !usesArgumentsPlaceholder && input.arguments.trim()) { - template = template + "\n\n" + input.arguments - } - - const shell = ConfigMarkdown.shell(template) - if (shell.length > 0) { - const results = await Promise.all( - shell.map(async ([, cmd]) => { - try { - return await $`${{ raw: cmd }}`.quiet().nothrow().text() - } catch (error) { - return `Error executing command: ${error instanceof Error ? error.message : String(error)}` - } - }), - ) - let index = 0 - template = template.replace(bashRegex, () => results[index++]) - } - template = template.trim() + const template = await renderCommandTemplate(await command.template, text, args) const taskModel = await (async () => { if (command.model) { @@ -1770,7 +1743,7 @@ export namespace SessionPrompt { { command: input.command, sessionID: input.sessionID, - arguments: input.arguments, + arguments: text, }, { parts }, ) @@ -1789,7 +1762,7 @@ export namespace SessionPrompt { Bus.publish(Command.Event.Executed, { name: input.command, sessionID: input.sessionID, - arguments: input.arguments, + arguments: text, messageID: result.info.id, }) diff --git a/packages/cli/test/cli/fixture/command-arguments-legacy.ts b/packages/cli/test/cli/fixture/command-arguments-legacy.ts new file mode 100644 index 00000000..92e1e41f --- /dev/null +++ b/packages/cli/test/cli/fixture/command-arguments-legacy.ts @@ -0,0 +1,18 @@ +import { bootstrap } from "../../../src/cli/bootstrap" +import { Session } from "../../../src/session" +import { SessionPrompt } from "../../../src/session/prompt" +import { Shutdown } from "../../../src/cli/shutdown" + +const data = await Bun.file(process.env.COMMAND_ARGUMENTS_DATA!).json() +if (typeof data.arguments !== "string") throw new Error("Explicit legacy string arguments required") +await bootstrap(process.cwd(), async () => { + const session = await Session.createNext({ directory: process.cwd(), title: "Legacy string control" }) + await SessionPrompt.command({ + sessionID: session.id, + command: "argumentprobe", + arguments: data.arguments, + model: "fixture/fixture", + }) + console.log(JSON.stringify({ completed: true, sessionID: session.id })) +}) +await Shutdown.flush() diff --git a/packages/cli/test/cli/fixture/command-template-capture.ts b/packages/cli/test/cli/fixture/command-template-capture.ts new file mode 100644 index 00000000..f60801b9 --- /dev/null +++ b/packages/cli/test/cli/fixture/command-template-capture.ts @@ -0,0 +1,255 @@ +import { expect } from "bun:test" +import { spawn } from "node:child_process" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" + +const template = "FIRST<$1>\nSECOND<$2>\nREST<$3>" +const guard = path.join(import.meta.dir, "stdin-guard.ts") +const pause = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +export type Input = ( + | { name: string; entry: "index" | "headless"; operands: string[]; stdin?: string } + | { name: string; arguments: string | { argv: string[]; stdin?: string } } +) & { template?: string; deadlineMs?: number; ownedRedirect?: boolean } +export async function capture(input: Input) { + const cwd = await fs.mkdtemp(path.join(os.tmpdir(), "aictrl-command-arguments-")) + const ownedFile = input.ownedRedirect ? path.join(cwd, "target; /usr/bin/printf INERT_REDIRECT") : undefined + if (ownedFile) await fs.mkdir(path.dirname(ownedFile), { recursive: true }) + const requests: Record[] = [] + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + if (new URL(request.url).pathname !== "/v1/chat/completions") throw new Error("Unexpected provider path") + requests.push((await request.json()) as Record) + const chunks = [ + { choices: [{ index: 0, delta: { role: "assistant", content: "Done" }, finish_reason: null }] }, + { + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }, + ] + return new Response(chunks.map((value) => `data: ${JSON.stringify(value)}\n\n`).join("") + "data: [DONE]\n\n", { + headers: { "content-type": "text/event-stream" }, + }) + }, + }) + await fs.writeFile( + path.join(cwd, "aictrl.json"), + JSON.stringify({ + plugin: [new URL("./command-template-plugin.ts", import.meta.url).href], + enabled_providers: ["fixture"], + provider: { + fixture: { + npm: "@ai-sdk/openai-compatible", + options: { apiKey: "fixture", baseURL: server.url.origin + "/v1" }, + models: { fixture: { name: "fixture", tool_call: true, limit: { context: 100000, output: 1000 } } }, + }, + }, + agent: { title: { disable: true } }, + command: { argumentprobe: { template: input.template ?? template } }, + }), + ) + const data = path.join(cwd, "arguments.json") + await fs.writeFile(data, JSON.stringify("arguments" in input ? { arguments: input.arguments } : {})) + await fs.writeFile(path.join(cwd, "observations.jsonl"), "") + const ledger = path.join(cwd, "network.jsonl") + await fs.writeFile(ledger, "") + const env = { + PATH: [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter), + HOME: path.join(cwd, "home"), + AICTRL_TEST_HOME: path.join(cwd, "home"), + AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(cwd, "managed"), + XDG_DATA_HOME: path.join(cwd, "data"), + XDG_CONFIG_HOME: path.join(cwd, "config"), + XDG_CACHE_HOME: path.join(cwd, "cache"), + XDG_STATE_HOME: path.join(cwd, "state"), + BUN_RUNTIME_TRANSPILER_CACHE_PATH: path.join(cwd, "bun-cache"), + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + TZ: "UTC", + AICTRL_DISABLE_DEFAULT_PLUGINS: "true", + AICTRL_DISABLE_AUTOCOMPACT: "true", + STDIN_FIXTURE_ORIGIN: server.url.origin, + STDIN_FIXTURE_LEDGER: ledger, + COMMAND_ARGUMENTS_DATA: data, + COMMAND_TEMPLATE_OBSERVATIONS: path.join(cwd, "observations.jsonl"), + } + const source = + "arguments" in input + ? path.join(import.meta.dir, "command-template-handler.ts") + : path.resolve(import.meta.dir, `../../../src/${input.entry}.ts`) + const command = [ + "run", + "--no-install", + "--no-env-file", + "--preload", + guard, + "--conditions=browser", + source, + ...("arguments" in input + ? [] + : [ + "run", + "--format", + "json", + "--model", + "fixture/fixture", + "--command", + "argumentprobe", + ...input.operands.map((value) => (ownedFile && value === "OWNED_REDIRECT_TARGET" ? ownedFile : value)), + ]), + ] + const child = spawn(process.execPath, command, { + cwd, + env, + detached: true, + stdio: ["stdin" in input ? "pipe" : "ignore", "pipe", "pipe"], + }) + let stdout = "", + stderr = "", + deadline = false + const exists = () => { + try { + process.kill(-child.pid!, 0) + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false + throw error + } + } + let staged = false + let firstBytesAt: number | undefined + let eofAt: number | undefined + let requestsBeforeEOF: number | undefined + const started = performance.now() + const stages: ReturnType[] = [] + child.stdin?.on("error", (error: NodeJS.ErrnoException) => { + if (error.code !== "EPIPE") stderr += String(error) + }) + child.stdout!.on("data", (value) => { + stdout += value.toString() + if ("stdin" in input && !staged && stdout.includes('"type":"invocation_start"')) { + staged = true + stages.push( + setTimeout(() => { + firstBytesAt = performance.now() - started + child.stdin!.write(input.stdin ?? "") + stages.push( + setTimeout(() => { + eofAt = performance.now() - started + requestsBeforeEOF = requests.length + child.stdin!.end() + }, 100), + ) + }, 100), + ) + } + }) + child.stderr!.on("data", (value) => { + stderr += value.toString() + }) + const timer = setTimeout(() => { + deadline = true + if (exists()) process.kill(-child.pid!, "SIGTERM") + }, input.deadlineMs ?? 12000) + const escalation = setTimeout( + () => { + if (deadline && exists()) process.kill(-child.pid!, "SIGKILL") + }, + (input.deadlineMs ?? 12000) + 200, + ) + let code: number | null = null + try { + code = await new Promise((resolve, reject) => { + child.once("error", reject) + child.once("close", (value) => resolve(value)) + }) + } finally { + for (const stage of stages) clearTimeout(stage) + child.stdin?.destroy() + clearTimeout(timer) + clearTimeout(escalation) + if (exists()) { + process.kill(-child.pid!, "SIGTERM") + await pause(150) + if (exists()) process.kill(-child.pid!, "SIGKILL") + } + server.stop(true) + } + const network = (await fs.readFile(ledger, "utf8")) + .split("\n") + .filter(Boolean) + .map((value) => JSON.parse(value)) + const events = stdout + .split("\n") + .filter((value) => value.startsWith("{")) + .map((value) => JSON.parse(value)) + let redirected: { path: string; text?: string; removed: boolean } | undefined + if (ownedFile) { + const text = await fs.readFile(ownedFile, "utf8").catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined + throw error + }) + if (text !== undefined) await fs.unlink(ownedFile) + const removed = await fs.stat(ownedFile).then( + () => false, + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return true + throw error + }, + ) + redirected = { path: ownedFile, text, removed } + } + const result = { + input, + redirected, + template: input.template ?? template, + firstBytesAt, + eofAt, + requestsBeforeEOF, + command, + cwd, + code, + deadline, + groupGone: !exists(), + stdout, + stderr, + requests, + events, + network, + observations: (await fs.readFile(path.join(cwd, "observations.jsonl"), "utf8")) + .split("\n") + .filter(Boolean) + .map((row) => JSON.parse(row)), + } + if (process.env.COMMAND_ARGUMENTS_EVIDENCE) { + await fs.mkdir(process.env.COMMAND_ARGUMENTS_EVIDENCE, { recursive: true }) + const prefix = "entry" in input ? input.entry : "legacy" + await fs.writeFile( + path.join(process.env.COMMAND_ARGUMENTS_EVIDENCE, `${prefix}-${input.name}.json`), + JSON.stringify(result, null, 2) + "\n", + ) + } + if (deadline) throw new Error("Command argument fixture deadline fired: " + JSON.stringify(result)) + if (!result.groupGone) throw new Error("Command argument fixture process group remains") + if (network.some((row) => row.allowed === false)) throw new Error("Command argument fixture blocked external attempt") + expect(code, stderr + stdout).toBe(0) + expect(requests).toHaveLength(1) + return result +} +export function userText(body: Record) { + const messages = body.messages as { role: string; content: string | { type: string; text?: string }[] }[] + return messages + .filter((message) => message.role === "user") + .map((message) => + typeof message.content === "string" + ? message.content + : message.content + .filter((part) => part.type === "text") + .map((part) => part.text ?? "") + .join(""), + ) + .join("\n") +} diff --git a/packages/cli/test/cli/fixture/command-template-handler.ts b/packages/cli/test/cli/fixture/command-template-handler.ts new file mode 100644 index 00000000..e4129ff4 --- /dev/null +++ b/packages/cli/test/cli/fixture/command-template-handler.ts @@ -0,0 +1,18 @@ +import { bootstrap } from "../../../src/cli/bootstrap" +import { Session } from "../../../src/session" +import { SessionPrompt } from "../../../src/session/prompt" +import { Shutdown } from "../../../src/cli/shutdown" + +const data = await Bun.file(process.env.COMMAND_ARGUMENTS_DATA!).json() +const args = SessionPrompt.CommandArguments.parse(data.arguments) +await bootstrap(process.cwd(), async () => { + const session = await Session.createNext({ directory: process.cwd(), title: "Projection control" }) + await SessionPrompt.command({ + sessionID: session.id, + command: "argumentprobe", + arguments: args, + model: "fixture/fixture", + }) + console.log(JSON.stringify({ completed: true, sessionID: session.id })) +}) +await Shutdown.flush() diff --git a/packages/cli/test/cli/fixture/command-template-plugin.ts b/packages/cli/test/cli/fixture/command-template-plugin.ts new file mode 100644 index 00000000..6c82e20c --- /dev/null +++ b/packages/cli/test/cli/fixture/command-template-plugin.ts @@ -0,0 +1,13 @@ +import { appendFileSync } from "node:fs" +import type { Plugin } from "@aictrl/plugin" + +const record = (data: unknown) => + appendFileSync(process.env.COMMAND_TEMPLATE_OBSERVATIONS!, JSON.stringify(data) + "\n") +export const CommandProjection: Plugin = async () => ({ + "command.execute.before": async (input) => { + record({ kind: "before", input }) + }, + event: async ({ event }) => { + if (event.type === "command.executed") record({ kind: "executed", event }) + }, +}) diff --git a/packages/cli/test/cli/fixture/integrated-capture.ts b/packages/cli/test/cli/fixture/integrated-capture.ts new file mode 100644 index 00000000..75fcc094 --- /dev/null +++ b/packages/cli/test/cli/fixture/integrated-capture.ts @@ -0,0 +1,345 @@ +import { expect } from "bun:test" +import { spawn, spawnSync } from "node:child_process" +import fs from "node:fs/promises" +import { existsSync, readFileSync } from "node:fs" +import os from "node:os" +import { Database } from "bun:sqlite" +import path from "node:path" + +const template = "FIRST<$1>\nSECOND<$2>\nREST<$3>" +const guard = path.join(import.meta.dir, "stdin-guard.ts") +const pause = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +export type Input = { + name: string + entry: "index" | "headless" + operands: string[] + stdin?: string + open?: boolean + tools?: string + configSelection?: string[] + extraArgs?: string[] + attach?: boolean + read?: boolean + profile?: boolean + globalInstructions?: boolean + disableProject?: boolean + denyRead?: boolean + deadlineMs?: number +} +export async function capture(input: Input) { + const cwd = await fs.mkdtemp(path.join(os.tmpdir(), "aictrl-command-arguments-")) + const files: Record = { + "AGENTS.md": "ROOT_AUTO_MARKER", + ".opencode/AGENTS.md": "OPENCODE_AUTO_MARKER", + "nested/AGENTS.md": "NESTED_AUTO_MARKER", + "nested/fixture.txt": "READ_FIXTURE_EXACT\n", + "config/aictrl/AGENTS.md": "TRUSTED_GLOBAL_MARKER", + "profile/AGENTS.md": "TRUSTED_PROFILE_MARKER", + "explicit.txt": "EXPLICIT_CALLER_MARKER", + "project-rule.txt": "PROJECT_RULE_MARKER", + } + if (input.profile === false) delete files["profile/AGENTS.md"] + const initialized = spawnSync("git", ["init", "--quiet"], { + cwd, + env: { + PATH: [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter), + HOME: path.join(cwd, "home"), + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + }, + }) + if (initialized.status !== 0) throw new Error("Owned fixture git init failed: " + initialized.stderr.toString()) + if (input.globalInstructions === false) delete files["config/aictrl/AGENTS.md"] + for (const [name, text] of Object.entries(files)) { + await fs.mkdir(path.dirname(path.join(cwd, name)), { recursive: true }) + await fs.writeFile(path.join(cwd, name), text) + } + await fs.writeFile(path.join(cwd, "aictrl.json"), JSON.stringify({ instructions: ["project-rule.txt"] })) + const http: { method: string; path: string; body: unknown }[] = [] + const requests: Record[] = [] + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + const route = new URL(request.url).pathname + const body = request.method === "POST" ? await request.json() : null + http.push({ method: request.method, path: route, body }) + if (route !== "/v1/chat/completions") return Response.json({ unexpected: route }, { status: 500 }) + requests.push(body as Record) + const call = input.read && requests.length === 1 + + const chunks = [ + { + choices: [ + { + index: 0, + delta: { + role: "assistant", + ...(call + ? { + tool_calls: [ + { + index: 0, + id: "read_integrated_1", + type: "function", + function: { + name: "read", + arguments: JSON.stringify({ filePath: path.join(cwd, "nested/fixture.txt") }), + }, + }, + ], + } + : { content: "INTEGRATED_DONE" }), + }, + finish_reason: null, + }, + ], + }, + { + choices: [{ index: 0, delta: {}, finish_reason: call ? "tool_calls" : "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }, + ] + return new Response(chunks.map((value) => `data: ${JSON.stringify(value)}\n\n`).join("") + "data: [DONE]\n\n", { + headers: { "content-type": "text/event-stream" }, + }) + }, + }) + const config = { + enabled_providers: ["fixture"], + provider: { + fixture: { + npm: "@ai-sdk/openai-compatible", + options: { apiKey: "fixture", baseURL: server.url.origin + "/v1" }, + models: { fixture: { name: "fixture", tool_call: true, limit: { context: 100000, output: 1000 } } }, + }, + }, + agent: { title: { disable: true } }, + permission: { "*": "allow", ...(input.denyRead ? { read: "deny" } : {}) }, + instructions: [path.join(cwd, "explicit.txt")], + plugin: [new URL("./integrated-observer-plugin.ts", import.meta.url).href], + command: { + argumentprobe: { template: "STATIC A<$1> B<$2> C<$3> RAW<$ARGUMENTS>" }, + }, + ...(input.configSelection !== undefined ? { tool_allowlist: input.configSelection } : {}), + } + await fs.writeFile(path.join(cwd, "observations.jsonl"), "") + const ledger = path.join(cwd, "network.jsonl") + await fs.writeFile(ledger, "") + const env = { + PATH: [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter), + HOME: path.join(cwd, "home"), + AICTRL_TEST_HOME: path.join(cwd, "home"), + AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(cwd, "managed"), + XDG_DATA_HOME: path.join(cwd, "data"), + XDG_CONFIG_HOME: path.join(cwd, "config"), + XDG_CACHE_HOME: path.join(cwd, "cache"), + XDG_STATE_HOME: path.join(cwd, "state"), + BUN_RUNTIME_TRANSPILER_CACHE_PATH: path.join(cwd, "bun-cache"), + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + TZ: "UTC", + AICTRL_DISABLE_DEFAULT_PLUGINS: "true", + AICTRL_DISABLE_AUTOCOMPACT: "true", + STDIN_FIXTURE_ORIGIN: server.url.origin, + STDIN_FIXTURE_LEDGER: ledger, + AICTRL_CONFIG_CONTENT: JSON.stringify(config), + AICTRL_DISABLE_PROJECT_CONFIG: String(input.disableProject !== false), + ...(input.profile === false ? {} : { AICTRL_CONFIG_DIR: path.join(cwd, "profile") }), + COMMAND_TEMPLATE_OBSERVATIONS: path.join(cwd, "observations.jsonl"), + } + const source = path.resolve(import.meta.dir, `../../../src/${input.entry}.ts`) + const command = [ + "run", + "--no-install", + "--no-env-file", + "--preload", + guard, + "--conditions=browser", + source, + "run", + "--format", + "json", + "--model", + "fixture/fixture", + ...(input.attach ? ["--attach", server.url.origin] : []), + ...(input.tools !== undefined ? ["--tools", input.tools] : []), + ...(input.extraArgs ?? []), + "--command", + "argumentprobe", + ...input.operands, + ] + const child = spawn(process.execPath, command, { + cwd, + env, + detached: true, + stdio: ["stdin" in input || input.open ? "pipe" : "ignore", "pipe", "pipe"], + }) + let stdout = "", + stderr = "", + deadline = false + const exists = () => { + try { + process.kill(-child.pid!, 0) + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false + throw error + } + } + let staged = false + let firstBytesAt: number | undefined + let eofAt: number | undefined + let requestsBeforeEOF: number | undefined + let beforeEOF: { http: unknown[]; dbExists: boolean; projectCacheExists: boolean; observations: string } | undefined + const started = performance.now() + const stages: ReturnType[] = [] + child.stdin?.on("error", (error: NodeJS.ErrnoException) => { + if (error.code !== "EPIPE") stderr += String(error) + }) + child.stdout!.on("data", (value) => { + stdout += value.toString() + if ("stdin" in input && !staged && stdout.includes('"type":"invocation_start"')) { + staged = true + stages.push( + setTimeout(() => { + firstBytesAt = performance.now() - started + child.stdin!.write(input.stdin ?? "") + stages.push( + setTimeout(() => { + eofAt = performance.now() - started + requestsBeforeEOF = requests.length + beforeEOF = { + http: [...http], + dbExists: existsSync(path.join(cwd, "data/aictrl/aictrl.db")), + projectCacheExists: existsSync(path.join(cwd, ".git/aictrl")), + observations: readFileSync(path.join(cwd, "observations.jsonl"), "utf8"), + } + child.stdin!.end() + }, 100), + ) + }, 100), + ) + } + }) + child.stderr!.on("data", (value) => { + stderr += value.toString() + }) + const timer = setTimeout(() => { + deadline = true + if (exists()) process.kill(-child.pid!, "SIGTERM") + }, input.deadlineMs ?? 12000) + const escalation = setTimeout( + () => { + if (deadline && exists()) process.kill(-child.pid!, "SIGKILL") + }, + (input.deadlineMs ?? 12000) + 200, + ) + let code: number | null = null + try { + code = await new Promise((resolve, reject) => { + child.once("error", reject) + child.once("close", (value) => resolve(value)) + }) + } finally { + for (const stage of stages) clearTimeout(stage) + child.stdin?.destroy() + clearTimeout(timer) + clearTimeout(escalation) + if (exists()) { + process.kill(-child.pid!, "SIGTERM") + await pause(150) + if (exists()) process.kill(-child.pid!, "SIGKILL") + } + server.stop(true) + } + const network = (await fs.readFile(ledger, "utf8")) + .split("\n") + .filter(Boolean) + .map((value) => JSON.parse(value)) + const events = stdout + .split("\n") + .filter((value) => value.startsWith("{")) + .map((value) => JSON.parse(value)) + const dbPath = path.join(cwd, "data/aictrl/aictrl.db") + let sessions: { id: string; title: string }[] = [], + messages: { data: Record }[] = [] + const dbExists = await fs.stat(dbPath).then( + () => true, + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return false + throw error + }, + ) + if (dbExists) { + const db = new Database(dbPath, { readonly: true }) + try { + sessions = db.query("SELECT id,title FROM session").all() as typeof sessions + messages = (db.query("SELECT data FROM message").all() as { data: string }[]).map((row) => ({ + data: JSON.parse(row.data), + })) + } finally { + db.close() + } + } + const result = { + input, + config, + http, + dbExists, + sessions, + messages, + projectCacheExists: await fs.stat(path.join(cwd, ".git/aictrl")).then( + () => true, + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return false + throw error + }, + ), + firstBytesAt, + eofAt, + requestsBeforeEOF, + beforeEOF, + command, + cwd, + code, + deadline, + groupGone: !exists(), + stdout, + stderr, + requests, + events, + network, + observations: (await fs.readFile(path.join(cwd, "observations.jsonl"), "utf8")) + .split("\n") + .filter(Boolean) + .map((row) => JSON.parse(row)), + } + if (process.env.COMMAND_ARGUMENTS_EVIDENCE) { + await fs.mkdir(process.env.COMMAND_ARGUMENTS_EVIDENCE, { recursive: true }) + const prefix = input.entry + await fs.writeFile( + path.join(process.env.COMMAND_ARGUMENTS_EVIDENCE, `${prefix}-${input.name}.json`), + JSON.stringify(result, null, 2) + "\n", + ) + } + if (deadline) throw new Error("Command argument fixture deadline fired: " + JSON.stringify(result)) + if (!result.groupGone) throw new Error("Command argument fixture process group remains") + if (network.some((row) => row.allowed === false)) throw new Error("Command argument fixture blocked external attempt") + return result +} +export function userText(body: Record) { + const messages = body.messages as { role: string; content: string | { type: string; text?: string }[] }[] + return messages + .filter((message) => message.role === "user") + .map((message) => + typeof message.content === "string" + ? message.content + : message.content + .filter((part) => part.type === "text") + .map((part) => part.text ?? "") + .join(""), + ) + .join("\n") +} diff --git a/packages/cli/test/cli/fixture/integrated-observer-plugin.ts b/packages/cli/test/cli/fixture/integrated-observer-plugin.ts new file mode 100644 index 00000000..42d9d3d0 --- /dev/null +++ b/packages/cli/test/cli/fixture/integrated-observer-plugin.ts @@ -0,0 +1,16 @@ +import { appendFileSync } from "node:fs" +import type { Plugin } from "@aictrl/plugin" +const record = (data: unknown) => + appendFileSync(process.env.COMMAND_TEMPLATE_OBSERVATIONS!, JSON.stringify(data) + "\n") +record({ kind: "plugin-import" }) +export const IntegratedObserver: Plugin = async () => { + record({ kind: "plugin-init" }) + return { + "command.execute.before": async (input) => { + record({ kind: "before", input }) + }, + event: async ({ event }) => { + if (event.type === "command.executed") record({ kind: "executed", event }) + }, + } +} diff --git a/packages/cli/test/cli/fixture/stdin-guard.ts b/packages/cli/test/cli/fixture/stdin-guard.ts new file mode 100644 index 00000000..03d900e2 --- /dev/null +++ b/packages/cli/test/cli/fixture/stdin-guard.ts @@ -0,0 +1,32 @@ +import { appendFileSync } from "node:fs" +const record = (event: unknown) => appendFileSync(process.env.STDIN_FIXTURE_LEDGER!, JSON.stringify(event) + "\n") +record({ kind: "preload", stdinTTY: Boolean(process.stdin.isTTY) }) +const original = globalThis.fetch +const spawn = Bun.spawn.bind(Bun) +Bun.spawn = ((...args: Parameters) => { + const first: unknown = args[0] + const command = Array.isArray(first) + ? first + : first && typeof first === "object" && "cmd" in first && Array.isArray(first.cmd) + ? first.cmd + : undefined + if (command && /(?:^|\/)bun$/.test(String(command[0])) && ["install", "add"].includes(String(command[1]))) { + record({ kind: "install", allowed: false }) + throw new Error("Stdin fixture forbids installation") + } + return spawn(...args) +}) as typeof Bun.spawn +globalThis.fetch = Object.assign( + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input)) + const allowed = url.origin === process.env.STDIN_FIXTURE_ORIGIN + record({ kind: "fetch", origin: url.origin, path: url.pathname, allowed }) + if (!allowed) throw new Error("Stdin fixture blocked external request: " + url.origin) + return original(input, init) + }, + { + preconnect: () => { + throw new Error("Stdin fixture forbids preconnect") + }, + }, +) as typeof original diff --git a/packages/cli/test/cli/run-command-arguments.test.ts b/packages/cli/test/cli/run-command-arguments.test.ts new file mode 100644 index 00000000..a5cbf69c --- /dev/null +++ b/packages/cli/test/cli/run-command-arguments.test.ts @@ -0,0 +1,435 @@ +import { describe, expect, test } from "bun:test" +import { spawn } from "node:child_process" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" + +const template = "FIRST<$1>\nSECOND<$2>\nREST<$3>" +const guard = path.join(import.meta.dir, "fixture/stdin-guard.ts") +const pause = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +type Input = ( + | { name: string; entry: "index" | "headless"; operands: string[]; stdin?: string } + | { name: string; raw: string } +) & { template?: string; deadlineMs?: number } +async function capture(input: Input) { + const cwd = await fs.mkdtemp(path.join(os.tmpdir(), "aictrl-command-arguments-")) + const requests: Record[] = [] + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + if (new URL(request.url).pathname !== "/v1/chat/completions") throw new Error("Unexpected provider path") + requests.push((await request.json()) as Record) + const chunks = [ + { choices: [{ index: 0, delta: { role: "assistant", content: "Done" }, finish_reason: null }] }, + { + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }, + ] + return new Response(chunks.map((value) => `data: ${JSON.stringify(value)}\n\n`).join("") + "data: [DONE]\n\n", { + headers: { "content-type": "text/event-stream" }, + }) + }, + }) + await fs.writeFile( + path.join(cwd, "aictrl.json"), + JSON.stringify({ + enabled_providers: ["fixture"], + provider: { + fixture: { + npm: "@ai-sdk/openai-compatible", + options: { apiKey: "fixture", baseURL: server.url.origin + "/v1" }, + models: { fixture: { name: "fixture", tool_call: true, limit: { context: 100000, output: 1000 } } }, + }, + }, + agent: { title: { disable: true } }, + command: { argumentprobe: { template: input.template ?? template } }, + }), + ) + const data = path.join(cwd, "arguments.json") + await fs.writeFile(data, JSON.stringify("raw" in input ? { arguments: input.raw } : {})) + const ledger = path.join(cwd, "network.jsonl") + await fs.writeFile(ledger, "") + const env = { + PATH: [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter), + HOME: path.join(cwd, "home"), + AICTRL_TEST_HOME: path.join(cwd, "home"), + AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(cwd, "managed"), + XDG_DATA_HOME: path.join(cwd, "data"), + XDG_CONFIG_HOME: path.join(cwd, "config"), + XDG_CACHE_HOME: path.join(cwd, "cache"), + XDG_STATE_HOME: path.join(cwd, "state"), + BUN_RUNTIME_TRANSPILER_CACHE_PATH: path.join(cwd, "bun-cache"), + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + TZ: "UTC", + AICTRL_DISABLE_DEFAULT_PLUGINS: "true", + AICTRL_DISABLE_AUTOCOMPACT: "true", + STDIN_FIXTURE_ORIGIN: server.url.origin, + STDIN_FIXTURE_LEDGER: ledger, + COMMAND_ARGUMENTS_DATA: data, + } + const source = + "raw" in input + ? path.join(import.meta.dir, "fixture/command-arguments-legacy.ts") + : path.resolve(import.meta.dir, `../../src/${input.entry}.ts`) + const command = [ + "run", + "--no-install", + "--no-env-file", + "--preload", + guard, + "--conditions=browser", + source, + ...("raw" in input + ? [] + : ["run", "--format", "json", "--model", "fixture/fixture", "--command", "argumentprobe", ...input.operands]), + ] + const child = spawn(process.execPath, command, { + cwd, + env, + detached: true, + stdio: ["stdin" in input ? "pipe" : "ignore", "pipe", "pipe"], + }) + let stdout = "", + stderr = "", + deadline = false + const exists = () => { + try { + process.kill(-child.pid!, 0) + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false + throw error + } + } + let staged = false + let firstBytesAt: number | undefined + let eofAt: number | undefined + let requestsBeforeEOF: number | undefined + const started = performance.now() + const stages: ReturnType[] = [] + child.stdin?.on("error", (error: NodeJS.ErrnoException) => { + if (error.code !== "EPIPE") stderr += String(error) + }) + child.stdout!.on("data", (value) => { + stdout += value.toString() + if ("stdin" in input && !staged && stdout.includes('"type":"invocation_start"')) { + staged = true + stages.push( + setTimeout(() => { + firstBytesAt = performance.now() - started + child.stdin!.write(input.stdin ?? "") + stages.push( + setTimeout(() => { + eofAt = performance.now() - started + requestsBeforeEOF = requests.length + child.stdin!.end() + }, 100), + ) + }, 100), + ) + } + }) + child.stderr!.on("data", (value) => { + stderr += value.toString() + }) + const timer = setTimeout(() => { + deadline = true + if (exists()) process.kill(-child.pid!, "SIGTERM") + }, input.deadlineMs ?? 12000) + const escalation = setTimeout( + () => { + if (deadline && exists()) process.kill(-child.pid!, "SIGKILL") + }, + (input.deadlineMs ?? 12000) + 200, + ) + let code: number | null = null + try { + code = await new Promise((resolve, reject) => { + child.once("error", reject) + child.once("close", (value) => resolve(value)) + }) + } finally { + for (const stage of stages) clearTimeout(stage) + child.stdin?.destroy() + clearTimeout(timer) + clearTimeout(escalation) + if (exists()) { + process.kill(-child.pid!, "SIGTERM") + await pause(150) + if (exists()) process.kill(-child.pid!, "SIGKILL") + } + server.stop(true) + } + const network = (await fs.readFile(ledger, "utf8")) + .split("\n") + .filter(Boolean) + .map((value) => JSON.parse(value)) + const events = stdout + .split("\n") + .filter((value) => value.startsWith("{")) + .map((value) => JSON.parse(value)) + const result = { + input, + template: input.template ?? template, + firstBytesAt, + eofAt, + requestsBeforeEOF, + command, + cwd, + code, + deadline, + groupGone: !exists(), + stdout, + stderr, + requests, + events, + network, + } + if (process.env.COMMAND_ARGUMENTS_EVIDENCE) { + await fs.mkdir(process.env.COMMAND_ARGUMENTS_EVIDENCE, { recursive: true }) + const prefix = "entry" in input ? input.entry : "legacy" + await fs.writeFile( + path.join(process.env.COMMAND_ARGUMENTS_EVIDENCE, `${prefix}-${input.name}.json`), + JSON.stringify(result, null, 2) + "\n", + ) + } + if (deadline) throw new Error("Command argument fixture deadline fired: " + JSON.stringify(result)) + if (!result.groupGone) throw new Error("Command argument fixture process group remains") + if (network.some((row) => row.allowed === false)) throw new Error("Command argument fixture blocked external attempt") + expect(code, stderr + stdout).toBe(0) + expect(requests).toHaveLength(1) + return result +} +function userText(body: Record) { + const messages = body.messages as { role: string; content: string | { type: string; text?: string }[] }[] + return messages + .filter((message) => message.role === "user") + .map((message) => + typeof message.content === "string" + ? message.content + : message.content + .filter((part) => part.type === "text") + .map((part) => part.text ?? "") + .join(""), + ) + .join("\n") +} + +// Literal expected templates are ground truth from original operand boundaries, not a copy of the decoder. +const cases = [ + { name: "plain-control", operands: ["alpha", "beta", "rest"], expected: "FIRST\nSECOND\nREST" }, + { + name: "space-control", + operands: ["alpha beta", "tail", "rest"], + expected: "FIRST\nSECOND\nREST", + }, + { + name: "backslash-control", + operands: ["alpha\\ beta", "tail", "rest"], + expected: "FIRST\nSECOND\nREST", + }, + { + name: "trailing-backslash-control", + operands: ["trail\\", "tail", "rest"], + expected: "FIRST\nSECOND\nREST", + }, + { + name: "quoted-operand", + operands: ['say "hi" now', "tail", "rest"], + expected: 'FIRST\nSECOND\nREST', + }, + { + name: "backslash-and-quotes", + operands: ['path\\ "label" end', "tail", "rest"], + expected: 'FIRST\nSECOND\nREST', + }, + { + name: "tab-operand", + operands: ["alpha\tbeta", "tail", "rest"], + expected: "FIRST\nSECOND\nREST", + }, + { + name: "newline-operand", + operands: ["alpha\nbeta", "tail", "rest"], + expected: "FIRST\nSECOND\nREST", + }, + { name: "empty-first-operand", operands: ["", "tail", "rest"], expected: "FIRST<>\nSECOND\nREST" }, + { name: "empty-second-operand", operands: ["alpha", "", "rest"], expected: "FIRST\nSECOND<>\nREST" }, + { + name: "literal-quotes", + operands: ['"quoted"', "tail", "rest"], + expected: 'FIRST<"quoted">\nSECOND\nREST', + }, + { + name: "apostrophe", + operands: ["alpha'bet", "tail", "rest"], + expected: "FIRST\nSECOND\nREST", + }, + { + name: "unicode-numeric-control", + operands: ["naïve λ", "0007", "1e3"], + expected: "FIRST\nSECOND<0007>\nREST<1e3>", + }, +] + +describe("actual local command argument grouping", () => { + for (const entry of ["index", "headless"] as const) { + for (const item of cases) + test(`${entry}: ${item.name}`, async () => { + const result = await capture({ entry, name: item.name, operands: item.operands }) + expect(userText(result.requests[0])).toBe(item.expected) + const emitted = result.events.find((event) => event.type === "user_message") + expect( + emitted?.parts + .filter((part: { type: string }) => part.type === "text") + .map((part: { text: string }) => part.text), + ).toEqual([item.expected]) + }, 15000) + } + for (const item of [ + { name: "quoted-space", raw: '"alpha beta" tail rest', expected: "FIRST\nSECOND\nREST" }, + { + name: "literal-backslash", + raw: '"C:\\temp\\path" tail rest', + expected: "FIRST\nSECOND\nREST", + }, + ]) + test(`legacy raw handler: ${item.name}`, async () => { + const result = await capture({ name: item.name, raw: item.raw }) + expect(userText(result.requests[0])).toBe(item.expected) + }, 15000) +}) + +// Inserted operand text must stay literal: substitution visits the original template only. +describe("literal placeholder data", () => { + for (const entry of ["index", "headless"] as const) { + for (const item of [ + { + name: "arguments-token", + operands: ["$ARGUMENTS", "tail", "rest"], + expected: "FIRST<$ARGUMENTS>\nSECOND\nREST", + }, + { + name: "numeric-and-replacement-tokens", + operands: ["$1 $$ $&", "$2", "$3"], + expected: "FIRST<$1 $$ $&>\nSECOND<$2>\nREST<$3>", + }, + ]) + test(`${entry}: literal-${item.name}`, async () => { + const result = await capture({ entry, name: "literal-" + item.name, operands: item.operands }) + expect(userText(result.requests[0])).toBe(item.expected) + const emitted = result.events.find((event) => event.type === "user_message") + expect( + emitted?.parts + .filter((part: { type: string }) => part.type === "text") + .map((part: { text: string }) => part.text), + ).toEqual([item.expected]) + }, 15000) + } + test("legacy handler: literal arguments-token", async () => { + const result = await capture({ name: "literal-arguments-token", raw: '"$ARGUMENTS" tail rest' }) + expect(userText(result.requests[0])).toBe("FIRST<$ARGUMENTS>\nSECOND\nREST") + }, 15000) +}) + +describe("structured raw projection and selected stdin", () => { + for (const entry of ["index", "headless"] as const) { + for (const item of [ + { + name: "raw-arguments", + operands: ['say "hi"', "", "tail\\path"], + template: "RAW<$ARGUMENTS>", + expected: 'RAW', + }, + { + name: "fallback", + operands: ['say "hi"', "tail\\path"], + template: "HEADER", + expected: 'HEADER\n\nsay "hi" tail\\path', + }, + { + name: "literal-raw-tokens", + operands: ["$ARGUMENTS $1 $$ $&", "tail"], + template: "RAW<$ARGUMENTS>", + expected: "RAW<$ARGUMENTS $1 $$ $& tail>", + }, + { + name: "late-stdin-position", + operands: ["alpha", "-", "beta"], + stdin: "late λ\nline", + template: "A<$1> B<$2> C<$3>", + expected: "A B C", + }, + { + name: "empty-selected-stdin", + operands: ["alpha", "-", "beta"], + stdin: "", + template: "A<$1> B<$2> C<$3>", + expected: "A B C<>", + }, + { + name: "late-stdin-raw", + operands: ["alpha", "-"], + stdin: "late λ\nline", + template: "RAW<$ARGUMENTS>", + expected: "RAW", + }, + { + name: "no-operand-stdin", + operands: [], + stdin: "late λ\nline", + template: "A<$1> B<$2>", + expected: "A B<>", + }, + { + name: "repeated-markers", + operands: ["-", "alpha", "-", "beta", "-"], + stdin: "late λ", + template: "A<$1> B<$2> C<$3>", + expected: "A B C", + }, + ]) + test(`${entry}: ${item.name}`, async () => { + const result = await capture({ entry, ...item }) + expect(userText(result.requests[0])).toBe(item.expected) + if ("stdin" in item) { + expect(result.firstBytesAt).toBeGreaterThan(90) + expect(result.eofAt!).toBeGreaterThan(result.firstBytesAt!) + expect(result.requestsBeforeEOF).toBe(0) + } + }, 15000) + } + for (const item of [ + { + name: "UNC", + raw: '"\\\\server\\share" tail rest', + expected: "FIRST<\\\\server\\share>\nSECOND\nREST", + }, + { name: "adjacent-image", raw: "[Image 12]tail end", expected: "FIRST<[Image 12]>\nSECOND\nREST" }, + ]) + test(`legacy exact ${item.name}`, async () => { + const result = await capture(item) + expect(userText(result.requests[0])).toBe(item.expected) + }, 15000) +}) + +// Schema-only contract checks: this does not claim HTTP receiver validation. +test("canonical command schema rejects malformed structured data", async () => { + const { SessionPrompt } = await import("../../src/session/prompt") + for (const value of [null, [], 1, {}, { argv: "text" }, { argv: [1] }, { argv: [], stdin: 1 }]) { + expect(SessionPrompt.CommandArguments.safeParse(value).success).toBe(false) + } + for (const value of ["legacy", { argv: [] }, { argv: ["", 'quote"', "\\path"], stdin: "" }]) { + expect(SessionPrompt.CommandArguments.parse(value)).toEqual(value) + } +}) + +test("command fixture deadline rejects before provider work", async () => { + await expect( + capture({ entry: "index", name: "forced-deadline", operands: ["inert"], deadlineMs: 1 }), + ).rejects.toThrow("Command argument fixture deadline fired") +}, 15000) diff --git a/packages/cli/test/cli/run-command-template.test.ts b/packages/cli/test/cli/run-command-template.test.ts new file mode 100644 index 00000000..a239cef4 --- /dev/null +++ b/packages/cli/test/cli/run-command-template.test.ts @@ -0,0 +1,412 @@ +import { describe, expect, test } from "bun:test" +import { capture, userText } from "./fixture/command-template-capture" + +function projection(result: Awaited>, expected: string) { + const before = result.observations.filter((row: { kind: string }) => row.kind === "before") + const executed = result.observations.filter((row: { kind: string }) => row.kind === "executed") + expect(before).toHaveLength(1) + expect(executed).toHaveLength(1) + expect(before[0].input.arguments).toBe(expected) + expect(executed[0].event.properties.arguments).toBe(expected) + expect(typeof before[0].input.arguments).toBe("string") + expect(typeof executed[0].event.properties.arguments).toBe("string") +} + +describe("current command expansion contracts", () => { + for (const entry of ["index", "headless"] as const) { + for (const item of [ + { + name: "repeated-out-of-order", + template: "X<$3> A<$1> AGAIN<$3> B<$2>", + expected: "X A AGAIN B", + }, + { name: "sparse-highest", template: "X<$4> A<$1> X2<$4>", expected: "X A X2" }, + { name: "missing-highest", template: "X<$9> A<$1> X2<$9>", expected: "X<> A X2<>" }, + ]) + test(`${entry}: ${item.name}`, async () => { + const result = await capture({ + entry, + name: item.name, + template: item.template, + operands: ["a", "b", "c", "d"], + }) + expect(userText(result.requests[0])).toBe(item.expected) + projection(result, "a b c d") + }, 15000) + } + for (const item of [ + { + name: "legacy-quoted-UNC", + arguments: '"\\\\server\\share" tail rest', + expected: "FIRST<\\\\server\\share>\nSECOND\nREST", + raw: '"\\\\server\\share" tail rest', + }, + { + name: "legacy-adjacent-quotes", + arguments: '"alpha beta""gamma" delta', + expected: "FIRST\nSECOND\nREST", + raw: '"alpha beta""gamma" delta', + }, + { + name: "legacy-adjacent-image", + arguments: "[Image 12]tail end", + expected: "FIRST<[Image 12]>\nSECOND\nREST", + raw: "[Image 12]tail end", + }, + { + name: "structured-projection", + arguments: { argv: ['say "hi"', "", "tail\\path"] }, + expected: 'FIRST\nSECOND<>\nREST', + raw: 'say "hi" tail\\path', + }, + { + name: "structured-stdin-projection", + arguments: { argv: ["alpha", "beta"], stdin: "late λ\nline" }, + expected: "FIRST\nSECOND\nREST", + raw: "alpha beta\nlate λ\nline", + }, + { + name: "structured-empty-stdin", + arguments: { argv: ["alpha", "beta"], stdin: "" }, + expected: "FIRST\nSECOND\nREST<>", + raw: "alpha beta\n", + }, + { + name: "legacy-newline-projection", + arguments: "alpha beta\nline", + expected: "FIRST\nSECOND\nREST", + raw: "alpha beta\nline", + }, + ]) + test( + item.name, + async () => { + const result = await capture({ name: item.name, arguments: item.arguments }) + expect(userText(result.requests[0])).toBe(item.expected) + projection(result, item.raw) + }, + 15000, + ) +}) + +// Every executable in these controls is the explicitly authorized inert absolute printf. +describe("original template shell authority", () => { + for (const item of [ + { + name: "argument-cannot-create-span", + operands: ["!`/usr/bin/printf INERT_ARG`"], + template: "DATA<$ARGUMENTS>", + expected: "DATA", + }, + { + name: "stdin-cannot-create-span", + operands: ["-"], + stdin: "!`/usr/bin/printf INERT_STDIN`", + template: "DATA<$1>", + expected: "DATA", + }, + { + name: "fallback-cannot-create-span", + operands: ["!`/usr/bin/printf INERT_FALLBACK`"], + template: "HEADER", + expected: "HEADER\n\n!`/usr/bin/printf INERT_FALLBACK`", + }, + { + name: "trusted-static-span", + operands: ["inert"], + template: "OUT $1", + expected: "OUT inert", + }, + { + name: "trusted-dynamic-vector", + operands: ["alpha beta", "tail; /usr/bin/printf INERT_SEMI"], + template: "OUT' $ARGUMENTS`>", + expected: "OUT<>", + }, + { + name: "trusted-output-not-rescanned", + operands: ["inert"], + template: "OUT $1", + expected: "OUT inert", + }, + ]) + test( + item.name, + async () => { + const result = await capture({ entry: "index", ...item }) + expect(userText(result.requests[0])).toBe(item.expected) + }, + 15000, + ) +}) + +describe("trusted template binding matrix", () => { + const items: (import("./fixture/command-template-capture").Input & { expected: string })[] = [ + { + entry: "headless", + name: "quoted-vector-data", + operands: ["alpha beta", "", "$UNSET", "`/usr/bin/printf INERT_BACKTICK`", "semi; /usr/bin/printf INERT_SEMI"], + template: "OUT' \"$ARGUMENTS\"`>", + expected: "OUT<>", + }, + { + entry: "headless", + name: "unquoted-vector-quotes-slashes", + operands: ["quote\" and apostrophe'", "\\path\\tail", "line\nbreak"], + template: "OUT' $ARGUMENTS`>", + expected: "OUT<<\\path\\tail>>", + }, + { + entry: "headless", + name: "unquoted-scalar-operators", + operands: ["semi; /usr/bin/printf INERT_SCALAR"], + template: "OUT' $1`>", + expected: "OUT<>", + }, + { + entry: "headless", + name: "quoted-scalar-data", + operands: ['say "hi" $UNSET'], + template: "OUT' \"$1\"`>", + expected: 'OUT<>', + }, + { + entry: "headless", + name: "scalar-concat", + operands: ["a b"], + template: "OUT' pre$1/post`>", + expected: "OUT<>", + }, + { + entry: "headless", + name: "vector-concat-native", + operands: ["a b", "c"], + template: "OUT' pre$ARGUMENTS/post`>", + expected: "OUT<>", + }, + { + entry: "headless", + name: "quoted-vector-concat-native", + operands: ["a b", "c"], + template: "OUT' \"pre$ARGUMENTS/post\"`>", + expected: "OUT<>", + }, + { + entry: "headless", + name: "missing-number-scalar", + operands: ["a"], + template: "OUT' $9`> A<$1>", + expected: "OUT<<>> A", + }, + { + entry: "headless", + name: "global-highest-across-spans", + operands: ["a", "b", "c", "d"], + template: "A<$1> OUT' $3`> B<$2>", + expected: "A OUT<> B", + }, + { + entry: "headless", + name: "multiple-static-spans", + operands: ["a b"], + template: "A DATA<$ARGUMENTS> B' $1`>", + expected: "A DATA B<>", + }, + { + entry: "headless", + name: "trusted-static-operators", + operands: ["inert"], + template: "OUT $1", + expected: "OUT inert", + }, + { + entry: "headless", + name: "trusted-static-substitution", + operands: ["inert"], + template: "OUT $1", + expected: "OUT inert", + }, + { + entry: "headless", + name: "output-placeholder-not-rescanned", + operands: ["inert"], + template: "OUT $1", + expected: "OUT<$ARGUMENTS $1> inert", + }, + { + name: "empty-vector", + arguments: { argv: [] }, + template: "OUT' $ARGUMENTS LAST`>", + expected: "OUT<>", + }, + { + name: "empty-stdin-vector-member", + arguments: { argv: [], stdin: "" }, + template: "OUT' $ARGUMENTS LAST`>", + expected: "OUT<<>>", + }, + { + entry: "headless", + name: "late-stdin-bound-data", + operands: ["alpha", "-"], + stdin: "!`/usr/bin/printf INERT_STDIN`", + template: "RAW<$ARGUMENTS> OUT' $ARGUMENTS`>", + expected: "RAW OUT<>", + }, + { + name: "legacy-decoded-vector", + arguments: '"space here" "\\\\server\\share"', + template: "RAW<$ARGUMENTS> OUT' $ARGUMENTS`>", + expected: 'RAW<"space here" "\\\\server\\share"> OUT<<\\\\server\\share>>', + }, + { + entry: "headless", + name: "headless-argument-span-literal", + operands: ["!`/usr/bin/printf INERT_ARG`"], + template: "DATA<$ARGUMENTS>", + expected: "DATA", + }, + { + entry: "headless", + name: "headless-fallback-span-literal", + operands: ["!`/usr/bin/printf INERT_FALLBACK`"], + template: "HEADER", + expected: "HEADER\n\n!`/usr/bin/printf INERT_FALLBACK`", + }, + ] + for (const item of items) + test( + item.name, + async () => { + const result = await capture(item) + expect(userText(result.requests[0])).toBe(item.expected) + if ("stdin" in item) { + expect(result.firstBytesAt).toBeGreaterThan(90) + expect(result.eofAt!).toBeGreaterThan(result.firstBytesAt!) + expect(result.requestsBeforeEOF).toBe(0) + } + }, + 15000, + ) +}) + +describe("legacy highest numbered placeholder controls", () => { + for (const item of [ + { + name: "legacy-repeated-out-of-order", + template: "X<$3> A<$1> AGAIN<$3> B<$2>", + expected: "X A AGAIN B", + }, + { name: "legacy-sparse-highest", template: "X<$4> A<$1> AGAIN<$4>", expected: "X A AGAIN" }, + ]) + test( + item.name, + async () => { + const result = await capture({ name: item.name, arguments: '"a b" c d e', template: item.template }) + expect(userText(result.requests[0])).toBe(item.expected) + projection(result, '"a b" c d e') + }, + 15000, + ) +}) + +test("template fixture forced deadline cannot pass as a rejection", async () => { + await expect( + capture({ entry: "index", name: "forced-template-deadline", operands: ["inert"], deadlineMs: 1 }), + ).rejects.toThrow("Command argument fixture deadline fired") +}, 15000) + +// These deliberately exercise author-selected programs; binding is not a program sandbox. +describe("original template shell context acceptance", () => { + for (const item of [ + { + name: "command-position-absolute-printf", + operands: ["/usr/bin/printf"], + template: "OUT' COMMAND_DATA`>", + expected: "OUT<>", + }, + { + name: "command-position-data-cannot-add-program", + operands: ["/usr/bin/printf; /usr/bin/printf INERT_PROGRAM"], + template: "OUT' COMMAND_DATA`>", + expected: "OUT<>", + }, + { + name: "unquoted-assignment-data", + operands: ["data ; /usr/bin/printf INERT_ASSIGN $UNSET"], + template: "OUT' \"$ASSIGN\"`>", + expected: "OUT<>", + }, + { + name: "quoted-assignment-data", + operands: ['say "hi" ; /usr/bin/printf INERT_ASSIGN'], + template: 'OUT\' "$ASSIGN"`>', + expected: 'OUT<>', + }, + { + name: "owned-redirection-data", + operands: ["OWNED_REDIRECT_TARGET"], + ownedRedirect: true, + template: "OUT $1`>", + expected: "OUT<>", + }, + { + name: "trusted-static-pipeline", + operands: ["inert"], + template: "OUT $1", + expected: "OUT inert", + }, + { + name: "nonzero-stdout-preserved", + operands: ["inert"], + template: "OUT $1", + expected: "OUT<> inert", + }, + { + name: "trusted-interpreter-is-authority", + operands: ['process.stdout.write("INERT_INTERPRETER")'], + template: "OUT", + expected: "OUT", + }, + ]) + test( + item.name, + async () => { + const result = await capture({ entry: "headless", ...item }) + expect(userText(result.requests[0])).toBe(item.expected) + if (item.name === "owned-redirection-data") { + expect(result.redirected?.text).toBe("REDIRECTED") + expect(result.redirected?.removed).toBe(true) + expect(result.redirected?.path.startsWith(result.cwd + "/")).toBe(true) + } + }, + 15000, + ) + test("quoted empty vector is one empty argument", async () => { + const result = await capture({ + name: "quoted-empty-vector", + arguments: { argv: [] }, + template: "OUT' \"$ARGUMENTS\" LAST`>", + }) + expect(userText(result.requests[0])).toBe("OUT<<>>") + }, 15000) + test("native unclosed quote tolerance is preserved", async () => { + const result = await capture({ + entry: "headless", + name: "malformed-trusted-quote", + operands: ["inert"], + template: "OUT $1", + }) + expect(userText(result.requests[0])).toBe("OUT inert") + }, 15000) +}) + +test("malformed quoted substitution preserves catch error", async () => { + const result = await capture({ + entry: "headless", + name: "malformed-quoted-substitution", + operands: ["inert"], + template: 'OUT $1', + }) + expect(userText(result.requests[0])).toBe("OUT inert") +}, 15000) diff --git a/packages/cli/test/cli/run-integrated-headless.test.ts b/packages/cli/test/cli/run-integrated-headless.test.ts new file mode 100644 index 00000000..1eaf57e7 --- /dev/null +++ b/packages/cli/test/cli/run-integrated-headless.test.ts @@ -0,0 +1,218 @@ +import { describe, expect, test } from "bun:test" +import { capture, userText } from "./fixture/integrated-capture" + +const expected = + 'STATIC A B C RAW' +const payload = "late λ\n!`/usr/bin/printf ARG_LITERAL`" +const full = ["bash", "edit", "glob", "grep", "read", "skill", "task", "webfetch", "write"] +function names(body: Record) { + return ((body.tools ?? []) as { function: { name: string } }[]).map((x) => x.function.name).sort() +} +function success( + result: Awaited>, + catalog: string[], + selection: string[] | null, + read: boolean, + enabled = false, + mode: "profile" | "global" | "project" | "none" = "profile", +) { + expect(result.code, result.stderr + result.stdout).toBe(0) + expect(result.requests).toHaveLength(read ? 2 : 1) + for (const request of result.requests) expect(names(request)).toEqual(catalog) + const inventory = result.events.find((event: { type: string }) => event.type === "tool_catalog") + expect(inventory?.tools.some((tool: { name: string }) => tool.name === "read")).toBe(true) + // Registry inventory is not the effective provider catalog. + expect(userText(result.requests[0])).toBe(expected) + expect(result.sessions.map((x) => x.title)).toEqual(['/argumentprobe two words quote" λ']) + const users = result.messages.filter((x) => x.data.role === "user") + expect(users).toHaveLength(1) + expect(users[0].data.toolSelection).toEqual({ version: 1, allowlist: selection }) + const first = JSON.stringify(result.requests[0]), + all = JSON.stringify(result.requests) + expect(first).toContain("EXPLICIT_CALLER_MARKER") + if (mode !== "none") + expect(first).toContain( + mode === "profile" + ? "TRUSTED_PROFILE_MARKER" + : mode === "global" + ? "TRUSTED_GLOBAL_MARKER" + : "OPENCODE_AUTO_MARKER", + ) + if (mode !== "global") expect(first).not.toContain("TRUSTED_GLOBAL_MARKER") + if (mode !== "profile") expect(first).not.toContain("TRUSTED_PROFILE_MARKER") + for (const marker of ["ROOT_AUTO_MARKER", "OPENCODE_AUTO_MARKER", "NESTED_AUTO_MARKER", "PROJECT_RULE_MARKER"]) { + if (enabled && (marker !== "OPENCODE_AUTO_MARKER" || mode === "project")) expect(all).toContain(marker) + else expect(all).not.toContain(marker) + } + if (read) { + const tool = (result.requests[1].messages as { role: string; content: unknown }[]).filter((x) => x.role === "tool") + expect(JSON.stringify(tool)).toContain("READ_FIXTURE_EXACT") + } + expect(result.firstBytesAt).toBeGreaterThan(90) + expect(result.eofAt!).toBeGreaterThan(result.firstBytesAt!) + expect(result.requestsBeforeEOF).toBe(0) + const before = result.observations.filter((x: { kind: string }) => x.kind === "before"), + executed = result.observations.filter((x: { kind: string }) => x.kind === "executed") + expect(result.observations.filter((x: { kind: string }) => x.kind === "plugin-import")).toHaveLength(1) + expect(result.observations.filter((x: { kind: string }) => x.kind === "plugin-init")).toHaveLength(1) + expect(before).toHaveLength(1) + expect(executed).toHaveLength(1) + expect(before[0].input.arguments).toBe('two words quote" λ\n' + payload) + expect(executed[0].event.properties.arguments).toBe('two words quote" λ\n' + payload) +} + +describe("coupled selector stdin command and instruction acceptance", () => { + for (const entry of ["index", "headless"] as const) { + test(`${entry}: selected Read consumes exact command data and excludes discovered instructions`, async () => { + const result = await capture({ + entry, + name: "selected-read", + tools: "read", + operands: ["two words", "-", 'quote" λ'], + stdin: payload, + read: true, + }) + success(result, ["read"], ["read"], true) + }, 15000) + test(`${entry}: explicit empty tools preserves command data title and events`, async () => { + const result = await capture({ + entry, + name: "empty-tools", + tools: "", + operands: ["two words", "-", 'quote" λ'], + stdin: payload, + }) + success(result, [], [], false) + }, 15000) + } + for (const item of [ + { name: "default-nine", catalog: full, selection: null, read: true }, + { name: "config-read", configSelection: ["read"], catalog: ["read"], selection: ["read"], read: true }, + { + name: "cli-replaces-empty-config", + configSelection: [], + tools: "read", + catalog: ["read"], + selection: ["read"], + read: true, + }, + { name: "config-empty", configSelection: [], catalog: [], selection: [], read: false }, + { name: "permissions-still-deny", tools: "read", denyRead: true, catalog: [], selection: ["read"], read: false }, + { + name: "trusted-global-control", + profile: false, + catalog: ["read"], + selection: ["read"], + tools: "read", + read: true, + }, + { + name: "opencode-enabled-control", + profile: false, + globalInstructions: false, + disableProject: false, + catalog: ["read"], + selection: ["read"], + tools: "read", + read: true, + }, + { + name: "opencode-disabled-control", + profile: false, + globalInstructions: false, + catalog: ["read"], + selection: ["read"], + tools: "read", + read: true, + }, + { + name: "enabled-discovery-control", + tools: "read", + disableProject: false, + catalog: ["read"], + selection: ["read"], + read: true, + }, + ]) + test( + item.name, + async () => { + const result = await capture({ + entry: "headless", + ...item, + operands: ["two words", "-", 'quote" λ'], + stdin: payload, + }) + success( + result, + item.catalog, + item.selection, + item.read, + item.disableProject === false, + item.name === "opencode-disabled-control" + ? "none" + : item.name === "opencode-enabled-control" + ? "project" + : item.name === "trusted-global-control" + ? "global" + : "profile", + ) + }, + 15000, + ) + for (const item of [ + { name: "attach-explicit-empty-before-stdin", attach: true, tools: "" }, + { name: "malformed-tools-before-stdin", tools: " read" }, + { name: "repeated-tools-before-stdin", extraArgs: ["--tools", "read", "--tools", "bash"] }, + { name: "missing-tools-before-stdin", extraArgs: ["--tools"] }, + ]) + test( + item.name, + async () => { + const result = await capture({ entry: "headless", ...item, operands: ["-"], open: true }) + expect(result.code, result.stderr + result.stdout).toBe(1) + expect(result.http).toEqual([]) + expect(result.requests).toEqual([]) + expect(result.dbExists).toBe(false) + expect(result.projectCacheExists).toBe(false) + expect(result.observations).toEqual([]) + }, + 15000, + ) + test("actual coupled helper deadline never passes as a negative result", async () => { + await expect( + capture({ entry: "index", name: "forced-deadline", tools: "read", operands: ["-"], open: true, deadlineMs: 1 }), + ).rejects.toThrow("deadline fired") + }, 15000) +}) + +test("config-derived empty attach selection rejects after selected EOF without any receiver or local mutation", async () => { + const result = await capture({ + entry: "headless", + name: "attach-config-empty-late-EOF", + attach: true, + configSelection: [], + operands: ["-"], + stdin: "selected data stays local", + }) + expect(result.firstBytesAt).toBeGreaterThan(90) + expect(result.eofAt!).toBeGreaterThan(result.firstBytesAt!) + expect(result.beforeEOF).toEqual({ http: [], dbExists: false, projectCacheExists: false, observations: "" }) + expect(result.code, result.stderr + result.stdout).toBe(1) + expect(result.http).toEqual([]) + expect(result.requests).toEqual([]) + expect(result.dbExists).toBe(false) + expect(result.projectCacheExists).toBe(false) + expect(result.observations).toEqual([]) + const error = result.events.find((event: { type: string }) => event.type === "invocation_error") + expect(error?.code).toBe("INVOCATION_INVALID_ARGUMENTS") + expect(error?.phase).toBe("bootstrap") + expect(result.stderr).toContain( + "Work-tool selection is not supported with --attach: this client cannot verify server-side enforcement.", + ) + expect(result.events.map((event: { type: string }) => event.type)).toEqual([ + "invocation_start", + "invocation_error", + "invocation_complete", + ]) +}, 15000) diff --git a/packages/cli/test/cli/run-stdin.test.ts b/packages/cli/test/cli/run-stdin.test.ts new file mode 100644 index 00000000..e19c78da --- /dev/null +++ b/packages/cli/test/cli/run-stdin.test.ts @@ -0,0 +1,499 @@ +import { describe, expect, test } from "bun:test" +import { spawn } from "node:child_process" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" + +const guard = path.join(import.meta.dir, "fixture/stdin-guard.ts") +const entries = ["index", "headless"] as const +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +async function run(input: { + entry: (typeof entries)[number] + name: string + args: string[] + stdin: "closed" | "open" | "late" + payload?: string + files?: Record + rootArgs?: string[] + deadlineMs?: number +}) { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "aictrl-stdin-")) + for (const [name, text] of Object.entries(input.files ?? {})) await fs.writeFile(path.join(directory, name), text) + const ledger = path.join(directory, "network.jsonl") + await fs.writeFile(ledger, "") + const requests: { method: string; path: string; body: Record | undefined }[] = [] + let stream: ReadableStreamDefaultController | undefined + const sessionID = "ses_stdin_fixture" + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + const url = new URL(request.url) + const body = request.method === "POST" ? ((await request.json()) as Record) : undefined + requests.push({ method: request.method, path: url.pathname, body }) + if (url.pathname === "/event") + return new Response( + new ReadableStream({ + start(controller) { + stream = controller + controller.enqueue(new TextEncoder().encode(": ready\n\n")) + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ) + if (url.pathname === "/session") return Response.json({ id: sessionID }) + if (url.pathname === `/session/${sessionID}/message` || url.pathname === `/session/${sessionID}/command`) { + stream?.enqueue( + new TextEncoder().encode( + `data: ${JSON.stringify({ type: "session.status", properties: { sessionID, status: { type: "idle" } } })}\n\n`, + ), + ) + return Response.json({}) + } + return Response.json({ unexpected: url.pathname }, { status: 500 }) + }, + }) + const env = { + PATH: [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter), + SHELL: "/bin/bash", + TZ: "UTC", + HOME: path.join(directory, "home"), + AICTRL_TEST_HOME: path.join(directory, "home"), + AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(directory, "managed"), + XDG_CONFIG_HOME: path.join(directory, "config"), + XDG_DATA_HOME: path.join(directory, "data"), + XDG_STATE_HOME: path.join(directory, "state"), + XDG_CACHE_HOME: path.join(directory, "cache"), + BUN_RUNTIME_TRANSPILER_CACHE_PATH: path.join(directory, "bun-cache"), + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + AICTRL_DISABLE_PROJECT_CONFIG: "true", + AICTRL_DISABLE_DEFAULT_PLUGINS: "true", + STDIN_FIXTURE_ORIGIN: server.url.origin, + STDIN_FIXTURE_LEDGER: ledger, + } + const command = [ + "run", + "--no-install", + "--no-env-file", + "--preload", + guard, + "--conditions=browser", + path.resolve(import.meta.dir, `../../src/${input.entry}.ts`), + ...(input.rootArgs ?? ["run", "--format", "json", "--attach", server.url.origin, ...input.args]), + ] + const child = spawn(process.execPath, command, { + cwd: directory, + env, + detached: true, + stdio: ["pipe", "pipe", "pipe"], + }) + let stdout = "" + let stderr = "" + let deadline = false + let kill = false + let signal: NodeJS.Signals | null = null + const started = performance.now() + const exists = () => { + try { + process.kill(-child.pid!, 0) + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false + throw error + } + } + const staged: ReturnType[] = [] + let firstBytesAt: number | undefined + let eofAt: number | undefined + let requestsBeforeEOF: number | undefined + let stagedInput = false + child.stdout.on("data", (data) => { + stdout += data.toString() + if (input.stdin === "late" && !stagedInput && stdout.includes('"type":"invocation_start"')) { + stagedInput = true + staged.push( + setTimeout(() => { + firstBytesAt = performance.now() - started + child.stdin.write(input.payload ?? "") + staged.push( + setTimeout(() => { + eofAt = performance.now() - started + requestsBeforeEOF = requests.length + child.stdin.end() + }, 150), + ) + }, 150), + ) + } + }) + child.stderr.on("data", (data) => { + stderr += data.toString() + }) + child.stdin.on("error", (error: NodeJS.ErrnoException) => { + if (error.code !== "EPIPE") stderr += String(error) + }) + if (input.payload && input.stdin !== "late") child.stdin.write(input.payload) + if (input.stdin === "closed") child.stdin.end() + const timer = setTimeout(() => { + deadline = true + if (exists()) process.kill(-child.pid!, "SIGTERM") + }, input.deadlineMs ?? 3000) + const escalation = setTimeout( + () => { + if (deadline && exists()) { + kill = true + process.kill(-child.pid!, "SIGKILL") + } + }, + (input.deadlineMs ?? 3000) + 200, + ) + let code: number | null = null + try { + code = await new Promise((resolve, reject) => { + child.once("error", reject) + child.once("close", (status, reason) => { + signal = reason + resolve(status) + }) + }) + } finally { + for (const timer of staged) clearTimeout(timer) + clearTimeout(timer) + clearTimeout(escalation) + child.stdin.destroy() + if (exists()) { + process.kill(-child.pid!, "SIGTERM") + await delay(150) + if (exists()) { + kill = true + process.kill(-child.pid!, "SIGKILL") + } + } + server.stop(true) + } + const groupGone = !exists() + const network = (await fs.readFile(ledger, "utf8")) + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)) + const result = { + ...input, + directory, + command, + code, + signal, + deadline, + kill, + groupGone, + milliseconds: performance.now() - started, + firstBytesAt, + eofAt, + requestsBeforeEOF, + stdout, + stderr, + requests, + network, + } + if (process.env.STDIN_TEST_EVIDENCE) { + await fs.mkdir(process.env.STDIN_TEST_EVIDENCE, { recursive: true }) + await fs.writeFile( + path.join(process.env.STDIN_TEST_EVIDENCE, `${input.entry}-${input.name}.json`), + JSON.stringify(result, null, 2) + "\n", + ) + } + if (deadline) throw new Error(`CLI deadline fired (never a successful rejection): ${JSON.stringify(result)}`) + if (!groupGone) throw new Error("Stdin test child process group remains") + if (network.some((row) => row.allowed === false)) + throw new Error("Stdin test attempted forbidden network: " + JSON.stringify(network)) + return result +} + +function message(result: Awaited>) { + const body = result.requests.find((request) => request.path.endsWith("/message"))?.body + return (body?.parts as { type: string; text?: string }[])?.find((part) => part.type === "text")?.text +} +function title(result: Awaited>) { + return result.requests.find((request) => request.path === "/session")?.body?.title +} + +describe("explicit stdin contract (#210)", () => { + for (const entry of entries) { + test(`${entry}: no-message EOF payload reaches actual SDK loopback`, async () => { + const result = await run({ + entry, + name: "payload-control", + args: [], + stdin: "closed", + payload: "fixture λ\nsecond line\n", + }) + expect(result.code, result.stderr + result.stdout).toBe(0) + expect(message(result)).toBe("\nfixture λ\nsecond line\n") + expect(title(result)).toBe("fixture λ\nsecond line") + }, 6000) + test(`${entry}: explicit prompt completes with unused open stdin`, async () => { + const result = await run({ + entry, + name: "explicit-open", + args: ["probe"], + stdin: "open", + payload: "must not append\n", + }) + expect(result.code, result.stderr + result.stdout).toBe(0) + expect(message(result)).toBe("probe") + expect(title(result)).toBe("probe") + }, 6000) + test(`${entry}: post-separator marker disappears from SDK text and title`, async () => { + const result = await run({ + entry, + name: "post-separator-marker", + args: ["summarize this", "--", "-"], + stdin: "closed", + payload: "context λ\n", + }) + expect(result.code, result.stderr + result.stdout).toBe(0) + // Structured operands preserve raw decoded prompt text; quotes are not added. + expect(message(result)).toBe("summarize this\ncontext λ\n") + expect(title(result)).toBe("summarize this") + }, 6000) + } +}) + +function failure(result: Awaited>, code: string) { + expect(result.code, result.stderr + result.stdout).toBe(1) + expect(result.requests).toHaveLength(0) + const events = result.stdout + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)) + expect(events.map((event) => event.type)).toEqual(["invocation_start", "invocation_error", "invocation_complete"]) + expect(events[1]).toMatchObject({ phase: "validation", code, invocationID: events[0].invocationID }) + expect(events[2]).toMatchObject({ status: "error", invocationID: events[0].invocationID }) +} + +describe("stdin operands and option ownership", () => { + for (const entry of entries) { + test.each([ + ["bare-marker", ["-"], "\nlate λ\n", "late λ"], + ["marker-among-words", ["review", "-", "this"], "review this\nlate λ\n", "review this"], + ["repeated-markers", ["-", "review", "-", "-"], "review\nlate λ\n", "review"], + ["post-markers", ["review", "--", "-", "-"], "review\nlate λ\n", "review"], + ["no-operands-late", [], "\nlate λ\n", "late λ"], + ] as const)( + `${entry}: %s reads late bytes through EOF exactly once`, + async (name, args, text, expectedTitle) => { + const result = await run({ entry, name, args: [...args], stdin: "late", payload: "late λ\n" }) + expect(result.code, result.stderr + result.stdout).toBe(0) + expect(result.firstBytesAt).toBeGreaterThan(100) + expect(result.eofAt!).toBeGreaterThan(result.firstBytesAt!) + expect(result.requestsBeforeEOF).toBe(0) + expect(message(result)).toBe(text) + expect(title(result)).toBe(expectedTitle) + expect(result.requests.map((request) => `${request.method} ${request.path}`).sort()).toEqual( + ["GET /event", "POST /session", "POST /session/ses_stdin_fixture/message"].sort(), + ) + }, + 6000, + ) + + test.each([ + ["empty-operand", [""], "INVOCATION_EMPTY_INPUT"], + ["no-operands-empty-eof", [], "INVOCATION_EMPTY_INPUT"], + ["marker-empty-eof", ["-"], "INVOCATION_EMPTY_INPUT"], + ] as const)( + `${entry}: %s keeps empty-input classification`, + async (name, args, code) => { + const result = await run({ + entry, + name, + args: [...args], + stdin: args.length && !(args as readonly string[]).includes("-") ? "open" : "closed", + }) + failure(result, code) + }, + 6000, + ) + + test.each([ + ["numeric", ["0007", "1e3"], "0007 1e3", "0007 1e3"], + ["quoted-unicode", ['say "hello" λ', "next\nline"], 'say "hello" λ next\nline', 'say "hello" λ next\nline'], + ["literal-embedded-dash", ["literal -"], "literal -", "literal -"], + ["post-separator-word", ["--", "--literal"], "--literal", "--literal"], + ["empty-explicit-title", ["--title", "", "two words"], "two words", "two words"], + ["hyphen-title-value", ["--title", "-", "probe"], "probe", "-"], + ["hyphen-variant-value", ["--variant", "-", "probe"], "probe", "probe"], + ["options-after-operand", ["probe", "--title", "explicit"], "probe", "explicit"], + ] as const)( + `${entry}: %s does not select unused stdin`, + async (name, args, text, expectedTitle) => { + const result = await run({ entry, name, args: [...args], stdin: "open", payload: "ignored pipe bytes\n" }) + expect(result.code, result.stderr + result.stdout).toBe(0) + expect(message(result)).toBe(text) + expect(title(result)).toBe(expectedTitle) + }, + 6000, + ) + + test.each(["--model", "-m"])( + `${entry}: %s owns a hyphen value without reading stdin`, + async (flag) => { + const result = await run({ entry, name: `model-ownership-${flag}`, args: [flag, "-", "probe"], stdin: "open" }) + expect(result.code).toBe(1) + expect(result.stderr).toContain('Invalid model identifier "-"') + expect(title(result)).toBe("probe") + expect(result.requests.some((request) => request.path.endsWith("/message"))).toBe(false) + }, + 6000, + ) + + test.each(["omitted", "marker"])( + `${entry}: selected %s whitespace EOF is empty input`, + async (kind) => { + const args = kind === "marker" ? ["-"] : [] + const result = await run({ entry, name: `whitespace-EOF-${kind}`, args, stdin: "late", payload: " \n\t" }) + failure(result, "INVOCATION_EMPTY_INPUT") + expect(result.requestsBeforeEOF).toBe(0) + }, + 6000, + ) + + test.each([ + [ + "command-with-marker", + ["--command", "check", "review", "-", "this"], + "late λ\n", + "check", + { argv: ["review", "this"], stdin: "late λ\n" }, + "/check review this", + ], + ["command-hyphen-value", ["--command", "-", "probe"], undefined, "-", { argv: ["probe"] }, "/- probe"], + [ + "command-empty-stdin", + ["--command", "check", "alpha", "-"], + "", + "check", + { argv: ["alpha"], stdin: "" }, + "/check alpha", + ], + [ + "command-no-operands", + ["--command", "check"], + "late λ\n", + "check", + { argv: [], stdin: "late λ\n" }, + "/check late λ", + ], + ] as const)( + `${entry}: %s preserves command ownership`, + async (name, args, payload, command, argumentsText, expectedTitle) => { + const result = await run({ + entry, + name, + args: [...args], + stdin: payload !== undefined ? "late" : "open", + payload, + }) + expect(result.code, result.stderr + result.stdout).toBe(0) + const body = result.requests.find((request) => request.path.endsWith("/command"))?.body + expect(body?.command).toBe(command) + // This fixture proves SDK wire shape only; the local-command suite proves handler assembly. + expect(body?.arguments).toEqual(argumentsText) + expect(title(result)).toBe(expectedTitle) + if (payload !== undefined) expect(result.requestsBeforeEOF).toBe(0) + }, + 6000, + ) + + test.each([ + ["file-greedy", ["-f", "a.txt", "b.txt", "--", "probe"], ["a.txt", "b.txt"]], + ["file-hyphen-equals", ["--file=-", "--", "probe"], ["-"]], + ["file-repeat", ["--file", "a.txt", "--file", "b.txt", "--", "probe"], ["a.txt", "b.txt"]], + ] as const)( + `${entry}: %s retains file option values`, + async (name, args, names) => { + const result = await run({ + entry, + name, + args: [...args], + stdin: "open", + files: { "a.txt": "a", "b.txt": "b", "-": "hyphen" }, + }) + expect(result.code, result.stderr + result.stdout).toBe(0) + const body = result.requests.find((request) => request.path.endsWith("/message"))?.body + const files = (body?.parts as { type: string; filename?: string }[]).filter((part) => part.type === "file") + expect(files.map((part) => part.filename)).toEqual([...names]) + expect(message(result)).toBe("probe") + }, + 6000, + ) + + test.each([ + ["missing-agent", ["--agent"]], + ["missing-session", ["--session"]], + ["missing-session-alias", ["-s"]], + ["missing-attach", ["--attach"]], + ["missing-dir", ["--dir"]], + ["missing-format", ["--format"]], + ["missing-output-result", ["--output-result"]], + ["missing-file-alias", ["-f"]], + ["missing-model", ["--model"]], + ["missing-model-alias", ["-m"]], + ["missing-title", ["--title"]], + ["missing-command", ["--command"]], + ["missing-file", ["--file"]], + ["orphan-file-hyphen", ["--file", "-", "probe"]], + ["orphan-file-alias", ["-f", "-", "probe"]], + ["missing-skill", ["--skill"]], + ["missing-schema", ["--output-schema"]], + ["missing-retries", ["--output-schema-retries"]], + ["missing-variant", ["--variant"]], + ["unknown-flag", ["--bogus", "probe"]], + ] as const)( + `${entry}: %s rejects before reading open stdin`, + async (name, args) => { + const result = await run({ entry, name, args: [...args], stdin: "open" }) + expect(result.code).toBe(1) + expect(result.requests).toHaveLength(0) + expect(result.stdout).not.toContain('"type":"invocation_start"') + }, + 6000, + ) + + test(`${entry}: optional no-value port leaves standalone marker selected`, async () => { + const result = await run({ + entry, + name: "port-marker", + args: ["--port", "-", "probe"], + stdin: "late", + payload: "late λ\n", + }) + expect(result.code, result.stderr + result.stdout).toBe(0) + expect(message(result)).toBe("probe\nlate λ\n") + expect(result.requestsBeforeEOF).toBe(0) + }, 6000) + test(`${entry}: skill nargs does not consume its following prompt`, async () => { + const result = await run({ entry, name: "skill-nargs", args: ["--skill", "alpha", "probe"], stdin: "open" }) + failure(result, "INVOCATION_INVALID_ARGUMENTS") + expect(result.stderr).toContain("--skill is not supported with --attach") + }, 6000) + test(`${entry}: root unknown commands remain strict`, async () => { + const result = await run({ + entry, + name: "root-unknown", + args: [], + rootArgs: ["no-such-command-210"], + stdin: "open", + }) + expect(result.code).toBe(1) + expect(result.requests).toHaveLength(0) + }, 6000) + } +}) + +// A killed subprocess must reject the helper, never satisfy an expected nonzero-result assertion. +test("stdin test deadline cannot pass as a rejected invocation", async () => { + await expect( + run({ entry: "index", name: "deadline-negative-control", args: ["-"], stdin: "open", deadlineMs: 600 }), + ).rejects.toThrow("CLI deadline fired (never a successful rejection)") +}, 6000) diff --git a/packages/cli/test/cli/run-tool-selection-attach.test.ts b/packages/cli/test/cli/run-tool-selection-attach.test.ts index 9205763c..52dc064c 100644 --- a/packages/cli/test/cli/run-tool-selection-attach.test.ts +++ b/packages/cli/test/cli/run-tool-selection-attach.test.ts @@ -340,7 +340,7 @@ describe("read-only attach canonical config resolution (#246)", () => { ["POST", "/session/ses_attachfixture/message"], ]) expect(result.requests[0].body).toEqual({ title: "Fixture attach prompt." }) - expect(result.requests[2].body).toEqual({ parts: [{ type: "text", text: '"Fixture attach prompt."\n' }] }) + expect(result.requests[2].body).toEqual({ parts: [{ type: "text", text: "Fixture attach prompt." }] }) expect(result.markers["plugin-marker"]).toBe(false) expect(result.markers[".aictrl/.gitignore"]).toBe(false) expect(result.markers[".aictrl/node_modules"]).toBe(false) diff --git a/packages/cli/test/cli/run-user-message.test.ts b/packages/cli/test/cli/run-user-message.test.ts index 221cec0b..80d88a3f 100644 --- a/packages/cli/test/cli/run-user-message.test.ts +++ b/packages/cli/test/cli/run-user-message.test.ts @@ -151,8 +151,8 @@ describe("run --format json user_message (#223)", () => { agent: "build", model: { providerID: "alibaba", modelID: "qwen-plus" }, sequenceNum: 1, - // run appends piped stdin (empty here) after a newline. - parts: [{ type: "text", text: "Hello\n", messageID: message.messageID, sessionID: message.sessionID }], + // An explicit prompt leaves unused stdin unread and adds no separator newline. + parts: [{ type: "text", text: "Hello", messageID: message.messageID, sessionID: message.sessionID }], }) expect(message.parts[0].synthetic).toBeUndefined() const text = result.events.find((event) => event.type === "text") @@ -190,7 +190,7 @@ describe("run --format json user_message (#223)", () => { expect(parts.filter((part) => part.synthetic === true).map((part) => part.text)).toEqual( expect.arrayContaining([expect.stringContaining("attached content")]), ) - expect(parts.at(-1)).toMatchObject({ type: "text", text: "Summarize\n" }) + expect(parts.at(-1)).toMatchObject({ type: "text", text: "Summarize" }) }, 25_000) test("carries the --command expansion", async () => { @@ -198,7 +198,7 @@ describe("run --format json user_message (#223)", () => { expect(result.exit, result.stderr).toBe(0) const [message, ...rest] = userMessageEvents(result.events) expect(rest).toEqual([]) - expect(message.parts).toMatchObject([{ type: "text", text: "Check input\n." }]) + expect(message.parts).toMatchObject([{ type: "text", text: "Check input." }]) }, 25_000) test("a StructuredOutput corrective turn adds no user_message: its reminder is never stored", async () => { @@ -209,7 +209,7 @@ describe("run --format json user_message (#223)", () => { expect(result.exit, result.stderr).toBe(0) expect(result.requests).toBe(2) expect(result.events.filter((event) => event.type === "structured_output_rejected")).toHaveLength(1) - expect(userMessageEvents(result.events)).toMatchObject([{ parts: [{ type: "text", text: "Hello\n" }] }]) + expect(userMessageEvents(result.events)).toMatchObject([{ parts: [{ type: "text", text: "Hello" }] }]) }, 25_000) test("emits the compaction request and its synthetic continue message", async () => { diff --git a/packages/cli/test/session/instruction-isolation.test.ts b/packages/cli/test/session/instruction-isolation.test.ts new file mode 100644 index 00000000..b9888e68 --- /dev/null +++ b/packages/cli/test/session/instruction-isolation.test.ts @@ -0,0 +1,157 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import path from "path" +import { InstructionPrompt } from "../../src/session/instruction" +import { Instance } from "../../src/project/instance" +import { Global } from "../../src/global" +import { HeadlessRun } from "../../src/flag/headless-run" +import { ReadTool } from "../../src/tool/read" +import { tmpdir } from "../fixture/fixture" + +describe("project instruction isolation (#250)", () => { + const keys = ["AICTRL_DISABLE_PROJECT_CONFIG", "AICTRL_CONFIG_DIR", "AICTRL_CONFIG_CONTENT"] as const + let env: (string | undefined)[] + let config: string + let headless: boolean + + beforeEach(() => { + env = keys.map((key) => process.env[key]) + config = Global.Path.config + headless = HeadlessRun.enabled() + for (const key of keys) delete process.env[key] + }) + + afterEach(() => { + keys.forEach((key, index) => { + if (env[index] === undefined) delete process.env[key] + else process.env[key] = env[index] + }) + Global.Path.config = config + if (headless) HeadlessRun.enable() + else HeadlessRun.disable() + }) + + test.each([false, true])("disabled=%s controls root and .opencode automatic instructions", async (disabled) => { + await using global = await tmpdir() + Global.Path.config = global.path + if (disabled) process.env.AICTRL_DISABLE_PROJECT_CONFIG = "true" + await using tmp = await tmpdir({ + init: async (dir) => { + await Bun.write(path.join(dir, "AGENTS.md"), "ROOT_ISOLATION_MARKER") + await Bun.write(path.join(dir, ".opencode", "AGENTS.md"), "OPENCODE_ISOLATION_MARKER") + }, + }) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const paths = await InstructionPrompt.systemPaths() + expect(paths.has(path.join(tmp.path, "AGENTS.md"))).toBe(!disabled) + expect(paths.has(path.join(tmp.path, ".opencode", "AGENTS.md"))).toBe(!disabled) + const instructions = (await InstructionPrompt.system()).join("\n") + expect(instructions.includes("ROOT_ISOLATION_MARKER")).toBe(!disabled) + expect(instructions.includes("OPENCODE_ISOLATION_MARKER")).toBe(!disabled) + }, + }) + }) + + test.each([ + [false, "resolve"], + [true, "resolve"], + [false, "read"], + [true, "read"], + ] as const)("disabled=%s controls nested %s instructions", async (disabled, mode) => { + await using global = await tmpdir() + Global.Path.config = global.path + if (disabled) process.env.AICTRL_DISABLE_PROJECT_CONFIG = "true" + await using tmp = await tmpdir({ + init: async (dir) => { + await Bun.write(path.join(dir, "nested", "AGENTS.md"), "NESTED_ISOLATION_MARKER") + await Bun.write(path.join(dir, "nested", "file.txt"), "ordinary file content") + }, + }) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const filepath = path.join(tmp.path, "nested", "file.txt") + if (mode === "resolve") { + const results = await InstructionPrompt.resolve([], filepath, "resolve-isolation") + expect( + results + .map((item) => item.content) + .join("\n") + .includes("NESTED_ISOLATION_MARKER"), + ).toBe(!disabled) + return + } + const read = await ReadTool.init() + const result = await read.execute( + { filePath: filepath }, + { + sessionID: "isolation-test", + messageID: "read-isolation", + callID: "read-isolation", + agent: "build", + abort: AbortSignal.any([]), + messages: [], + metadata: () => {}, + ask: async () => {}, + }, + ) + expect(result.output).toContain("ordinary file content") + expect(result.output.includes("NESTED_ISOLATION_MARKER")).toBe(!disabled) + expect(result.output.includes("")).toBe(!disabled) + expect(result.metadata.loaded).toEqual(disabled ? [] : [path.join(tmp.path, "nested", "AGENTS.md")]) + }, + }) + }) + + test.each([false, true])( + "headless=%s retains trusted and explicit caller instructions when disabled", + async (active) => { + if (active) HeadlessRun.enable() + else HeadlessRun.disable() + process.env.AICTRL_DISABLE_PROJECT_CONFIG = "true" + await using global = await tmpdir({ + init: async (dir) => { + await Bun.write(path.join(dir, "AGENTS.md"), "GLOBAL_ISOLATION_MARKER") + }, + }) + Global.Path.config = global.path + await using profile = await tmpdir({ + init: async (dir) => { + await Bun.write(path.join(dir, "AGENTS.md"), "PROFILE_ISOLATION_MARKER") + await Bun.write(path.join(dir, "relative.md"), "RELATIVE_ISOLATION_MARKER") + }, + }) + await using caller = await tmpdir({ + init: async (dir) => { + await Bun.write(path.join(dir, "caller.md"), "CALLER_ISOLATION_MARKER") + }, + }) + process.env.AICTRL_CONFIG_CONTENT = JSON.stringify({ instructions: [path.join(caller.path, "caller.md")] }) + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const instructions = (await InstructionPrompt.system()).join("\n") + expect(instructions).toContain("GLOBAL_ISOLATION_MARKER") + expect(instructions).toContain("CALLER_ISOLATION_MARKER") + }, + }) + process.env.AICTRL_CONFIG_DIR = profile.path + process.env.AICTRL_CONFIG_CONTENT = JSON.stringify({ + instructions: ["relative.md", path.join(caller.path, "caller.md")], + }) + await using other = await tmpdir() + await Instance.provide({ + directory: other.path, + fn: async () => { + const instructions = (await InstructionPrompt.system()).join("\n") + expect(instructions).toContain("PROFILE_ISOLATION_MARKER") + expect(instructions).not.toContain("GLOBAL_ISOLATION_MARKER") + expect(instructions).toContain("RELATIVE_ISOLATION_MARKER") + expect(instructions).toContain("CALLER_ISOLATION_MARKER") + }, + }) + }, + ) +}) diff --git a/packages/sdk/script/build.ts b/packages/sdk/script/build.ts index 4d9e3879..29b6c447 100755 --- a/packages/sdk/script/build.ts +++ b/packages/sdk/script/build.ts @@ -79,6 +79,17 @@ for (const name of ["EventSessionStructuredOutput", "EventSessionStructuredOutpu throw new Error("Legacy Event declaration not found; update the SDK schema bridge") source = source.replace("export type Event =\n", `export type Event =\n | ${name}\n`) } +// Command invocation keeps the legacy endpoint, but uses the canonical arguments component. +const command = source.indexOf("export type SessionCommandData = {") +const pathStart = source.indexOf("\n path:", command) +if (command < 0 || pathStart < 0) throw new Error("Legacy command body not found; update the SDK schema bridge") +const body = source.slice(command, pathStart) +const argument = / arguments: (?:string|import\("\.\.\/v2\/gen\/types\.gen\.js"\)\.CommandArguments)/ +if (!argument.test(body)) throw new Error("Legacy command arguments not found; update the SDK schema bridge") +source = + source.slice(0, command) + + body.replace(argument, ' arguments: import("../v2/gen/types.gen.js").CommandArguments') + + source.slice(pathStart) await legacy.write(source) await $`bun prettier --write src/gen` diff --git a/packages/sdk/src/gen/types.gen.ts b/packages/sdk/src/gen/types.gen.ts index ee8ff3b7..fed84832 100644 --- a/packages/sdk/src/gen/types.gen.ts +++ b/packages/sdk/src/gen/types.gen.ts @@ -2478,7 +2478,7 @@ export type SessionCommandData = { messageID?: string agent?: string model?: string - arguments: string + arguments: import("../v2/gen/types.gen.js").CommandArguments command: string } path: { diff --git a/packages/sdk/src/v2/gen/types.gen.ts b/packages/sdk/src/v2/gen/types.gen.ts index 017df7d0..adeb0357 100644 --- a/packages/sdk/src/v2/gen/types.gen.ts +++ b/packages/sdk/src/v2/gen/types.gen.ts @@ -4,6 +4,73 @@ export type ClientOptions = { baseUrl: `${string}://${string}` | (string & {}) } +export type CommandArguments = + | string + | { + argv: Array + stdin?: string + } + +export type CommandInput = { + messageID?: string + sessionID: string + agent?: string + model?: string + arguments: CommandArguments + command: string + toolAllowlist?: Array + format?: OutputFormat + variant?: string + parts?: Array<{ + id?: string + type: "file" + mime: string + filename?: string + url: string + source?: + | { + text: { + value: string + start: number + end: number + } + type: "file" + path: string + } + | { + text: { + value: string + start: number + end: number + } + type: "symbol" + path: string + range: { + start: { + line: number + character: number + } + end: { + line: number + character: number + } + } + name: string + kind: number + } + | { + text: { + value: string + start: number + end: number + } + type: "resource" + clientName: string + uri: string + } + }> +} + export type StepFinishPart = { id: string sessionID: string From 6808855322ba6f0c7e5456cd323ad2b3b9b8cf08 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 21 Oct 2026 12:36:42 +0100 Subject: [PATCH 3/7] fix(ci): harden token budget capture and evidence --- RELEASING.md | 2 +- script/token-budget/README.md | 8 +- script/token-budget/release-count.py | 7 +- .../release-fixtures/token-stub.test.ts | 89 +++++++ .../release-fixtures/token-stub.ts | 2 +- script/token-budget/run.py | 243 +++++++++++++++--- script/token-budget/test_protocol.py | 147 ++++++++++- 7 files changed, 450 insertions(+), 48 deletions(-) create mode 100644 script/token-budget/release-fixtures/token-stub.test.ts diff --git a/RELEASING.md b/RELEASING.md index 218fe148..09a0d6b2 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -18,7 +18,7 @@ and network access during setup only), then capture and count offline: ```bash - python3.12 script/token-budget/setup.py --directory /absolute/new-counter-env + python3.12 -I -B script/token-budget/setup.py --directory /absolute/new-counter-env bun run --no-install --no-env-file script/token-budget/release.ts /absolute/release-inputs.json /absolute/new-release-evidence /absolute/new-counter-env/venv/bin/python3 -I -B script/token-budget/release-count.py /absolute/new-release-evidence/campaign.json ``` diff --git a/script/token-budget/README.md b/script/token-budget/README.md index 88db0714..d5a7c934 100644 --- a/script/token-budget/README.md +++ b/script/token-budget/README.md @@ -16,9 +16,13 @@ python3.12 -I -B script/token-budget/setup.py --directory /tmp/my-token-environm The guard uses an exclusive nonblocking lock in the owner-only `/tmp/aictrl-token-budget` directory. Concurrent invocations fail before changing the fixed workspace. Each cold child runs in the literal `/tmp/aictrl-token-budget/workspace`, newly initialized as a Git repository, with UTC and Date fixed before production imports to 21 October 2026. This controls all three raw cwd occurrences, including the two in Bash descriptions/schema. No text normalization is applied. Existing unowned, shared, or symlink fixture directories are rejected. The directory/lock remain for safe reuse; raw captures remain in the requested evidence directory. -Only loopback fixture HTTP is allowed by the before-import fetch/preconnect/install guard reused from the selector fixture. Child environments are explicitly allowlisted, without provider credentials or inherited project configuration. The real CLI has a mandatory 15-second deadline; the outer owned process group has 25 seconds and rejects surviving descendants even if its parent exits successfully. These fixture controls are not an OS network sandbox. +The before-import guard observes `globalThis.fetch` calls in the CLI process and rejects origins other than the loopback fixture. It rejects `fetch.preconnect` and direct `Bun.spawn` calls whose executable is `bun` and whose next argument is exactly `add` or `install`. This ledger is partial: other network APIs, `Bun.spawnSync`, Node child-process APIs, shell-launched programs, and package-manager aliases are outside those controls. A clean ledger establishes no rejected operation on the instrumented paths; it does not establish complete network or installation accounting. Child environments are explicitly allowlisted, without provider credentials or inherited project configuration. These trusted fixture controls are not an OS network sandbox. -`count.py` verifies the pinned tokenizer version, exact rank SHA256, raw environment, catalog, single request and clean process/network evidence. `reference.py` denies socket/DNS use during accounting. The enforced metric is `o200k_base` encoding of compact sorted Unicode JSON containing actual SDK system-role messages, serialized tools, and instructions when present. The fixture user text is excluded. The full first model input is separately reported using `instructions/messages/input/tools/response_format/text`; transport controls are excluded. System/tool diagnostic counts use separate tokenizations and are not additive BPE contributions. +The real CLI has a mandatory 15-second deadline; each outer capture has 25 seconds. A fresh forked Linux subreaper owns only that invocation's capture and adopted descendants, including descendants that create a new session. After the capture parent exits, already-dead children are reaped without being classified as live survivors. Live adopted children are terminated and reaped, and the capture is rejected even if its parent exited successfully. Cleanup repeatedly handles newly adopted children and fails if it cannot verify completion within its cleanup deadline. The caller's other children are outside this ownership scope. The result records the supervisor and descendant cleanup evidence; the legacy `remaining_group_killed` field now indicates any observed live adopted descendant, not just the original process group. + +Evidence output must be a new directory under an existing parent. Its root and per-run directories are owner-only (`0700`), so raw fixture requests and allowlisted environment records are not exposed by an ambient permissive umask. Existing output directories are never overwritten. The fixed workspace directory uses the same ownership/private-directory validation in production and lock controls. + +`count.py` verifies the pinned tokenizer version, exact rank SHA256, raw environment, catalog, single request and the recorded process/partial-fetch-ledger evidence. `reference.py` denies socket/DNS use during accounting. The enforced metric is `o200k_base` encoding of compact sorted Unicode JSON containing actual SDK system-role messages, serialized tools, and instructions when present. The fixture user text is excluded. The full first model input is separately reported using `instructions/messages/input/tools/response_format/text`; transport controls are excluded. System/tool diagnostic counts use separate tokenizations and are not additive BPE contributions. `budgets.json` contains two fixed explicit ceilings. Raising either is a reviewed source edit requiring raw evidence and an explanation. `--observe` records an evaluation without enforcing ceilings and is never used by CI. `--source` selects an isolated source copy for QA source-growth controls; CI always uses the repository entry. The default two cold repetitions must produce identical canonical prefix bytes for each arm. diff --git a/script/token-budget/release-count.py b/script/token-budget/release-count.py index 2cdcd19f..a5059889 100644 --- a/script/token-budget/release-count.py +++ b/script/token-budget/release-count.py @@ -17,7 +17,12 @@ def analyze(campaign_path): raise ValueError('Refuse overwriting accounting results') if campaign['protocolVersion'] != 3 or campaign['failures'] or campaign['REPS'] < 2: raise ValueError('Incomplete or failed release campaign') - final = campaign['cleanupChecks'][-1]['finalCampaignCheck'] + checks = campaign.get('cleanupChecks') + if not isinstance(checks, list) or not checks or not isinstance(checks[-1], dict): + raise ValueError('Incomplete release campaign: final cleanup evidence missing') + final = checks[-1].get('finalCampaignCheck') + if not isinstance(final, dict) or not {'exit', 'timedOut', 'stdout'} <= final.keys(): + raise ValueError('Incomplete release campaign: final cleanup evidence missing') if final['exit'] != 0 or final['timedOut'] or final['stdout'].strip(): raise ValueError('Final campaign container cleanup not verified') encoding = load_encoding() diff --git a/script/token-budget/release-fixtures/token-stub.test.ts b/script/token-budget/release-fixtures/token-stub.test.ts new file mode 100644 index 00000000..a3d04292 --- /dev/null +++ b/script/token-budget/release-fixtures/token-stub.test.ts @@ -0,0 +1,89 @@ +import { expect, test } from "bun:test" +import { MARKER, Stub } from "./token-stub" + +for (const api of ["chat", "responses"] as const) { + const body = (main: boolean) => ({ + model: "fixture", + ...(api === "chat" + ? { + messages: [ + { role: "user", content: main ? `${MARKER} short` : "Generate a title" }, + { role: "tool", tool_call_id: "foreign_history_id", content: "Earlier unrelated result" }, + ], + tools: main + ? [ + { + type: "function", + function: { + name: "read", + parameters: { + type: "object", + properties: { filePath: { type: "string" } }, + required: ["filePath"], + }, + }, + }, + ] + : [], + } + : { + input: [ + { role: "user", content: main ? `${MARKER} short` : "Generate a title" }, + { type: "function_call_output", call_id: "foreign_history_id", output: "Earlier unrelated result" }, + ], + tools: main + ? [ + { + type: "function", + name: "read", + parameters: { type: "object", properties: { filePath: { type: "string" } }, required: ["filePath"] }, + }, + ] + : [], + }), + }) + + test(`${api}: auxiliary foreign history does not advance or reject the main script`, async () => { + const stub = new Stub(0) + stub.reset("short") + try { + const response = await fetch( + new URL(api === "chat" ? "/v1/chat/completions" : "/v1/responses", stub.server.url), + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body(false)), + }, + ) + expect(response.status).toBe(200) + expect(await response.text()).toContain("Bench title") + expect(stub.log).toHaveLength(1) + expect(stub.log[0]).toMatchObject({ kind: "aux", step: 0 }) + expect(stub.log[0].toolResults?.[0]?.callId).toBe("foreign_history_id") + expect(stub.log.some((row) => row.error)).toBe(false) + } finally { + await stub.server.stop(true) + } + }) + + test(`${api}: main foreign history still fails explicit scripted-ID validation`, async () => { + const stub = new Stub(0) + stub.reset("short") + try { + const response = await fetch( + new URL(api === "chat" ? "/v1/chat/completions" : "/v1/responses", stub.server.url), + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body(true)), + }, + ) + expect(response.status).toBe(400) + expect(await response.text()).toContain("Tool results lost all scripted call IDs") + expect(stub.log.some((row) => row.error?.includes("Tool results lost all scripted call IDs"))).toBe(true) + expect(stub.log.some((row) => row.kind === "main")).toBe(false) + } finally { + await stub.server.stop(true) + } + }) +} diff --git a/script/token-budget/release-fixtures/token-stub.ts b/script/token-budget/release-fixtures/token-stub.ts index bb62a9cd..71d1a901 100644 --- a/script/token-budget/release-fixtures/token-stub.ts +++ b/script/token-budget/release-fixtures/token-stub.ts @@ -151,7 +151,7 @@ export class Stub { const tools = toolList(json, api) const main = tools.length > 0 && body.includes(MARKER) const resultEvidence = fixtureEvidence(json, api, this.scenario) - const done = scriptProgress(resultEvidence) + const done = main ? scriptProgress(resultEvidence) : 0 const furthest = Math.max(0, ...this.log.filter((r) => r.kind === "main").map((r) => r.step)) if (main && done < furthest) throw new Error(`Tool history regressed from scripted step ${furthest} to ${done}`) const turn = main ? this.next(done, tools) : { text: "Bench title" } diff --git a/script/token-budget/run.py b/script/token-budget/run.py index 49b49d57..8f424768 100644 --- a/script/token-budget/run.py +++ b/script/token-budget/run.py @@ -2,6 +2,7 @@ """Linux-only public CLI SDK capture; no downloads or model traffic.""" import argparse import ctypes +import errno import fcntl import json import os @@ -12,57 +13,220 @@ import subprocess import sys import time +import traceback ROOT = Path(__file__).resolve().parent FIXTURE = Path('/tmp/aictrl-token-budget') WORKSPACE = FIXTURE / 'workspace' -def execute(command, cwd, env, limit=25): - child = subprocess.Popen(command, cwd=cwd, env=env, stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) - timed = False - try: - stdout, stderr = child.communicate(timeout=limit) - except subprocess.TimeoutExpired: - timed = True - os.killpg(child.pid, signal.SIGKILL) - stdout, stderr = child.communicate(timeout=5) +def _reap_descendants(limit=2): + # Called only in a fresh forked subreaper: it has no caller-owned children. + # waitid(WNOWAIT) proves each PID is our unreaped direct child. Until we + # explicitly waitpid it, even an exiting child cannot release/reuse its PID. + end = time.monotonic() + limit surviving = False - try: - os.killpg(child.pid, 0) - surviving = True - os.killpg(child.pid, signal.SIGKILL) - except ProcessLookupError: - pass - # As a Linux subreaper, reap orphaned members after terminating our owned group. - end = time.monotonic() + 2 + records = [] while True: - try: - pid, _ = os.waitpid(-child.pid, os.WNOHANG) - if not pid and time.monotonic() < end: - time.sleep(0.01) + children = Path(f'/proc/self/task/{os.getpid()}/children').read_text().split() + if not children: + try: + os.waitpid(-1, os.WNOHANG) + except ChildProcessError: + return surviving, records + raise RuntimeError('Owned child inventory incomplete; refusing unproven cleanup') + for value in children: + pid = int(value) + state = os.waitid(os.P_PID, pid, os.WEXITED | os.WNOHANG | os.WNOWAIT) + if state is not None: + os.waitpid(pid, 0) + records.append({'pid': pid, 'state': 'reaped', 'owner': os.getpid()}) continue - if not pid: - break - except ChildProcessError: - break + surviving = True + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + # Still unreaped: this identity cannot have become a foreign PID. + pass + records.append({'pid': pid, 'state': 'live-killed', 'owner': os.getpid()}) + if time.monotonic() >= end: + raise RuntimeError('Owned descendants remain after cleanup deadline: ' + ','.join(children)) + time.sleep(0.005) + + +def _capture(command, cwd, env, limit, cancelled): + child = None + primary = None + timed = False + surviving = False + records = [] + stdout = stderr = '' try: - os.killpg(child.pid, 0) - raise RuntimeError('Owned process group remains after cleanup') - except ProcessLookupError: - pass + child = subprocess.Popen(command, cwd=cwd, env=env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) + end = time.monotonic() + limit + while child.poll() is None: + if cancelled(): + raise InterruptedError('Capture supervisor interrupted') + remaining = end - time.monotonic() + if remaining <= 0: + timed = True + break + try: + stdout, stderr = child.communicate(timeout=min(0.05, remaining)) + except subprocess.TimeoutExpired: + # Recheck the actual parent, even while a descendant holds a pipe. + pass + except BaseException as error: + primary = error + raise + finally: + errors = [] + if child is not None and child.poll() is None: + try: + child.kill() + except ProcessLookupError: + pass + except Exception as error: + errors.append(error) + try: + child.wait(timeout=5) + except Exception as error: + errors.append(error) + try: + surviving, records = _reap_descendants() + except Exception as error: + errors.append(error) + if child is not None: + try: + stdout, stderr = child.communicate(timeout=5) + except Exception as error: + errors.append(error) + finally: + for stream in (child.stdout, child.stderr): + if stream is not None: + try: + stream.close() + except Exception as error: + errors.append(error) + if errors: + detail = 'Owned capture cleanup failed: ' + '; '.join(str(error) for error in errors) + if primary is not None: + primary.add_note(detail) + else: + raise RuntimeError(detail) from errors[0] return {'exit': child.returncode, 'timedOut': timed, 'remaining_group_killed': surviving, - 'process_group': child.pid, 'command': command}, stdout, stderr + 'process_group': child.pid, 'supervisor': os.getpid(), + 'descendant_cleanup': records, 'command': command}, stdout, stderr + + +def execute(command, cwd, env, limit=25): + # A separate fork is an ownership boundary, not a network/security sandbox. + # Its only children are the capture and descendants adopted from that capture. + read, write = os.pipe() + try: + supervisor = os.fork() + except BaseException: + os.close(read) + os.close(write) + raise + if supervisor == 0: + status = 1 + try: + os.close(read) + interrupted = False + + def cancel(number, frame): + nonlocal interrupted + interrupted = True + + try: + # Do not retain the caller's unrelated pipes/files in this fork. + for value in os.listdir('/proc/self/fd'): + descriptor = int(value) + if descriptor > 2 and descriptor != write: + try: + os.close(descriptor) + except OSError as error: + if error.errno != errno.EBADF: + raise + signal.signal(signal.SIGTERM, cancel) + signal.signal(signal.SIGINT, cancel) + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), 'Unable to establish owned Linux child subreaper') + result, stdout, stderr = _capture(command, cwd, env, limit, lambda: interrupted) + payload = {'result': result, 'stdout': stdout, 'stderr': stderr} + except BaseException as error: + payload = {'error': {'type': type(error).__name__, 'message': str(error), + 'errno': getattr(error, 'errno', None), 'strerror': getattr(error, 'strerror', None), + 'filename': getattr(error, 'filename', None), + 'traceback': ''.join(traceback.format_exception(error))}} + with os.fdopen(write, 'w') as stream: + json.dump(payload, stream) + status = 0 if 'result' in payload else 1 + finally: + os._exit(status) + os.close(write) + with os.fdopen(read) as stream: + primary = None + try: + # Drain before waitpid: stdout/stderr can exceed the result pipe buffer. + payload = json.load(stream) + except BaseException as error: + primary = error + try: + os.kill(supervisor, signal.SIGTERM) + except ProcessLookupError: + pass + except Exception as cleanup_error: + primary.add_note('Supervisor cancellation failed: ' + str(cleanup_error)) + try: + # Finish owned cleanup and drain its reply before waiting. + stream.read() + except Exception as cleanup_error: + primary.add_note('Supervisor reply drain failed: ' + str(cleanup_error)) + raise + finally: + try: + _, status = os.waitpid(supervisor, 0) + except Exception as cleanup_error: + if primary is None: + raise + primary.add_note('Supervisor reap failed: ' + str(cleanup_error)) + if 'error' in payload: + error = payload['error'] + if error['errno'] is not None: + failure = OSError(error['errno'], error['strerror'], error['filename']) + else: + failure = RuntimeError('Capture supervisor failed: ' + error['type'] + ': ' + error['message']) + failure.add_note(error['traceback']) + raise failure + if not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0: + raise RuntimeError('Capture supervisor exited without successful cleanup: ' + str(status)) + return payload['result'], payload['stdout'], payload['stderr'] def verify(result): if result['remaining_group_killed']: - raise RuntimeError('Owned process group survived controller exit; capture rejected after cleanup') + raise RuntimeError('Owned descendant survived controller exit; capture rejected after cleanup') if result['exit'] or result['timedOut']: raise RuntimeError('Capture failed or exceeded deadline: ' + json.dumps(result)) +def private_directory(path, reuse=False): + try: + path.mkdir(mode=0o700) + except FileExistsError: + if not reuse: + raise + info = path.lstat() + if (not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077 + or (not reuse and stat.S_IMODE(info.st_mode) != 0o700)): + raise RuntimeError('Fixture/evidence directory must be an owned private directory, never a symlink') + return path + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--output', type=Path, required=True, help='New evidence directory; never overwritten') @@ -78,29 +242,24 @@ def main(): bun = shutil.which('bun') if not bun or not shutil.which('rg'): raise RuntimeError('Require real bun and ripgrep on PATH; install prerequisites before offline capture') - if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0: - raise OSError('Unable to establish Linux child subreaper') - try: - FIXTURE.mkdir(mode=0o700) - except FileExistsError: - info = FIXTURE.lstat() - if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077: - raise RuntimeError('Fixed fixture must be an owned private directory, never a symlink') + private_directory(FIXTURE, reuse=True) fd = os.open(FIXTURE / 'lock', os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) with os.fdopen(fd, 'a') as lock: try: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: raise RuntimeError('Exclusive fixed workspace is already in use') + if args.output.is_symlink(): + raise ValueError('Evidence output must be a new directory, never a symlink') output = args.output.resolve() if output.is_relative_to(FIXTURE): raise ValueError('Evidence output must be outside the fixed fixture directory') - output.mkdir(parents=True, exist_ok=False) + private_directory(output) records = [] for arm in ['default', 'four']: for repeat in range(1, args.repeats + 1): out = output / f'{arm}-{repeat}' - out.mkdir() + private_directory(out) if WORKSPACE.is_symlink(): raise RuntimeError('Refusing a symlink fixed workspace') if WORKSPACE.exists(): diff --git a/script/token-budget/test_protocol.py b/script/token-budget/test_protocol.py index a5199930..e7de55e8 100644 --- a/script/token-budget/test_protocol.py +++ b/script/token-budget/test_protocol.py @@ -11,6 +11,7 @@ import sys import tempfile import unittest +from unittest import mock ROOT = Path(__file__).resolve().parent BUN = shutil.which('bun') @@ -61,6 +62,134 @@ def test_surviving_group_rejected_even_after_parent_exit_zero(self): with self.assertRaises(ProcessLookupError): os.killpg(result['process_group'], 0) + def test_setsid_child_with_inherited_output_is_owned_rejected_and_reaped(self): + source = """import os,time +read,write=os.pipe() +pid=os.fork() +if pid==0: + os.close(read);os.setsid();os.write(write,b'ready');os.close(write);time.sleep(10);os._exit(0) +os.close(write);os.read(read,5);os.close(read);print(pid,flush=True) +""" + result, stdout, _ = RUN.execute([sys.executable, '-I', '-c', source], self.directory, self.env, 2) + self.assertEqual(result['exit'], 0) + self.assertFalse(result['timedOut']) + self.assertTrue(result['remaining_group_killed']) + self.assertNotEqual(result['supervisor'], os.getpid()) + pid = int(stdout.strip()) + self.assertTrue(any(row['pid'] == pid and row['state'] == 'live-killed' + and row['owner'] == result['supervisor'] for row in result['descendant_cleanup'])) + with self.assertRaisesRegex(RuntimeError, 'survived controller exit'): + RUN.verify(result) + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + + def test_setsid_child_and_grandchild_are_repeatedly_adopted_and_reaped(self): + source = """import os,time,json +read,write=os.pipe() +pid=os.fork() +if pid==0: + os.close(read);os.setsid() + grand=os.fork() + if grand==0: + os.setsid();os.write(write,(str(os.getpid())+'\\n').encode());os.close(write);time.sleep(10);os._exit(0) + os.close(write);time.sleep(10);os._exit(0) +os.close(write) +with os.fdopen(read) as stream: grand=int(stream.readline()) +print(json.dumps([pid,grand]),flush=True) +""" + result, stdout, _ = RUN.execute([sys.executable, '-I', '-c', source], self.directory, self.env, 2) + self.assertFalse(result['timedOut']) + self.assertTrue(result['remaining_group_killed']) + pids = json.loads(stdout) + killed = {row['pid'] for row in result['descendant_cleanup'] if row['state'] == 'live-killed'} + self.assertTrue(set(pids) <= killed) + for pid in pids: + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + + def test_zombie_only_adoptee_does_not_reject_a_clean_capture(self): + source = """import os +pid=os.fork() +if pid==0: os._exit(0) +os.waitid(os.P_PID,pid,os.WEXITED|os.WNOWAIT) +print(pid,flush=True) +""" + result, stdout, _ = RUN.execute([sys.executable, '-I', '-c', source], self.directory, self.env, 2) + RUN.verify(result) + self.assertFalse(result['remaining_group_killed']) + pid = int(stdout.strip()) + self.assertIn({'pid': pid, 'state': 'reaped', 'owner': result['supervisor']}, result['descendant_cleanup']) + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + + def test_caller_unrelated_live_child_is_not_signalled_reaped_or_waited(self): + foreign = subprocess.Popen([sys.executable, '-I', '-c', 'import time;time.sleep(10)'], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + identity = Path(f'/proc/{foreign.pid}/stat').read_text().rsplit(')', 1)[1].split()[19] + result, stdout, _ = RUN.execute([sys.executable, '-I', '-c', 'print("owned")'], + self.directory, self.env, 2) + RUN.verify(result) + self.assertEqual(stdout.strip(), 'owned') + self.assertIsNone(foreign.poll()) + self.assertEqual(Path(f'/proc/{foreign.pid}/stat').read_text().rsplit(')', 1)[1].split()[19], identity) + self.assertNotIn(foreign.pid, [row['pid'] for row in result['descendant_cleanup']]) + finally: + foreign.kill() + foreign.wait(timeout=3) + + def test_result_pipe_drains_output_larger_than_pipe_capacity(self): + source = 'import sys;sys.stdout.write("λ"*262144);sys.stderr.write("x"*262144)' + result, stdout, stderr = RUN.execute([sys.executable, '-I', '-c', source], self.directory, self.env, 2) + RUN.verify(result) + self.assertEqual(stdout, 'λ' * 262144) + self.assertEqual(stderr, 'x' * 262144) + + def test_root_spawn_error_remains_actionable(self): + with self.assertRaises(FileNotFoundError): + RUN.execute([str(self.directory / 'missing-owned-command')], self.directory, self.env, 2) + + def test_cleanup_failure_preserves_primary_spawn_error_with_diagnostic(self): + with mock.patch.object(RUN, '_reap_descendants', side_effect=RuntimeError('owned cleanup probe failed')): + with self.assertRaises(FileNotFoundError) as failure: + RUN.execute([str(self.directory / 'missing-owned-command')], self.directory, self.env, 2) + self.assertIn('owned cleanup probe failed', '\n'.join(failure.exception.__notes__)) + + def test_cleanup_failure_rejects_an_otherwise_successful_capture(self): + with mock.patch.object(RUN, '_reap_descendants', side_effect=RuntimeError('owned cleanup probe failed')): + with self.assertRaisesRegex(RuntimeError, 'Owned capture cleanup failed'): + RUN.execute([sys.executable, '-I', '-c', 'print("clean")'], self.directory, self.env, 2) + + def test_private_evidence_directory_ignores_permissive_umask_and_refuses_overwrite(self): + previous = os.umask(0) + try: + output = RUN.private_directory(self.directory / 'evidence') + arm = RUN.private_directory(output / 'default-1') + finally: + os.umask(previous) + self.assertEqual(output.stat().st_mode & 0o777, 0o700) + self.assertEqual(arm.stat().st_mode & 0o777, 0o700) + marker = output / 'accepted.json' + marker.write_text('retained') + with self.assertRaises(FileExistsError): + RUN.private_directory(output) + self.assertEqual(marker.read_text(), 'retained') + + def test_reused_fixture_directory_rejects_shared_mode_and_symlinks(self): + shared = self.directory / 'shared' + shared.mkdir(mode=0o700) + shared.chmod(0o755) + with self.assertRaisesRegex(RuntimeError, 'owned private directory'): + RUN.private_directory(shared, reuse=True) + target = self.directory / 'owned' + RUN.private_directory(target) + link = self.directory / 'link' + link.symlink_to(target, target_is_directory=True) + with self.assertRaisesRegex(RuntimeError, 'never a symlink'): + RUN.private_directory(link, reuse=True) + with self.assertRaises(FileExistsError): + RUN.private_directory(link) + def test_before_import_guard_rejects_network_preconnect_and_install(self): guard = self.directory / 'guard.ts' env = {**self.env, 'GUARD_MODULE': str(ROOT / 'guard.ts'), 'GUARD_FILE': str(guard), @@ -91,7 +220,7 @@ def test_before_import_guard_rejects_network_preconnect_and_install(self): self.assertEqual(ledger, [{'origin': 'https://example.invalid', 'path': '/offline-probe', 'allowed': False}]) def test_fixed_workspace_lock_rejects_parallel_campaign_before_output(self): - RUN.FIXTURE.mkdir(mode=0o700, exist_ok=True) + RUN.private_directory(RUN.FIXTURE, reuse=True) fd = os.open(RUN.FIXTURE / 'lock', os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) output = self.directory / 'must-not-exist' with os.fdopen(fd, 'a') as lock: @@ -111,5 +240,21 @@ def test_clock_preload_is_fixed_before_imports(self): self.assertEqual(p.returncode, 0, p.stderr) self.assertEqual(json.loads(p.stdout), {'iso': '2026-10-21T12:00:00.000Z', 'date': 'Wed Oct 21 2026', 'now': 1792584000000}) +class ReleaseManifest(unittest.TestCase): + def test_missing_final_cleanup_is_explicit_incomplete_campaign(self): + counter = module('release_count', ROOT / 'release-count.py') + for checks in [None, [], [{'name': 'last-run', 'removed': {}, 'remaining': {}}], + [{'finalCampaignCheck': {}}], ['not-an-object']]: + with self.subTest(checks=checks), tempfile.TemporaryDirectory(prefix='release-incomplete-') as directory: + campaign = Path(directory) / 'campaign.json' + data = {'protocolVersion': 3, 'failures': [], 'REPS': 2} + if checks is not None: + data['cleanupChecks'] = checks + campaign.write_text(json.dumps(data)) + with self.assertRaisesRegex(ValueError, 'Incomplete release campaign: final cleanup evidence missing'): + counter.analyze(campaign) + self.assertFalse(campaign.with_name('release-comparison.json').exists()) + + if __name__ == '__main__': unittest.main() From e6762b3e55a81dbf4b808d9a96387e011fbecb1a Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 21 Oct 2026 14:15:07 +0100 Subject: [PATCH 4/7] fix: preserve release capture failure and cleanup evidence --- script/token-budget/release.ts | 136 ++++++++++++++++++++++++++------- 1 file changed, 109 insertions(+), 27 deletions(-) diff --git a/script/token-budget/release.ts b/script/token-budget/release.ts index d2a0606b..c0a0959f 100644 --- a/script/token-budget/release.ts +++ b/script/token-budget/release.ts @@ -75,6 +75,17 @@ let proc: ReturnType | undefined const campaignId = `release248-${process.pid}-${Date.now()}` const failures: unknown[] = [] const cleanupChecks: unknown[] = [] +const errors: unknown[] = [] +const cleanups: { name: typeof ownedContainer; child: typeof proc; promise: Promise }[] = [] +let interruption: { signal: "SIGINT" | "SIGTERM"; error: Error } | undefined +function recordFailure(error: unknown, field: "campaignError" | "cleanupError" = "campaignError") { + if (errors.includes(error)) return + errors.push(error) + failures.push({ [field]: error instanceof Error ? error.message : String(error) }) +} +function checkInterrupted() { + if (interruption) throw interruption.error +} const arms: string[] = ["current-default", "previous-default"] const support: Record = {} const raw: Record> = { short: Object.fromEntries(arms.map((x) => [x, []])) } @@ -128,7 +139,9 @@ function checkpoint() { ) } async function docker(args: string[]) { + checkInterrupted() const child = Bun.spawn(["docker", ...args], { env: hostEnv, stdin: "ignore", stdout: "pipe", stderr: "pipe" }) + proc = child let timeout = false const timer = setTimeout(() => { timeout = true @@ -141,12 +154,16 @@ async function docker(args: string[]) { child.exited, ]) if (code || timeout) { + const error = new Error(`Docker preflight failed (${code}, timeout=${timeout}): ${err}`) + recordFailure(error) await cleanup() - throw new Error(`Docker preflight failed (${code}, timeout=${timeout}): ${err}`) + throw error } + checkInterrupted() return out } finally { clearTimeout(timer) + if (proc === child) proc = undefined } } async function cleanupCommand(args: string[]) { @@ -167,30 +184,70 @@ async function cleanupCommand(args: string[]) { clearTimeout(timer) } } -async function cleanup() { - try { - if (ownedContainer) { - const name = ownedContainer - const removed = await cleanupCommand(["rm", "-f", name]) - const remaining = await cleanupCommand(["ps", "-aq", "--filter", `name=^/${name}$`]) - cleanupChecks.push({ name, removed, remaining }) +function cleanup() { + const name = ownedContainer + const child = proc + const running = cleanups.find((item) => item.name === name && item.child === child) + if (running) return running.promise + const promise = (async () => { + const failed: unknown[] = [] + try { + if (child && child.exitCode === null) child.kill("SIGKILL") + } catch (error) { + failed.push(error) + } + if (name) { + let removed: Awaited> | undefined + let remaining: Awaited> | undefined + try { + removed = await cleanupCommand(["rm", "-f", name]) + } catch (error) { + failed.push(error) + } + try { + remaining = await cleanupCommand(["ps", "-aq", "--filter", `name=^/${name}$`]) + } catch (error) { + failed.push(error) + } + cleanupChecks.push({ name, removed, remaining, errors: failed.map(String) }) if ( + !removed || + !remaining || removed.timedOut || remaining.timedOut || remaining.exit !== 0 || remaining.stdout.trim() || (removed.exit !== 0 && !removed.stderr.includes("No such container")) ) - throw new Error(`Unverified container cleanup: ${name}`) + failed.push(new Error(`Unverified container cleanup: ${name}`)) } - } finally { - if (proc && proc.exitCode === null) proc.kill("SIGKILL") - } -} -for (const signal of ["SIGINT", "SIGTERM"] as const) - process.on(signal, () => { - void cleanup().finally(() => process.exit(128 + (signal === "SIGINT" ? 2 : 15))) + try { + if (child) await child.exited + } catch (error) { + failed.push(error) + } + if (failed.length) { + const error = new AggregateError(failed, failed.map(String).join("; "), { cause: failed[0] }) + recordFailure(error, "cleanupError") + throw error + } + })().finally(() => { + const index = cleanups.findIndex((item) => item.promise === promise) + cleanups.splice(index, 1) }) + cleanups.push({ name, child, promise }) + return promise +} +const handlers = (["SIGINT", "SIGTERM"] as const).map((signal) => { + const handler = () => { + if (interruption) return + interruption = { signal, error: new Error(`Campaign interrupted by ${signal}`) } + recordFailure(interruption.error) + void cleanup().catch((error) => recordFailure(error, "cleanupError")) + } + process.on(signal, handler) + return { signal, handler } +}) try { const image = JSON.parse(await docker(["image", "inspect", input.fixtureImage]))[0] if ( @@ -201,6 +258,7 @@ try { ) throw new Error("Fixture image identity/platform mismatch") for (const name of ["current", "previous"] as const) { + checkInterrupted() const binary = input[name] const script = `const fs=require('fs'),cp=require('child_process'),crypto=require('crypto'); const binary='/release/aictrl'; console.log(JSON.stringify({uid:process.getuid(),home:fs.readdirSync(process.env.HOME),binarySha256:crypto.createHash('sha256').update(fs.readFileSync(binary)).digest('hex'),version:cp.execFileSync(binary,['--version'],{encoding:'utf8'}).trim(),help:cp.execFileSync(binary,['run','--help'],{encoding:'utf8'}),fixtures:Object.fromEntries(['notes.txt','big.txt',...Array.from({length:3},(_,i)=>'notes-'+i+'.txt'),...Array.from({length:50},(_,i)=>'big-'+i+'.txt')].map(name=>{const b=fs.readFileSync('/workspace/'+name);return[name,{bytes:b.length,sha256:crypto.createHash('sha256').update(b).digest('hex')}]}))}));` ownedContainer = `${campaignId}-preflight-${name}` @@ -227,6 +285,7 @@ try { ]), ) await cleanup() + checkInterrupted() ownedContainer = undefined fs.writeFileSync(path.join(output, `${name}-preflight.json`), JSON.stringify(preflight, null, 2) + "\n") if ( @@ -256,10 +315,12 @@ try { raw.short[`${name}-selected`] = [] } } + checkInterrupted() stub = new Stub(50) stub.keepBodies = true for (let rep = 0; rep < input.repeats; rep++) { for (const arm of rep % 2 ? [...arms].reverse() : arms) { + checkInterrupted() stub.reset("short") const runId = `${String(rep + 1).padStart(2, "0")}-${arm}-short` ownedContainer = `${campaignId}-${runId}` @@ -318,12 +379,14 @@ try { ) let timeout = false let cleanupFailure: string | undefined + checkInterrupted() const child = Bun.spawn(argv, { env: hostEnv, stdin: "ignore", stdout: "pipe", stderr: "pipe" }) proc = child const timer = setTimeout(() => { timeout = true + recordFailure(new Error(`Capture ${runId} timed out`)) void cleanup().catch((error) => { - cleanupFailure = String(error) + cleanupFailure ??= String(error) }) }, TIMEOUT) let stdout = "", @@ -340,7 +403,7 @@ try { try { await cleanup() } catch (error) { - cleanupFailure = String(error) + cleanupFailure ??= String(error) } if (!cleanupFailure) ownedContainer = undefined proc = undefined @@ -432,25 +495,44 @@ try { } } } catch (error) { - failures.push({ campaignError: error instanceof Error ? error.message : String(error) }) - checkpoint() - throw error + recordFailure(error) } finally { try { await cleanup() + } catch (error) { + recordFailure(error, "cleanupError") + } + try { const remaining = await cleanupCommand(["ps", "-aq", "--filter", `label=aictrl.harness-benchmark=${campaignId}`]) cleanupChecks.push({ finalCampaignCheck: remaining }) if (remaining.timedOut || remaining.exit !== 0 || remaining.stdout.trim()) throw new Error("Campaign-owned containers remain or cleanup state could not be inspected") - checkpoint() } catch (error) { - failures.push({ cleanupError: String(error) }) - checkpoint() - throw error - } finally { - stub?.server.stop(true) + recordFailure(error, "cleanupError") + } + try { + await stub?.server.stop(true) + } catch (error) { + recordFailure(error, "cleanupError") + } + await Promise.allSettled(cleanups.map((item) => item.promise)) + try { fs.rmSync(hostHome, { recursive: true, force: true }) + } catch (error) { + recordFailure(error, "cleanupError") + } + try { + checkpoint() + } catch (error) { + recordFailure(error) } + for (const { signal, handler } of handlers) process.off(signal, handler) +} +if (errors.length) { + const error = new AggregateError(errors, errors.map(String).join("; "), { cause: errors[0] }) + if (!interruption) throw error + console.error(error) + process.exit(128 + (interruption.signal === "SIGINT" ? 2 : 15)) } console.log( `Captured ${input.repeats} sessions per supported arm; containers verified absent in ${output}; run release-count.py next`, From 419c21bf74778a9cb1fa7b0612ae1f9e211ca2cf Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 21 Oct 2026 15:16:28 +0100 Subject: [PATCH 5/7] fix: integrate reviewed headless CLI corrections --- .github/workflows/ci.yml | 7 + README.md | 2 +- RELEASING.md | 2 +- SECURITY.md | 16 + examples/released-smoke/README.md | 2 +- examples/released-smoke/smoke.mjs | 77 ++- packages/cli/README.md | 6 +- packages/cli/src/auth/wellknown.ts | 76 +++ packages/cli/src/cli/cmd/auth.ts | 19 +- packages/cli/src/cli/cmd/run.ts | 2 +- packages/cli/src/plugin/codex.ts | 11 +- packages/cli/test/auth/wellknown.test.ts | 131 ++++ packages/cli/test/cli/fixture/README.md | 27 + .../cli/fixture/command-template-capture.ts | 492 ++++++++------ .../test/cli/fixture/integrated-capture.ts | 616 ++++++++++-------- packages/cli/test/cli/request-arrival.test.ts | 112 ++++ .../test/cli/run-command-arguments.test.ts | 520 ++++++++++----- packages/cli/test/cli/run-stdin.test.ts | 437 ++++++++----- .../cli/test/cli/run-tool-selection.test.ts | 388 +++++++---- .../cli/test/session/tool-guidance.test.ts | 261 +++++--- packages/plugin/src/tool.ts | 11 +- packages/sdk/script/build.ts | 2 +- packages/sdk/src/v2/index.ts | 2 +- script/token-budget/README.md | 8 +- script/token-budget/release-count.py | 7 +- .../release-fixtures/token-stub.test.ts | 89 +++ .../release-fixtures/token-stub.ts | 2 +- script/token-budget/release.ts | 136 +++- script/token-budget/run.py | 243 +++++-- script/token-budget/test_protocol.py | 147 ++++- 30 files changed, 2690 insertions(+), 1161 deletions(-) create mode 100644 SECURITY.md create mode 100644 packages/cli/src/auth/wellknown.ts create mode 100644 packages/cli/test/auth/wellknown.test.ts create mode 100644 packages/cli/test/cli/fixture/README.md create mode 100644 packages/cli/test/cli/request-arrival.test.ts create mode 100644 script/token-budget/release-fixtures/token-stub.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86a2e6dd..2f1b08c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,9 @@ on: pull_request: branches: [main] +permissions: + contents: read + jobs: verify: runs-on: ubuntu-latest @@ -48,6 +51,10 @@ jobs: - name: Enforce actual SDK prefix token budgets run: '"$RUNNER_TEMP/token-budget-environment/venv/bin/python3" -I -B script/token-budget/run.py --output "$RUNNER_TEMP/token-budget-evidence"' + - name: Verify token fixture loopback binding + working-directory: script/token-budget/release-fixtures + run: bun test token-stub.test.ts + - name: Verify token-budget fault controls run: '"$RUNNER_TEMP/token-budget-environment/venv/bin/python3" -I -B -m unittest discover -s script/token-budget -p "test_*.py" -v' env: diff --git a/README.md b/README.md index adccc41b..5edb862e 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Stdin is read only when a standalone `-` is a message operand, or when no messag The marker can appear before or after `--`; repeated markers still read stdin once. Markers are removed from the model message, command arguments and derived session title. `--` ends option parsing, so `run -- -` also selects stdin. An operand such as `"literal -"` remains ordinary text. An explicit empty operand (`run ""`) does not select implicit stdin. Selected stdin is read to EOF and appended after one actual newline. Ordinary prompt operands are joined with spaces without adding artificial quotes or backslash escapes; quotes, backslashes and whitespace inside an operand remain its text. -Option values belong to their options: `--model=-` or `--command=-` does not select stdin, and scalar options also accept a separated hyphen value. Use `--file=-` to attach a file literally named `-`; `--file -` is missing its required file value. Multiple file values and repeated `--skill` flags retain their existing behavior. Required option values are checked by the argument parser; explicit `--title ""` remains supported. +Required-value options own a separated hyphen: `--model -` and `--command -` do not select stdin, nor do their `=` forms. Optional `--port` leaves a separated `-` as the stdin operand (`--port -`). Use `--file=-` to attach a file literally named `-`; `--file -` is missing its required file value. Multiple file values and repeated `--skill` flags retain their existing behavior. Required option values are checked by the argument parser; explicit `--title ""` remains supported. An explicit marker reads stdin even from a terminal. Any selected stream can wait for EOF indefinitely; this change adds no stdin timeout. Callers must close selected input or apply their own supervisor deadline. diff --git a/RELEASING.md b/RELEASING.md index 218fe148..09a0d6b2 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -18,7 +18,7 @@ and network access during setup only), then capture and count offline: ```bash - python3.12 script/token-budget/setup.py --directory /absolute/new-counter-env + python3.12 -I -B script/token-budget/setup.py --directory /absolute/new-counter-env bun run --no-install --no-env-file script/token-budget/release.ts /absolute/release-inputs.json /absolute/new-release-evidence /absolute/new-counter-env/venv/bin/python3 -I -B script/token-budget/release-count.py /absolute/new-release-evidence/campaign.json ``` diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..d4e6a35f --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,16 @@ +# Security policy + +## Report a vulnerability + +Report suspected vulnerabilities in aictrl CLI through [GitHub private vulnerability reporting](https://github.com/aictrl-dev/cli/security/advisories/new). You can also open this repository's Security and quality tab and choose **Report a vulnerability**. + +Please include: + +- The affected CLI version and relevant operating system or runtime details. +- A minimal reproduction and the observed behavior. +- The potential security impact. +- Sanitized logs or examples that help reproduce the problem. + +Do not post exploit details, credentials, tokens, or private data in public issues. Remove secrets and personal data from any logs or examples you share. + +For ordinary bugs without a suspected security impact, use the [public issue tracker](https://github.com/aictrl-dev/cli/issues). diff --git a/examples/released-smoke/README.md b/examples/released-smoke/README.md index 8a2e2af7..0d2066d6 100644 --- a/examples/released-smoke/README.md +++ b/examples/released-smoke/README.md @@ -18,7 +18,7 @@ node examples/released-smoke/smoke.mjs \ The runner verifies the native hash/version, two provider requests, an effective wire catalog containing only `read`, the completed read result, schema-version-1 NDJSON, successful terminal events and exit 0. A second invocation supplies invalid root schema `[]`: it must exit 2 with `OUTPUT_SCHEMA_CONFIG` before any additional model request. This is schema rejection, not successful structured output. -The CLI gets closed stdin, fresh HOME/XDG/workspace directories and an explicit environment allowlist without host credentials. Each subprocess has a 30-second deadline; the whole run has 60 seconds. Cleanup terminates POSIX process groups, escalates to SIGKILL after five seconds, and removes temporary state. A successful run prints `"status": "pass"`; a nonzero exit is failure. Evidence contains only the fixed inert fixture, public prompts and a placeholder key. Remove `CLI_DIR` after inspecting its evidence. +The CLI gets closed stdin, fresh HOME/XDG/workspace directories and an explicit environment allowlist without host credentials. Each CLI subprocess has a 30-second deadline. After setup and before cleanup/evidence writing, the execution phase has a shared 60-second abort timer. Setup, cleanup and evidence I/O are outside that timer; CI imposes an enclosing five-minute job timeout. Local callers that need a whole-process bound must provide their own external timeout. Cleanup terminates POSIX process groups, escalates to SIGKILL after five seconds, and removes temporary state. A successful run prints `"status": "pass"`; a nonzero exit is failure. Evidence contains only the fixed inert fixture, public prompts and a placeholder key. Remove `CLI_DIR` after inspecting its evidence. ## GitHub Actions diff --git a/examples/released-smoke/smoke.mjs b/examples/released-smoke/smoke.mjs index a546e11d..8272f0db 100644 --- a/examples/released-smoke/smoke.mjs +++ b/examples/released-smoke/smoke.mjs @@ -34,10 +34,11 @@ let root let server let overallTimer let pendingFailure +let signalVersion = 0 const report = { recipe: marker, expectedVersion: "0.5.4", - sourceCommit: "6c661f4cf3e3d555a0f290bc22e071c6de4bd19b", + releaseSourceCommit: "6c661f4cf3e3d555a0f290bc22e071c6de4bd19b", startedUTC: new Date().toISOString(), nodeVersion: process.version, binary, @@ -45,6 +46,7 @@ const report = { runs, requests, protocolErrors, + cleanupErrors: [], } const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) function signalGroup(pid, signal) { @@ -61,7 +63,14 @@ function abort(reason) { controller.abort(new Error(reason)) for (const pid of groups) signalGroup(pid, "SIGTERM") } -const interrupted = (signal) => abort(`Runner received ${signal}`) +const interrupted = (signal) => { + if (signalVersion) return + signalVersion = 1 + pendingFailure ??= controller.signal.aborted ? controller.signal.reason : new Error(`Runner received ${signal}`) + report.status = "fail" + report.error ??= { message: pendingFailure.message, stack: pendingFailure.stack } + abort(`Runner received ${signal}`) +} const onINT = () => interrupted("SIGINT") const onTERM = () => interrupted("SIGTERM") process.on("SIGINT", onINT) @@ -102,16 +111,31 @@ async function run(label, argv, env, cwd) { term() }, 30_000) controller.signal.addEventListener("abort", term, { once: true }) + const captureLimit = 2 * 1024 * 1024 + const capturedBytes = { stdout: 0, stderr: 0 } + const overflowed = new Set() const collect = (name, data) => { - record[name] += data.toString("utf8") - if (Buffer.byteLength(record[name]) > 2 * 1024 * 1024) { - abort(`${label} ${name} exceeds the fixture's 2 MiB capture bound`) + if (overflowed.has(name)) return + const bytes = Buffer.byteLength(data) + const remaining = captureLimit - capturedBytes[name] + if (bytes <= remaining) { + record[name] += data + capturedBytes[name] += bytes + return } + const encoded = Buffer.from(data, "utf8") + let end = remaining + while (end > 0 && (encoded[end] & 0xc0) === 0x80) end-- + record[name] += encoded.subarray(0, end).toString("utf8") + capturedBytes[name] += end + overflowed.add(name) + abort(`${label} ${name} exceeds the fixture's 2 MiB capture bound`) } child.stdout.setEncoding("utf8") child.stderr.setEncoding("utf8") child.stdout.on("data", (data) => collect("stdout", data)) child.stderr.on("data", (data) => collect("stderr", data)) + let failure try { const result = await new Promise((resolve, reject) => { child.once("error", reject) @@ -121,13 +145,21 @@ async function run(label, argv, env, cwd) { if (deadline) throw new Error(`${label} exceeded its external 30-second deadline`) controller.signal.throwIfAborted() return record + } catch (error) { + failure = error + throw error } finally { clearTimeout(timer) clearTimeout(killTimer) controller.signal.removeEventListener("abort", term) if (pid) { - await cleanupGroup(pid) - groups.delete(pid) + try { + await cleanupGroup(pid) + groups.delete(pid) + } catch (error) { + report.cleanupErrors.push(error.message) + if (!failure) throw error + } } } } @@ -346,16 +378,15 @@ try { } report.status = "pass" } catch (error) { - pendingFailure = error + pendingFailure ??= error report.status = "fail" - report.error = { message: error.message, stack: error.stack } + report.error ??= { message: pendingFailure.message, stack: pendingFailure.stack } } finally { clearTimeout(overallTimer) if (server?.listening) { server.closeAllConnections() await new Promise((resolve) => server.close(resolve)) } - report.cleanupErrors = [] for (const pid of groups) { try { await cleanupGroup(pid) @@ -376,18 +407,28 @@ try { report.cleanupErrors.push(error.message) } report.cleanup = { processGroupsRemaining: groups.size, temporaryRootRemoved: removed } - process.removeListener("SIGINT", onINT) - process.removeListener("SIGTERM", onTERM) report.finishedUTC = new Date().toISOString() - if (evidence) { - await mkdir(evidence, { recursive: true }) - await writeFile(path.join(evidence, "report.json"), JSON.stringify(report, null, 2) + "\n") - for (const record of runs) { - await writeFile(path.join(evidence, `${record.label}.stdout`), record.stdout) - await writeFile(path.join(evidence, `${record.label}.stderr`), record.stderr) + try { + if (evidence) { + await mkdir(evidence, { recursive: true }) + for (const record of runs) { + await writeFile(path.join(evidence, `${record.label}.stdout`), record.stdout) + await writeFile(path.join(evidence, `${record.label}.stderr`), record.stderr) + } + const version = signalVersion + await writeFile(path.join(evidence, "report.json"), JSON.stringify(report, null, 2) + "\n") + if (version !== signalVersion) + await writeFile(path.join(evidence, "report.json"), JSON.stringify(report, null, 2) + "\n") } + } catch (error) { + pendingFailure ??= error + report.status = "fail" + report.error ??= { message: pendingFailure.message, stack: pendingFailure.stack } + report.evidenceError = { message: error.message, stack: error.stack } } console.log(JSON.stringify(report, null, 2)) + process.removeListener("SIGINT", onINT) + process.removeListener("SIGTERM", onTERM) } if (pendingFailure) { console.error(`[owned CLI smoke] ${pendingFailure.message}`) diff --git a/packages/cli/README.md b/packages/cli/README.md index ea88b557..7cd9bf95 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -65,8 +65,10 @@ eligible native Task tool to be selected, without adding permission allow rules. are protocol/control tools, outside the work allowlist. Schema mode still needs permission to call `StructuredOutput`; selecting work tools never grants that permission. An empty work allowlist can therefore coexist with an explicitly -permitted StructuredOutput protocol tool. Custom tools with those names are still -work tools; protocol identity belongs to the harness-created definition. +permitted StructuredOutput protocol tool. Custom tools with protocol-like names +remain work tools, except that an active output schema reserves the +`StructuredOutput` wire name for the harness-created protocol tool. A custom +`StructuredOutput` is selectable as work only outside schema mode. Custom tools can use the optional typed `context.selection` facade to inspect `names`/`restricted`, assert availability, or call diff --git a/packages/cli/src/auth/wellknown.ts b/packages/cli/src/auth/wellknown.ts new file mode 100644 index 00000000..ad1bd852 --- /dev/null +++ b/packages/cli/src/auth/wellknown.ts @@ -0,0 +1,76 @@ +import z from "zod" + +export namespace WellKnownAuth { + const LOOPBACK_HOSTS = new Set(["localhost", "127.0.0.1", "[::1]"]) + + export const Document = z.object({ + auth: z.object({ + command: z.array(z.string().min(1)).min(1), + env: z.string().min(1), + }), + }) + export type Command = z.infer["auth"] + + // The provider dictates a local command, so plain http is only allowed for local development. + export function endpoint(providerURL: string): string { + let url: URL + try { + url = new URL(providerURL) + } catch (cause) { + throw new Error("Invalid auth provider URL. Supply an absolute https URL.", { cause }) + } + const loopback = url.protocol === "http:" && LOOPBACK_HOSTS.has(url.hostname) + if (url.protocol !== "https:" && !loopback) { + throw new Error("Auth provider URL must use https (except http on loopback hosts).") + } + url.search = "" + url.hash = "" + url.pathname = `${url.pathname.replace(/\/+$/, "")}/.well-known/aictrl` + return url.toString() + } + + export function display(value: string | readonly string[]): string { + return JSON.stringify(value).replace( + /[\u007f-\u009f\u061c\u200e\u200f\u2028-\u202e\u2066-\u2069]/g, + (character) => `\\u${character.charCodeAt(0).toString(16).padStart(4, "0")}`, + ) + } + + export async function fetchCommand(providerURL: string): Promise { + const target = endpoint(providerURL) + let response: Response + try { + response = await fetch(target, { redirect: "error" }) + } catch (cause) { + throw new Error( + "Failed to fetch auth provider config. Check connectivity and use a direct URL without redirects.", + { + cause, + }, + ) + } + if (!response.ok) { + throw new Error(`Failed to fetch auth provider config (HTTP ${response.status}). Check the provider URL.`, { + cause: response, + }) + } + let document: unknown + try { + document = await response.json() + } catch (cause) { + throw new Error("Auth provider config is not valid JSON. Contact the provider to fix its well-known document.", { + cause, + }) + } + try { + return Document.parse(document).auth + } catch (cause) { + throw new Error( + "Auth provider config must supply auth.command as a nonempty string array and auth.env as a string.", + { + cause, + }, + ) + } + } +} diff --git a/packages/cli/src/cli/cmd/auth.ts b/packages/cli/src/cli/cmd/auth.ts index 63da5fa0..cf0a5492 100644 --- a/packages/cli/src/cli/cmd/auth.ts +++ b/packages/cli/src/cli/cmd/auth.ts @@ -1,4 +1,5 @@ import { Auth } from "../../auth" +import { WellKnownAuth } from "../../auth/wellknown" import { cmd } from "./cmd" import * as prompts from "@clack/prompts" import { UI } from "../ui" @@ -263,9 +264,17 @@ export const AuthLoginCommand = cmd({ UI.empty() prompts.intro("Add credential") if (args.url) { - const wellknown = await fetch(`${args.url}/.well-known/aictrl`).then((x) => x.json() as any) - prompts.log.info(`Running \`${wellknown.auth.command.join(" ")}\``) - const proc = Process.spawn(wellknown.auth.command, { + const wellknown = await WellKnownAuth.fetchCommand(args.url) + const approved = await prompts.confirm({ + message: `${WellKnownAuth.display(args.url)} wants to run ${WellKnownAuth.display(wellknown.command)} to get a token. Run it?`, + initialValue: false, + }) + if (prompts.isCancel(approved)) throw new UI.CancelledError() + if (!approved) { + prompts.outro("Cancelled") + return + } + const proc = Process.spawn(wellknown.command, { stdout: "pipe", }) if (!proc.stdout) { @@ -281,10 +290,10 @@ export const AuthLoginCommand = cmd({ } await Auth.set(args.url, { type: "wellknown", - key: wellknown.auth.env, + key: wellknown.env, token: token.trim(), }) - prompts.log.success("Logged into " + args.url) + prompts.log.success("Logged into " + WellKnownAuth.display(args.url)) prompts.outro("Done") return } diff --git a/packages/cli/src/cli/cmd/run.ts b/packages/cli/src/cli/cmd/run.ts index cb28761b..31f5a16f 100644 --- a/packages/cli/src/cli/cmd/run.ts +++ b/packages/cli/src/cli/cmd/run.ts @@ -1231,7 +1231,7 @@ export const RunCommand = cmd({ invocation.phase("bootstrap") if (args.attach) { // This checkout has no receiver/capability contract that can enforce selection. - const configured = await Config.readToolSelection(process.cwd()) + const configured = await invocation.guard(() => Config.readToolSelection(process.cwd())) if (configured !== undefined) { await fail( "Work-tool selection is not supported with --attach: this client cannot verify server-side enforcement. Remove --tools and local tool_allowlist or run locally.", diff --git a/packages/cli/src/plugin/codex.ts b/packages/cli/src/plugin/codex.ts index 91ac7083..8d421077 100644 --- a/packages/cli/src/plugin/codex.ts +++ b/packages/cli/src/plugin/codex.ts @@ -20,7 +20,8 @@ interface PkceCodes { } async function generatePKCE(): Promise { - const verifier = generateRandomString(43) + // RFC 7636: base64url-encode 32 random bytes to get an unbiased 43-character verifier. + const verifier = base64UrlEncode(crypto.getRandomValues(new Uint8Array(32)).buffer) const encoder = new TextEncoder() const data = encoder.encode(verifier) const hash = await crypto.subtle.digest("SHA-256", data) @@ -28,14 +29,6 @@ async function generatePKCE(): Promise { return { verifier, challenge } } -function generateRandomString(length: number): string { - const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" - const bytes = crypto.getRandomValues(new Uint8Array(length)) - return Array.from(bytes) - .map((b) => chars[b % chars.length]) - .join("") -} - function base64UrlEncode(buffer: ArrayBuffer): string { const bytes = new Uint8Array(buffer) const binary = String.fromCharCode(...bytes) diff --git a/packages/cli/test/auth/wellknown.test.ts b/packages/cli/test/auth/wellknown.test.ts new file mode 100644 index 00000000..ffa2529f --- /dev/null +++ b/packages/cli/test/auth/wellknown.test.ts @@ -0,0 +1,131 @@ +import { test, expect } from "bun:test" +import { WellKnownAuth } from "../../src/auth/wellknown" + +test("builds the well-known endpoint for https providers", () => { + expect(WellKnownAuth.endpoint("https://auth.example.com")).toBe("https://auth.example.com/.well-known/aictrl") +}) + +test("allows plain http only for loopback hosts", () => { + expect(WellKnownAuth.endpoint("http://localhost:4096")).toBe("http://localhost:4096/.well-known/aictrl") + expect(WellKnownAuth.endpoint("http://127.0.0.1:4096")).toBe("http://127.0.0.1:4096/.well-known/aictrl") + expect(WellKnownAuth.endpoint("http://[::1]:4096")).toBe("http://[::1]:4096/.well-known/aictrl") +}) + +test("rejects insecure or non-http provider URLs", () => { + expect(() => WellKnownAuth.endpoint("http://auth.example.com")).toThrow("must use https") + expect(() => WellKnownAuth.endpoint("file:///etc/passwd")).toThrow("must use https") + expect(() => WellKnownAuth.endpoint("auth.example.com")).toThrow() +}) + +test("accepts a well-formed auth command", () => { + const parsed = WellKnownAuth.Document.parse({ auth: { command: ["cli", "token"], env: "TOKEN" }, config: {} }) + expect(parsed.auth).toEqual({ command: ["cli", "token"], env: "TOKEN" }) +}) + +test("rejects malformed auth commands", () => { + expect(() => WellKnownAuth.Document.parse({ auth: { command: "cli token", env: "TOKEN" } })).toThrow() + expect(() => WellKnownAuth.Document.parse({ auth: { command: [], env: "TOKEN" } })).toThrow() + expect(() => WellKnownAuth.Document.parse({ auth: { command: ["cli"] } })).toThrow() + expect(() => WellKnownAuth.Document.parse({})).toThrow() +}) + +test("appends the endpoint to the base path without query, fragment, or trailing slashes", () => { + for (const suffix of ["", "/", "///", "?tenant=example", "/?tenant=example#login"]) { + expect(WellKnownAuth.endpoint(`https://auth.example.com/base${suffix}`)).toBe( + "https://auth.example.com/base/.well-known/aictrl", + ) + } +}) + +test("malformed provider URLs have a minimal actionable message and retain the cause", () => { + try { + WellKnownAuth.endpoint("invalid-owned-url\nPRIVATE_DOCUMENT_MARKER") + throw new Error("Expected an invalid URL error") + } catch (error) { + expect(error).toBeInstanceOf(Error) + expect((error as Error).message).toContain("Supply an absolute https URL") + expect((error as Error).message).not.toContain("PRIVATE_DOCUMENT_MARKER") + expect((error as Error).cause).toBeInstanceOf(Error) + } +}) + +test("terminal display escapes controls while preserving exact values and argument boundaries", () => { + const command = ["owned-cli", "two words", 'quote" and `literal`', "\x1b[2J\x07\r\n\t\0\x7f\x85\u2028\u2029\u202e"] + const rendered = WellKnownAuth.display(command) + expect(JSON.parse(rendered)).toEqual(command) + expect(rendered).not.toMatch(/[\x00-\x1f\x7f-\x9f\u2028\u2029\u202e]/) + expect(rendered).toContain('"two words"') + expect(rendered).toContain("\\u001b") + expect(rendered).toContain("\\u202e") + const provider = "https://auth.example.com/?hint=\x1b[2J\n\u2066" + expect(JSON.parse(WellKnownAuth.display(provider))).toBe(provider) + expect(WellKnownAuth.display(provider)).not.toMatch(/[\x00-\x1f\u2066]/) +}) + +test("fetches the canonical endpoint and returns only the validated auth command", async () => { + const paths: string[] = [] + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + paths.push(new URL(request.url).pathname) + return Response.json({ auth: { command: ["owned-cli", "two words"], env: "OWNED_TOKEN" } }) + }, + }) + try { + expect(await WellKnownAuth.fetchCommand(`${server.url}base///?tenant=example#login`)).toEqual({ + command: ["owned-cli", "two words"], + env: "OWNED_TOKEN", + }) + expect(paths).toEqual(["/base/.well-known/aictrl"]) + } finally { + await server.stop(true) + } +}) + +test("never follows an auth document redirect to another owned loopback server", async () => { + let followed = 0 + const target = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch() { + followed++ + return Response.json({ auth: { command: ["owned-cli"], env: "OWNED_TOKEN" } }) + }, + }) + const redirect = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch: () => Response.redirect(target.url), + }) + try { + await expect(WellKnownAuth.fetchCommand(redirect.url.toString())).rejects.toThrow("without redirects") + expect(followed).toBe(0) + } finally { + await Promise.all([redirect.stop(true), target.stop(true)]) + } +}) + +for (const [name, response, message] of [ + ["HTTP error", () => new Response("PRIVATE_DOCUMENT_MARKER", { status: 503 }), "HTTP 503"], + ["invalid JSON", () => new Response("PRIVATE_DOCUMENT_MARKER"), "not valid JSON"], + ["invalid schema", () => Response.json({ auth: { command: "PRIVATE_DOCUMENT_MARKER" } }), "auth.command"], +] as const) { + test(`${name} produces an actionable error with cause without echoing the document`, async () => { + const server = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: response }) + try { + let failure: Error | undefined + try { + await WellKnownAuth.fetchCommand(server.url.toString()) + } catch (error) { + failure = error as Error + } + expect(failure).toBeInstanceOf(Error) + expect(failure!.message).toContain(message) + expect(failure!.message).not.toContain("PRIVATE_DOCUMENT_MARKER") + expect(failure!.cause).toBeDefined() + } finally { + await server.stop(true) + } + }) +} diff --git a/packages/cli/test/cli/fixture/README.md b/packages/cli/test/cli/fixture/README.md new file mode 100644 index 00000000..b5a40d34 --- /dev/null +++ b/packages/cli/test/cli/fixture/README.md @@ -0,0 +1,27 @@ +# Command and stdin fixture controls + +`stdin-guard.ts` observes `globalThis.fetch` in the preloaded CLI process. It logs +those calls and rejects origins other than the owned loopback fixture. It also +rejects `fetch.preconnect` and direct `Bun.spawn` calls whose first executable is +`bun` and whose next argument is exactly `install` or `add`. + +These are partial observations, not complete network or installation accounting. +The ledger does not cover other network APIs, `Bun.spawnSync`, Node child-process +APIs, package-manager aliases, or programs launched through a shell. A clean +ledger proves only that the instrumented paths reported no rejected operation. +The fixtures use inert trusted commands and explicit child environments without +provider credentials. They are not an OS network sandbox and do not establish +safe execution of arbitrary code. + +The command and stdin harnesses stop their owned listeners and child process +groups on success and failure, including failures during fixture-file setup. +They remove only their own `mkdtemp` directory after reading final observations +and checking results. Set `COMMAND_ARGUMENTS_EVIDENCE` or `STDIN_TEST_EVIDENCE` +for the respective harness to retain its raw directory as well as the requested +JSON evidence. Cleanup failures are reported separately; an existing setup or +capture error remains the primary error. A cleanup failure after an otherwise +successful run rejects the fixture. + +Attached command tests demonstrate the client's wire shape against an owned +stub, not compatibility with an HTTP receiver. The structured-argument receiver +and release-coordination gate remain open as described in the root README. diff --git a/packages/cli/test/cli/fixture/command-template-capture.ts b/packages/cli/test/cli/fixture/command-template-capture.ts index f60801b9..a6fc9dea 100644 --- a/packages/cli/test/cli/fixture/command-template-capture.ts +++ b/packages/cli/test/cli/fixture/command-template-capture.ts @@ -14,230 +14,298 @@ export type Input = ( ) & { template?: string; deadlineMs?: number; ownedRedirect?: boolean } export async function capture(input: Input) { const cwd = await fs.mkdtemp(path.join(os.tmpdir(), "aictrl-command-arguments-")) - const ownedFile = input.ownedRedirect ? path.join(cwd, "target; /usr/bin/printf INERT_REDIRECT") : undefined - if (ownedFile) await fs.mkdir(path.dirname(ownedFile), { recursive: true }) - const requests: Record[] = [] - const server = Bun.serve({ - hostname: "127.0.0.1", - port: 0, - async fetch(request) { - if (new URL(request.url).pathname !== "/v1/chat/completions") throw new Error("Unexpected provider path") - requests.push((await request.json()) as Record) - const chunks = [ - { choices: [{ index: 0, delta: { role: "assistant", content: "Done" }, finish_reason: null }] }, - { - choices: [{ index: 0, delta: {}, finish_reason: "stop" }], - usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, - }, - ] - return new Response(chunks.map((value) => `data: ${JSON.stringify(value)}\n\n`).join("") + "data: [DONE]\n\n", { - headers: { "content-type": "text/event-stream" }, - }) + const evidence = process.env.COMMAND_ARGUMENTS_EVIDENCE + const cleanup: (() => void | Promise)[] = [ + async () => { + if (!evidence) await fs.rm(cwd, { recursive: true, force: true }) }, - }) - await fs.writeFile( - path.join(cwd, "aictrl.json"), - JSON.stringify({ - plugin: [new URL("./command-template-plugin.ts", import.meta.url).href], - enabled_providers: ["fixture"], - provider: { - fixture: { - npm: "@ai-sdk/openai-compatible", - options: { apiKey: "fixture", baseURL: server.url.origin + "/v1" }, - models: { fixture: { name: "fixture", tool_call: true, limit: { context: 100000, output: 1000 } } }, - }, - }, - agent: { title: { disable: true } }, - command: { argumentprobe: { template: input.template ?? template } }, - }), - ) - const data = path.join(cwd, "arguments.json") - await fs.writeFile(data, JSON.stringify("arguments" in input ? { arguments: input.arguments } : {})) - await fs.writeFile(path.join(cwd, "observations.jsonl"), "") - const ledger = path.join(cwd, "network.jsonl") - await fs.writeFile(ledger, "") - const env = { - PATH: [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter), - HOME: path.join(cwd, "home"), - AICTRL_TEST_HOME: path.join(cwd, "home"), - AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(cwd, "managed"), - XDG_DATA_HOME: path.join(cwd, "data"), - XDG_CONFIG_HOME: path.join(cwd, "config"), - XDG_CACHE_HOME: path.join(cwd, "cache"), - XDG_STATE_HOME: path.join(cwd, "state"), - BUN_RUNTIME_TRANSPILER_CACHE_PATH: path.join(cwd, "bun-cache"), - GIT_CONFIG_NOSYSTEM: "1", - GIT_CONFIG_GLOBAL: "/dev/null", - TZ: "UTC", - AICTRL_DISABLE_DEFAULT_PLUGINS: "true", - AICTRL_DISABLE_AUTOCOMPACT: "true", - STDIN_FIXTURE_ORIGIN: server.url.origin, - STDIN_FIXTURE_LEDGER: ledger, - COMMAND_ARGUMENTS_DATA: data, - COMMAND_TEMPLATE_OBSERVATIONS: path.join(cwd, "observations.jsonl"), - } - const source = - "arguments" in input - ? path.join(import.meta.dir, "command-template-handler.ts") - : path.resolve(import.meta.dir, `../../../src/${input.entry}.ts`) - const command = [ - "run", - "--no-install", - "--no-env-file", - "--preload", - guard, - "--conditions=browser", - source, - ...("arguments" in input - ? [] - : [ - "run", - "--format", - "json", - "--model", - "fixture/fixture", - "--command", - "argumentprobe", - ...input.operands.map((value) => (ownedFile && value === "OWNED_REDIRECT_TARGET" ? ownedFile : value)), - ]), ] - const child = spawn(process.execPath, command, { - cwd, - env, - detached: true, - stdio: ["stdin" in input ? "pipe" : "ignore", "pipe", "pipe"], - }) - let stdout = "", - stderr = "", - deadline = false - const exists = () => { - try { - process.kill(-child.pid!, 0) - return true - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ESRCH") return false - throw error - } - } - let staged = false - let firstBytesAt: number | undefined - let eofAt: number | undefined - let requestsBeforeEOF: number | undefined - const started = performance.now() - const stages: ReturnType[] = [] - child.stdin?.on("error", (error: NodeJS.ErrnoException) => { - if (error.code !== "EPIPE") stderr += String(error) - }) - child.stdout!.on("data", (value) => { - stdout += value.toString() - if ("stdin" in input && !staged && stdout.includes('"type":"invocation_start"')) { - staged = true - stages.push( - setTimeout(() => { - firstBytesAt = performance.now() - started - child.stdin!.write(input.stdin ?? "") - stages.push( - setTimeout(() => { - eofAt = performance.now() - started - requestsBeforeEOF = requests.length - child.stdin!.end() - }, 100), - ) - }, 100), - ) - } - }) - child.stderr!.on("data", (value) => { - stderr += value.toString() - }) - const timer = setTimeout(() => { - deadline = true - if (exists()) process.kill(-child.pid!, "SIGTERM") - }, input.deadlineMs ?? 12000) - const escalation = setTimeout( - () => { - if (deadline && exists()) process.kill(-child.pid!, "SIGKILL") - }, - (input.deadlineMs ?? 12000) + 200, - ) - let code: number | null = null + let failed = false try { - code = await new Promise((resolve, reject) => { - child.once("error", reject) - child.once("close", (value) => resolve(value)) + const ownedFile = input.ownedRedirect ? path.join(cwd, "target; /usr/bin/printf INERT_REDIRECT") : undefined + if (ownedFile) await fs.mkdir(path.dirname(ownedFile), { recursive: true }) + const requests: Record[] = [] + let requestArrivals = 0 + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + requestArrivals++ + if (new URL(request.url).pathname !== "/v1/chat/completions") throw new Error("Unexpected provider path") + requests.push((await request.json()) as Record) + const chunks = [ + { choices: [{ index: 0, delta: { role: "assistant", content: "Done" }, finish_reason: null }] }, + { + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }, + ] + return new Response(chunks.map((value) => `data: ${JSON.stringify(value)}\n\n`).join("") + "data: [DONE]\n\n", { + headers: { "content-type": "text/event-stream" }, + }) + }, }) - } finally { - for (const stage of stages) clearTimeout(stage) - child.stdin?.destroy() - clearTimeout(timer) - clearTimeout(escalation) - if (exists()) { - process.kill(-child.pid!, "SIGTERM") - await pause(150) - if (exists()) process.kill(-child.pid!, "SIGKILL") + cleanup.push(() => server.stop(true)) + await fs.writeFile( + path.join(cwd, "aictrl.json"), + JSON.stringify({ + plugin: [new URL("./command-template-plugin.ts", import.meta.url).href], + enabled_providers: ["fixture"], + provider: { + fixture: { + npm: "@ai-sdk/openai-compatible", + options: { apiKey: "fixture", baseURL: server.url.origin + "/v1" }, + models: { fixture: { name: "fixture", tool_call: true, limit: { context: 100000, output: 1000 } } }, + }, + }, + agent: { title: { disable: true } }, + command: { argumentprobe: { template: input.template ?? template } }, + }), + ) + const data = path.join(cwd, "arguments.json") + await fs.writeFile(data, JSON.stringify("arguments" in input ? { arguments: input.arguments } : {})) + await fs.writeFile(path.join(cwd, "observations.jsonl"), "") + const ledger = path.join(cwd, "network.jsonl") + await fs.writeFile(ledger, "") + const env = { + PATH: [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter), + HOME: path.join(cwd, "home"), + AICTRL_TEST_HOME: path.join(cwd, "home"), + AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(cwd, "managed"), + XDG_DATA_HOME: path.join(cwd, "data"), + XDG_CONFIG_HOME: path.join(cwd, "config"), + XDG_CACHE_HOME: path.join(cwd, "cache"), + XDG_STATE_HOME: path.join(cwd, "state"), + BUN_RUNTIME_TRANSPILER_CACHE_PATH: path.join(cwd, "bun-cache"), + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + TZ: "UTC", + AICTRL_DISABLE_DEFAULT_PLUGINS: "true", + AICTRL_DISABLE_AUTOCOMPACT: "true", + STDIN_FIXTURE_ORIGIN: server.url.origin, + STDIN_FIXTURE_LEDGER: ledger, + COMMAND_ARGUMENTS_DATA: data, + COMMAND_TEMPLATE_OBSERVATIONS: path.join(cwd, "observations.jsonl"), } - server.stop(true) - } - const network = (await fs.readFile(ledger, "utf8")) - .split("\n") - .filter(Boolean) - .map((value) => JSON.parse(value)) - const events = stdout - .split("\n") - .filter((value) => value.startsWith("{")) - .map((value) => JSON.parse(value)) - let redirected: { path: string; text?: string; removed: boolean } | undefined - if (ownedFile) { - const text = await fs.readFile(ownedFile, "utf8").catch((error: NodeJS.ErrnoException) => { - if (error.code === "ENOENT") return undefined - throw error + const source = + "arguments" in input + ? path.join(import.meta.dir, "command-template-handler.ts") + : path.resolve(import.meta.dir, `../../../src/${input.entry}.ts`) + const command = [ + "run", + "--no-install", + "--no-env-file", + "--preload", + guard, + "--conditions=browser", + source, + ...("arguments" in input + ? [] + : [ + "run", + "--format", + "json", + "--model", + "fixture/fixture", + "--command", + "argumentprobe", + ...input.operands.map((value) => (ownedFile && value === "OWNED_REDIRECT_TARGET" ? ownedFile : value)), + ]), + ] + const child = spawn(process.execPath, command, { + cwd, + env, + detached: true, + stdio: ["stdin" in input ? "pipe" : "ignore", "pipe", "pipe"], + }) + cleanup.push(async () => { + if (!child.pid) return + try { + process.kill(-child.pid, "SIGKILL") + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return + throw error + } + await pause(150) + try { + process.kill(-child.pid, 0) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return + throw error + } + throw new Error("Fixture process group remains after final cleanup") }) - if (text !== undefined) await fs.unlink(ownedFile) - const removed = await fs.stat(ownedFile).then( - () => false, - (error: NodeJS.ErrnoException) => { - if (error.code === "ENOENT") return true + let stdout = "", + stderr = "", + deadline = false + const exists = () => { + try { + process.kill(-child.pid!, 0) + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false throw error + } + } + let staged = false + let firstBytesAt: number | undefined + let eofAt: number | undefined + let requestsBeforeEOF: number | undefined + const started = performance.now() + const stages: ReturnType[] = [] + child.stdin?.on("error", (error: NodeJS.ErrnoException) => { + if (error.code !== "EPIPE") stderr += String(error) + }) + child.stdout!.on("data", (value) => { + stdout += value.toString() + if ("stdin" in input && !staged && stdout.includes('"type":"invocation_start"')) { + staged = true + stages.push( + setTimeout(() => { + firstBytesAt = performance.now() - started + child.stdin!.write(input.stdin ?? "") + stages.push( + setTimeout(() => { + eofAt = performance.now() - started + requestsBeforeEOF = requestArrivals + child.stdin!.end() + }, 100), + ) + }, 100), + ) + } + }) + child.stderr!.on("data", (value) => { + stderr += value.toString() + }) + const timer = setTimeout(() => { + deadline = true + if (exists()) process.kill(-child.pid!, "SIGTERM") + }, input.deadlineMs ?? 12000) + const escalation = setTimeout( + () => { + if (deadline && exists()) process.kill(-child.pid!, "SIGKILL") }, + (input.deadlineMs ?? 12000) + 200, ) - redirected = { path: ownedFile, text, removed } - } - const result = { - input, - redirected, - template: input.template ?? template, - firstBytesAt, - eofAt, - requestsBeforeEOF, - command, - cwd, - code, - deadline, - groupGone: !exists(), - stdout, - stderr, - requests, - events, - network, - observations: (await fs.readFile(path.join(cwd, "observations.jsonl"), "utf8")) + cleanup.push(() => { + for (const stage of stages) clearTimeout(stage) + clearTimeout(timer) + clearTimeout(escalation) + child.stdin?.destroy() + }) + let code: number | null = null + try { + code = await new Promise((resolve, reject) => { + child.once("error", reject) + child.once("close", (value) => resolve(value)) + }) + } catch (error) { + failed = true + throw error + } finally { + try { + for (const stage of stages) clearTimeout(stage) + child.stdin?.destroy() + clearTimeout(timer) + clearTimeout(escalation) + if (exists()) { + process.kill(-child.pid!, "SIGTERM") + await pause(150) + if (exists()) process.kill(-child.pid!, "SIGKILL") + } + } catch (error) { + console.error("Fixture child cleanup failed", { directory: cwd, error }) + if (!failed) throw error + } + } + // Partial fetch/install observations only; coverage is documented in README.md. + const network = (await fs.readFile(ledger, "utf8")) .split("\n") .filter(Boolean) - .map((row) => JSON.parse(row)), - } - if (process.env.COMMAND_ARGUMENTS_EVIDENCE) { - await fs.mkdir(process.env.COMMAND_ARGUMENTS_EVIDENCE, { recursive: true }) - const prefix = "entry" in input ? input.entry : "legacy" - await fs.writeFile( - path.join(process.env.COMMAND_ARGUMENTS_EVIDENCE, `${prefix}-${input.name}.json`), - JSON.stringify(result, null, 2) + "\n", - ) + .map((value) => JSON.parse(value)) + const events = stdout + .split("\n") + .filter((value) => value.startsWith("{")) + .map((value) => JSON.parse(value)) + let redirected: { path: string; text?: string; removed: boolean } | undefined + if (ownedFile) { + const text = await fs.readFile(ownedFile, "utf8").catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined + throw error + }) + if (text !== undefined) await fs.unlink(ownedFile) + const removed = await fs.stat(ownedFile).then( + () => false, + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return true + throw error + }, + ) + redirected = { path: ownedFile, text, removed } + } + const result = { + input, + redirected, + template: input.template ?? template, + firstBytesAt, + eofAt, + requestsBeforeEOF, + requestArrivals, + command, + cwd, + code, + deadline, + groupGone: !exists(), + stdout, + stderr, + requests, + events, + network, + observations: (await fs.readFile(path.join(cwd, "observations.jsonl"), "utf8")) + .split("\n") + .filter(Boolean) + .map((row) => JSON.parse(row)), + } + if (evidence) { + await fs.mkdir(evidence, { recursive: true }) + const prefix = "entry" in input ? input.entry : "legacy" + await fs.writeFile(path.join(evidence, `${prefix}-${input.name}.json`), JSON.stringify(result, null, 2) + "\n") + } + if (deadline) throw new Error("Command argument fixture deadline fired: " + JSON.stringify(result)) + if (!result.groupGone) throw new Error("Command argument fixture process group remains") + const denied = network + .filter((row: { allowed?: boolean }) => row.allowed === false) + .map((row: { kind: string; origin?: string; path?: string }) => ({ + kind: row.kind, + origin: row.origin, + path: row.path, + })) + if (denied.length) + throw new Error( + "Command argument fixture blocked external attempt: " + + JSON.stringify({ code, requestCount: requests.length, denied }), + ) + expect(code, stderr + stdout).toBe(0) + expect(requests).toHaveLength(1) + return result + } catch (error) { + failed = true + throw error + } finally { + const errors: unknown[] = [] + for (const dispose of cleanup.reverse()) { + try { + await dispose() + } catch (error) { + errors.push(error) + } + } + if (errors.length) { + console.error("Fixture cleanup failed", { directory: cwd, errors }) + if (!failed) throw new AggregateError(errors, "Fixture cleanup failed") + } } - if (deadline) throw new Error("Command argument fixture deadline fired: " + JSON.stringify(result)) - if (!result.groupGone) throw new Error("Command argument fixture process group remains") - if (network.some((row) => row.allowed === false)) throw new Error("Command argument fixture blocked external attempt") - expect(code, stderr + stdout).toBe(0) - expect(requests).toHaveLength(1) - return result } export function userText(body: Record) { const messages = body.messages as { role: string; content: string | { type: string; text?: string }[] }[] diff --git a/packages/cli/test/cli/fixture/integrated-capture.ts b/packages/cli/test/cli/fixture/integrated-capture.ts index 75fcc094..57be5b11 100644 --- a/packages/cli/test/cli/fixture/integrated-capture.ts +++ b/packages/cli/test/cli/fixture/integrated-capture.ts @@ -29,305 +29,365 @@ export type Input = { } export async function capture(input: Input) { const cwd = await fs.mkdtemp(path.join(os.tmpdir(), "aictrl-command-arguments-")) - const files: Record = { - "AGENTS.md": "ROOT_AUTO_MARKER", - ".opencode/AGENTS.md": "OPENCODE_AUTO_MARKER", - "nested/AGENTS.md": "NESTED_AUTO_MARKER", - "nested/fixture.txt": "READ_FIXTURE_EXACT\n", - "config/aictrl/AGENTS.md": "TRUSTED_GLOBAL_MARKER", - "profile/AGENTS.md": "TRUSTED_PROFILE_MARKER", - "explicit.txt": "EXPLICIT_CALLER_MARKER", - "project-rule.txt": "PROJECT_RULE_MARKER", - } - if (input.profile === false) delete files["profile/AGENTS.md"] - const initialized = spawnSync("git", ["init", "--quiet"], { - cwd, - env: { - PATH: [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter), - HOME: path.join(cwd, "home"), - GIT_CONFIG_NOSYSTEM: "1", - GIT_CONFIG_GLOBAL: "/dev/null", + const evidence = process.env.COMMAND_ARGUMENTS_EVIDENCE + const cleanup: (() => void | Promise)[] = [ + async () => { + if (!evidence) await fs.rm(cwd, { recursive: true, force: true }) }, - }) - if (initialized.status !== 0) throw new Error("Owned fixture git init failed: " + initialized.stderr.toString()) - if (input.globalInstructions === false) delete files["config/aictrl/AGENTS.md"] - for (const [name, text] of Object.entries(files)) { - await fs.mkdir(path.dirname(path.join(cwd, name)), { recursive: true }) - await fs.writeFile(path.join(cwd, name), text) - } - await fs.writeFile(path.join(cwd, "aictrl.json"), JSON.stringify({ instructions: ["project-rule.txt"] })) - const http: { method: string; path: string; body: unknown }[] = [] - const requests: Record[] = [] - const server = Bun.serve({ - hostname: "127.0.0.1", - port: 0, - async fetch(request) { - const route = new URL(request.url).pathname - const body = request.method === "POST" ? await request.json() : null - http.push({ method: request.method, path: route, body }) - if (route !== "/v1/chat/completions") return Response.json({ unexpected: route }, { status: 500 }) - requests.push(body as Record) - const call = input.read && requests.length === 1 + ] + let failed = false + try { + const files: Record = { + "AGENTS.md": "ROOT_AUTO_MARKER", + ".opencode/AGENTS.md": "OPENCODE_AUTO_MARKER", + "nested/AGENTS.md": "NESTED_AUTO_MARKER", + "nested/fixture.txt": "READ_FIXTURE_EXACT\n", + "config/aictrl/AGENTS.md": "TRUSTED_GLOBAL_MARKER", + "profile/AGENTS.md": "TRUSTED_PROFILE_MARKER", + "explicit.txt": "EXPLICIT_CALLER_MARKER", + "project-rule.txt": "PROJECT_RULE_MARKER", + } + if (input.profile === false) delete files["profile/AGENTS.md"] + const initialized = spawnSync("git", ["init", "--quiet"], { + cwd, + env: { + PATH: [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter), + HOME: path.join(cwd, "home"), + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + }, + }) + if (initialized.status !== 0) throw new Error("Owned fixture git init failed: " + initialized.stderr.toString()) + if (input.globalInstructions === false) delete files["config/aictrl/AGENTS.md"] + for (const [name, text] of Object.entries(files)) { + await fs.mkdir(path.dirname(path.join(cwd, name)), { recursive: true }) + await fs.writeFile(path.join(cwd, name), text) + } + await fs.writeFile(path.join(cwd, "aictrl.json"), JSON.stringify({ instructions: ["project-rule.txt"] })) + const http: { method: string; path: string; body: unknown }[] = [] + const requests: Record[] = [] + let requestArrivals = 0 + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + requestArrivals++ + const route = new URL(request.url).pathname + const arrival: (typeof http)[number] = { method: request.method, path: route, body: undefined } + http.push(arrival) + const body = request.method === "POST" ? await request.json() : null + arrival.body = body + if (route !== "/v1/chat/completions") return Response.json({ unexpected: route }, { status: 500 }) + requests.push(body as Record) + const call = input.read && requests.length === 1 - const chunks = [ - { - choices: [ - { - index: 0, - delta: { - role: "assistant", - ...(call - ? { - tool_calls: [ - { - index: 0, - id: "read_integrated_1", - type: "function", - function: { - name: "read", - arguments: JSON.stringify({ filePath: path.join(cwd, "nested/fixture.txt") }), + const chunks = [ + { + choices: [ + { + index: 0, + delta: { + role: "assistant", + ...(call + ? { + tool_calls: [ + { + index: 0, + id: "read_integrated_1", + type: "function", + function: { + name: "read", + arguments: JSON.stringify({ filePath: path.join(cwd, "nested/fixture.txt") }), + }, }, - }, - ], - } - : { content: "INTEGRATED_DONE" }), + ], + } + : { content: "INTEGRATED_DONE" }), + }, + finish_reason: null, }, - finish_reason: null, - }, - ], - }, - { - choices: [{ index: 0, delta: {}, finish_reason: call ? "tool_calls" : "stop" }], - usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + ], + }, + { + choices: [{ index: 0, delta: {}, finish_reason: call ? "tool_calls" : "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }, + ] + return new Response(chunks.map((value) => `data: ${JSON.stringify(value)}\n\n`).join("") + "data: [DONE]\n\n", { + headers: { "content-type": "text/event-stream" }, + }) + }, + }) + cleanup.push(() => server.stop(true)) + const config = { + enabled_providers: ["fixture"], + provider: { + fixture: { + npm: "@ai-sdk/openai-compatible", + options: { apiKey: "fixture", baseURL: server.url.origin + "/v1" }, + models: { fixture: { name: "fixture", tool_call: true, limit: { context: 100000, output: 1000 } } }, }, - ] - return new Response(chunks.map((value) => `data: ${JSON.stringify(value)}\n\n`).join("") + "data: [DONE]\n\n", { - headers: { "content-type": "text/event-stream" }, - }) - }, - }) - const config = { - enabled_providers: ["fixture"], - provider: { - fixture: { - npm: "@ai-sdk/openai-compatible", - options: { apiKey: "fixture", baseURL: server.url.origin + "/v1" }, - models: { fixture: { name: "fixture", tool_call: true, limit: { context: 100000, output: 1000 } } }, }, - }, - agent: { title: { disable: true } }, - permission: { "*": "allow", ...(input.denyRead ? { read: "deny" } : {}) }, - instructions: [path.join(cwd, "explicit.txt")], - plugin: [new URL("./integrated-observer-plugin.ts", import.meta.url).href], - command: { - argumentprobe: { template: "STATIC A<$1> B<$2> C<$3> RAW<$ARGUMENTS>" }, - }, - ...(input.configSelection !== undefined ? { tool_allowlist: input.configSelection } : {}), - } - await fs.writeFile(path.join(cwd, "observations.jsonl"), "") - const ledger = path.join(cwd, "network.jsonl") - await fs.writeFile(ledger, "") - const env = { - PATH: [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter), - HOME: path.join(cwd, "home"), - AICTRL_TEST_HOME: path.join(cwd, "home"), - AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(cwd, "managed"), - XDG_DATA_HOME: path.join(cwd, "data"), - XDG_CONFIG_HOME: path.join(cwd, "config"), - XDG_CACHE_HOME: path.join(cwd, "cache"), - XDG_STATE_HOME: path.join(cwd, "state"), - BUN_RUNTIME_TRANSPILER_CACHE_PATH: path.join(cwd, "bun-cache"), - GIT_CONFIG_NOSYSTEM: "1", - GIT_CONFIG_GLOBAL: "/dev/null", - TZ: "UTC", - AICTRL_DISABLE_DEFAULT_PLUGINS: "true", - AICTRL_DISABLE_AUTOCOMPACT: "true", - STDIN_FIXTURE_ORIGIN: server.url.origin, - STDIN_FIXTURE_LEDGER: ledger, - AICTRL_CONFIG_CONTENT: JSON.stringify(config), - AICTRL_DISABLE_PROJECT_CONFIG: String(input.disableProject !== false), - ...(input.profile === false ? {} : { AICTRL_CONFIG_DIR: path.join(cwd, "profile") }), - COMMAND_TEMPLATE_OBSERVATIONS: path.join(cwd, "observations.jsonl"), - } - const source = path.resolve(import.meta.dir, `../../../src/${input.entry}.ts`) - const command = [ - "run", - "--no-install", - "--no-env-file", - "--preload", - guard, - "--conditions=browser", - source, - "run", - "--format", - "json", - "--model", - "fixture/fixture", - ...(input.attach ? ["--attach", server.url.origin] : []), - ...(input.tools !== undefined ? ["--tools", input.tools] : []), - ...(input.extraArgs ?? []), - "--command", - "argumentprobe", - ...input.operands, - ] - const child = spawn(process.execPath, command, { - cwd, - env, - detached: true, - stdio: ["stdin" in input || input.open ? "pipe" : "ignore", "pipe", "pipe"], - }) - let stdout = "", - stderr = "", - deadline = false - const exists = () => { - try { - process.kill(-child.pid!, 0) - return true - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ESRCH") return false - throw error + agent: { title: { disable: true } }, + permission: { "*": "allow", ...(input.denyRead ? { read: "deny" } : {}) }, + instructions: [path.join(cwd, "explicit.txt")], + plugin: [new URL("./integrated-observer-plugin.ts", import.meta.url).href], + command: { + argumentprobe: { template: "STATIC A<$1> B<$2> C<$3> RAW<$ARGUMENTS>" }, + }, + ...(input.configSelection !== undefined ? { tool_allowlist: input.configSelection } : {}), } - } - let staged = false - let firstBytesAt: number | undefined - let eofAt: number | undefined - let requestsBeforeEOF: number | undefined - let beforeEOF: { http: unknown[]; dbExists: boolean; projectCacheExists: boolean; observations: string } | undefined - const started = performance.now() - const stages: ReturnType[] = [] - child.stdin?.on("error", (error: NodeJS.ErrnoException) => { - if (error.code !== "EPIPE") stderr += String(error) - }) - child.stdout!.on("data", (value) => { - stdout += value.toString() - if ("stdin" in input && !staged && stdout.includes('"type":"invocation_start"')) { - staged = true - stages.push( - setTimeout(() => { - firstBytesAt = performance.now() - started - child.stdin!.write(input.stdin ?? "") - stages.push( - setTimeout(() => { - eofAt = performance.now() - started - requestsBeforeEOF = requests.length - beforeEOF = { - http: [...http], - dbExists: existsSync(path.join(cwd, "data/aictrl/aictrl.db")), - projectCacheExists: existsSync(path.join(cwd, ".git/aictrl")), - observations: readFileSync(path.join(cwd, "observations.jsonl"), "utf8"), - } - child.stdin!.end() - }, 100), - ) - }, 100), - ) + await fs.writeFile(path.join(cwd, "observations.jsonl"), "") + const ledger = path.join(cwd, "network.jsonl") + await fs.writeFile(ledger, "") + const env = { + PATH: [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter), + HOME: path.join(cwd, "home"), + AICTRL_TEST_HOME: path.join(cwd, "home"), + AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(cwd, "managed"), + XDG_DATA_HOME: path.join(cwd, "data"), + XDG_CONFIG_HOME: path.join(cwd, "config"), + XDG_CACHE_HOME: path.join(cwd, "cache"), + XDG_STATE_HOME: path.join(cwd, "state"), + BUN_RUNTIME_TRANSPILER_CACHE_PATH: path.join(cwd, "bun-cache"), + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + TZ: "UTC", + AICTRL_DISABLE_DEFAULT_PLUGINS: "true", + AICTRL_DISABLE_AUTOCOMPACT: "true", + STDIN_FIXTURE_ORIGIN: server.url.origin, + STDIN_FIXTURE_LEDGER: ledger, + AICTRL_CONFIG_CONTENT: JSON.stringify(config), + AICTRL_DISABLE_PROJECT_CONFIG: String(input.disableProject !== false), + ...(input.profile === false ? {} : { AICTRL_CONFIG_DIR: path.join(cwd, "profile") }), + COMMAND_TEMPLATE_OBSERVATIONS: path.join(cwd, "observations.jsonl"), } - }) - child.stderr!.on("data", (value) => { - stderr += value.toString() - }) - const timer = setTimeout(() => { - deadline = true - if (exists()) process.kill(-child.pid!, "SIGTERM") - }, input.deadlineMs ?? 12000) - const escalation = setTimeout( - () => { - if (deadline && exists()) process.kill(-child.pid!, "SIGKILL") - }, - (input.deadlineMs ?? 12000) + 200, - ) - let code: number | null = null - try { - code = await new Promise((resolve, reject) => { - child.once("error", reject) - child.once("close", (value) => resolve(value)) + const source = path.resolve(import.meta.dir, `../../../src/${input.entry}.ts`) + const command = [ + "run", + "--no-install", + "--no-env-file", + "--preload", + guard, + "--conditions=browser", + source, + "run", + "--format", + "json", + "--model", + "fixture/fixture", + ...(input.attach ? ["--attach", server.url.origin] : []), + ...(input.tools !== undefined ? ["--tools", input.tools] : []), + ...(input.extraArgs ?? []), + "--command", + "argumentprobe", + ...input.operands, + ] + const child = spawn(process.execPath, command, { + cwd, + env, + detached: true, + stdio: ["stdin" in input || input.open ? "pipe" : "ignore", "pipe", "pipe"], }) - } finally { - for (const stage of stages) clearTimeout(stage) - child.stdin?.destroy() - clearTimeout(timer) - clearTimeout(escalation) - if (exists()) { - process.kill(-child.pid!, "SIGTERM") + cleanup.push(async () => { + if (!child.pid) return + try { + process.kill(-child.pid, "SIGKILL") + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return + throw error + } await pause(150) - if (exists()) process.kill(-child.pid!, "SIGKILL") + try { + process.kill(-child.pid, 0) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return + throw error + } + throw new Error("Fixture process group remains after final cleanup") + }) + let stdout = "", + stderr = "", + deadline = false + const exists = () => { + try { + process.kill(-child.pid!, 0) + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false + throw error + } } - server.stop(true) - } - const network = (await fs.readFile(ledger, "utf8")) - .split("\n") - .filter(Boolean) - .map((value) => JSON.parse(value)) - const events = stdout - .split("\n") - .filter((value) => value.startsWith("{")) - .map((value) => JSON.parse(value)) - const dbPath = path.join(cwd, "data/aictrl/aictrl.db") - let sessions: { id: string; title: string }[] = [], - messages: { data: Record }[] = [] - const dbExists = await fs.stat(dbPath).then( - () => true, - (error: NodeJS.ErrnoException) => { - if (error.code === "ENOENT") return false - throw error - }, - ) - if (dbExists) { - const db = new Database(dbPath, { readonly: true }) + let staged = false + let firstBytesAt: number | undefined + let eofAt: number | undefined + let requestsBeforeEOF: number | undefined + let beforeEOF: { http: unknown[]; dbExists: boolean; projectCacheExists: boolean; observations: string } | undefined + const started = performance.now() + const stages: ReturnType[] = [] + child.stdin?.on("error", (error: NodeJS.ErrnoException) => { + if (error.code !== "EPIPE") stderr += String(error) + }) + child.stdout!.on("data", (value) => { + stdout += value.toString() + if ("stdin" in input && !staged && stdout.includes('"type":"invocation_start"')) { + staged = true + stages.push( + setTimeout(() => { + firstBytesAt = performance.now() - started + child.stdin!.write(input.stdin ?? "") + stages.push( + setTimeout(() => { + eofAt = performance.now() - started + requestsBeforeEOF = requestArrivals + beforeEOF = { + http: http.map((arrival) => ({ ...arrival })), + dbExists: existsSync(path.join(cwd, "data/aictrl/aictrl.db")), + projectCacheExists: existsSync(path.join(cwd, ".git/aictrl")), + observations: readFileSync(path.join(cwd, "observations.jsonl"), "utf8"), + } + child.stdin!.end() + }, 100), + ) + }, 100), + ) + } + }) + child.stderr!.on("data", (value) => { + stderr += value.toString() + }) + const timer = setTimeout(() => { + deadline = true + if (exists()) process.kill(-child.pid!, "SIGTERM") + }, input.deadlineMs ?? 12000) + const escalation = setTimeout( + () => { + if (deadline && exists()) process.kill(-child.pid!, "SIGKILL") + }, + (input.deadlineMs ?? 12000) + 200, + ) + cleanup.push(() => { + for (const stage of stages) clearTimeout(stage) + clearTimeout(timer) + clearTimeout(escalation) + child.stdin?.destroy() + }) + let code: number | null = null try { - sessions = db.query("SELECT id,title FROM session").all() as typeof sessions - messages = (db.query("SELECT data FROM message").all() as { data: string }[]).map((row) => ({ - data: JSON.parse(row.data), - })) + code = await new Promise((resolve, reject) => { + child.once("error", reject) + child.once("close", (value) => resolve(value)) + }) + } catch (error) { + failed = true + throw error } finally { - db.close() + try { + for (const stage of stages) clearTimeout(stage) + child.stdin?.destroy() + clearTimeout(timer) + clearTimeout(escalation) + if (exists()) { + process.kill(-child.pid!, "SIGTERM") + await pause(150) + if (exists()) process.kill(-child.pid!, "SIGKILL") + } + } catch (error) { + console.error("Fixture child cleanup failed", { directory: cwd, error }) + if (!failed) throw error + } } - } - const result = { - input, - config, - http, - dbExists, - sessions, - messages, - projectCacheExists: await fs.stat(path.join(cwd, ".git/aictrl")).then( + // Partial fetch/install observations only; coverage is documented in README.md. + const network = (await fs.readFile(ledger, "utf8")) + .split("\n") + .filter(Boolean) + .map((value) => JSON.parse(value)) + const events = stdout + .split("\n") + .filter((value) => value.startsWith("{")) + .map((value) => JSON.parse(value)) + const dbPath = path.join(cwd, "data/aictrl/aictrl.db") + let sessions: { id: string; title: string }[] = [], + messages: { data: Record }[] = [] + const dbExists = await fs.stat(dbPath).then( () => true, (error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return false throw error }, - ), - firstBytesAt, - eofAt, - requestsBeforeEOF, - beforeEOF, - command, - cwd, - code, - deadline, - groupGone: !exists(), - stdout, - stderr, - requests, - events, - network, - observations: (await fs.readFile(path.join(cwd, "observations.jsonl"), "utf8")) - .split("\n") - .filter(Boolean) - .map((row) => JSON.parse(row)), - } - if (process.env.COMMAND_ARGUMENTS_EVIDENCE) { - await fs.mkdir(process.env.COMMAND_ARGUMENTS_EVIDENCE, { recursive: true }) - const prefix = input.entry - await fs.writeFile( - path.join(process.env.COMMAND_ARGUMENTS_EVIDENCE, `${prefix}-${input.name}.json`), - JSON.stringify(result, null, 2) + "\n", ) + if (dbExists) { + const db = new Database(dbPath, { readonly: true }) + try { + sessions = db.query("SELECT id,title FROM session").all() as typeof sessions + messages = (db.query("SELECT data FROM message").all() as { data: string }[]).map((row) => ({ + data: JSON.parse(row.data), + })) + } finally { + db.close() + } + } + const result = { + input, + config, + http, + dbExists, + sessions, + messages, + projectCacheExists: await fs.stat(path.join(cwd, ".git/aictrl")).then( + () => true, + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return false + throw error + }, + ), + firstBytesAt, + eofAt, + requestsBeforeEOF, + requestArrivals, + beforeEOF, + command, + cwd, + code, + deadline, + groupGone: !exists(), + stdout, + stderr, + requests, + events, + network, + observations: (await fs.readFile(path.join(cwd, "observations.jsonl"), "utf8")) + .split("\n") + .filter(Boolean) + .map((row) => JSON.parse(row)), + } + if (evidence) { + await fs.mkdir(evidence, { recursive: true }) + const prefix = input.entry + await fs.writeFile(path.join(evidence, `${prefix}-${input.name}.json`), JSON.stringify(result, null, 2) + "\n") + } + if (deadline) throw new Error("Command argument fixture deadline fired: " + JSON.stringify(result)) + if (!result.groupGone) throw new Error("Command argument fixture process group remains") + if (network.some((row) => row.allowed === false)) + throw new Error("Command argument fixture blocked external attempt") + return result + } catch (error) { + failed = true + throw error + } finally { + const errors: unknown[] = [] + for (const dispose of cleanup.reverse()) { + try { + await dispose() + } catch (error) { + errors.push(error) + } + } + if (errors.length) { + console.error("Fixture cleanup failed", { directory: cwd, errors }) + if (!failed) throw new AggregateError(errors, "Fixture cleanup failed") + } } - if (deadline) throw new Error("Command argument fixture deadline fired: " + JSON.stringify(result)) - if (!result.groupGone) throw new Error("Command argument fixture process group remains") - if (network.some((row) => row.allowed === false)) throw new Error("Command argument fixture blocked external attempt") - return result } export function userText(body: Record) { const messages = body.messages as { role: string; content: string | { type: string; text?: string }[] }[] diff --git a/packages/cli/test/cli/request-arrival.test.ts b/packages/cli/test/cli/request-arrival.test.ts new file mode 100644 index 00000000..9e14d766 --- /dev/null +++ b/packages/cli/test/cli/request-arrival.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, test } from "bun:test" +import http from "node:http" +import fs from "node:fs/promises" +import path from "node:path" +import ts from "typescript" + +// Exercise the actual owned handler bodies without starting a CLI or claiming +// that a CLI request has been reproduced before stdin EOF. +const fixtures = [ + ["template", "fixture/command-template-capture.ts", "/v1/chat/completions"], + ["integrated", "fixture/integrated-capture.ts", "/v1/chat/completions"], + ["stdin", "run-stdin.test.ts", "/session"], +] as const + +describe("server-observed request arrival", () => { + test.each(fixtures)("%s records arrival while JSON is incomplete", async (name, file, route) => { + const source = ts.createSourceFile( + file, + await fs.readFile(path.join(import.meta.dir, file), "utf8"), + ts.ScriptTarget.Latest, + true, + ) + let handler: ts.MethodDeclaration | undefined + const visit = (node: ts.Node) => { + if (handler) return + if (ts.isCallExpression(node) && node.expression.getText(source) === "Bun.serve") { + const options = node.arguments[0] + if (!options || !ts.isObjectLiteralExpression(options)) throw new Error("Fixture serve options changed") + handler = options.properties.find( + (property): property is ts.MethodDeclaration => + ts.isMethodDeclaration(property) && property.name.getText(source) === "fetch", + ) + if (!handler) throw new Error("Fixture fetch handler missing") + return + } + ts.forEachChild(node, visit) + } + visit(source) + if (!handler) throw new Error("Fixture server missing") + const compiled = new Bun.Transpiler({ loader: "ts" }).transformSync( + `const fixture = { ${handler.getText(source)} }`, + ) + const state = { requests: [] as Record[], http: [] as Record[] } + const capture = new Function( + "state", + "path", + `const requests = state.requests, http = state.http; + let requestArrivals = 0, stream; + const input = { read: false }, cwd = "/owned-arrival-control", sessionID = "ses_stdin_fixture"; + ${compiled} + return { fetch: fixture.fetch, count: () => requestArrivals };`, + )(state, path) as { fetch: (request: Request) => Promise; count: () => number } + let enter: () => void = () => {} + const entered = new Promise((resolve) => { + enter = resolve + }) + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + enter() + return capture.fetch(request) + }, + }) + const body = JSON.stringify({ messages: [] }) + let client: http.ClientRequest | undefined + let deadline: ReturnType | undefined + try { + const response = new Promise((resolve, reject) => { + client = http.request( + new URL(route, server.url), + { + method: "POST", + headers: { "content-type": "application/json", "content-length": Buffer.byteLength(body) }, + }, + (result) => { + result.resume() + result.once("end", () => resolve(result.statusCode ?? 0)) + result.once("error", reject) + }, + ) + client.once("error", reject) + client.write(body.slice(0, 1)) + }) + const failure = new Promise((_, reject) => { + deadline = setTimeout(() => reject(new Error("Arrival control deadline")), 3000) + }) + await Promise.race([ + entered, + response.then(() => { + throw new Error("Response completed before the incomplete-body observation") + }), + failure, + ]) + // Handler entry is explicit; the remaining JSON bytes are still withheld. + expect(capture.count()).toBe(1) + expect(state.requests).toHaveLength(0) + if (name === "integrated") { + expect(state.http).toHaveLength(1) + expect(state.http[0]).toEqual({ method: "POST", path: route, body: undefined }) + } + client!.end(body.slice(1)) + expect(await Promise.race([response, failure])).toBe(200) + expect(state.requests).toHaveLength(1) + expect(capture.count()).toBe(1) + } finally { + clearTimeout(deadline) + client?.destroy() + await server.stop(true) + } + }) +}) diff --git a/packages/cli/test/cli/run-command-arguments.test.ts b/packages/cli/test/cli/run-command-arguments.test.ts index a5cbf69c..5ecef3be 100644 --- a/packages/cli/test/cli/run-command-arguments.test.ts +++ b/packages/cli/test/cli/run-command-arguments.test.ts @@ -1,8 +1,10 @@ -import { describe, expect, test } from "bun:test" +import { describe, expect, spyOn, test } from "bun:test" import { spawn } from "node:child_process" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" +import type { CommandArguments, CommandInput } from "../../../sdk/src/v2/index" +import { capture as templateCapture } from "./fixture/command-template-capture" const template = "FIRST<$1>\nSECOND<$2>\nREST<$3>" const guard = path.join(import.meta.dir, "fixture/stdin-guard.ts") @@ -14,195 +16,260 @@ type Input = ( ) & { template?: string; deadlineMs?: number } async function capture(input: Input) { const cwd = await fs.mkdtemp(path.join(os.tmpdir(), "aictrl-command-arguments-")) - const requests: Record[] = [] - const server = Bun.serve({ - hostname: "127.0.0.1", - port: 0, - async fetch(request) { - if (new URL(request.url).pathname !== "/v1/chat/completions") throw new Error("Unexpected provider path") - requests.push((await request.json()) as Record) - const chunks = [ - { choices: [{ index: 0, delta: { role: "assistant", content: "Done" }, finish_reason: null }] }, - { - choices: [{ index: 0, delta: {}, finish_reason: "stop" }], - usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, - }, - ] - return new Response(chunks.map((value) => `data: ${JSON.stringify(value)}\n\n`).join("") + "data: [DONE]\n\n", { - headers: { "content-type": "text/event-stream" }, - }) + const evidence = process.env.COMMAND_ARGUMENTS_EVIDENCE + const cleanup: (() => void | Promise)[] = [ + async () => { + if (!evidence) await fs.rm(cwd, { recursive: true, force: true }) }, - }) - await fs.writeFile( - path.join(cwd, "aictrl.json"), - JSON.stringify({ - enabled_providers: ["fixture"], - provider: { - fixture: { - npm: "@ai-sdk/openai-compatible", - options: { apiKey: "fixture", baseURL: server.url.origin + "/v1" }, - models: { fixture: { name: "fixture", tool_call: true, limit: { context: 100000, output: 1000 } } }, + ] + let failed = false + try { + const requests: Record[] = [] + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + if (new URL(request.url).pathname !== "/v1/chat/completions") throw new Error("Unexpected provider path") + requests.push((await request.json()) as Record) + const chunks = [ + { choices: [{ index: 0, delta: { role: "assistant", content: "Done" }, finish_reason: null }] }, + { + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }, + ] + return new Response(chunks.map((value) => `data: ${JSON.stringify(value)}\n\n`).join("") + "data: [DONE]\n\n", { + headers: { "content-type": "text/event-stream" }, + }) + }, + }) + cleanup.push(() => server.stop(true)) + await fs.writeFile( + path.join(cwd, "aictrl.json"), + JSON.stringify({ + enabled_providers: ["fixture"], + provider: { + fixture: { + npm: "@ai-sdk/openai-compatible", + options: { apiKey: "fixture", baseURL: server.url.origin + "/v1" }, + models: { fixture: { name: "fixture", tool_call: true, limit: { context: 100000, output: 1000 } } }, + }, }, + agent: { title: { disable: true } }, + command: { argumentprobe: { template: input.template ?? template } }, + }), + ) + const data = path.join(cwd, "arguments.json") + await fs.writeFile(data, JSON.stringify("raw" in input ? { arguments: input.raw } : {})) + const ledger = path.join(cwd, "network.jsonl") + await fs.writeFile(ledger, "") + const env = { + PATH: [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter), + HOME: path.join(cwd, "home"), + AICTRL_TEST_HOME: path.join(cwd, "home"), + AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(cwd, "managed"), + XDG_DATA_HOME: path.join(cwd, "data"), + XDG_CONFIG_HOME: path.join(cwd, "config"), + XDG_CACHE_HOME: path.join(cwd, "cache"), + XDG_STATE_HOME: path.join(cwd, "state"), + BUN_RUNTIME_TRANSPILER_CACHE_PATH: path.join(cwd, "bun-cache"), + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + TZ: "UTC", + AICTRL_DISABLE_DEFAULT_PLUGINS: "true", + AICTRL_DISABLE_AUTOCOMPACT: "true", + STDIN_FIXTURE_ORIGIN: server.url.origin, + STDIN_FIXTURE_LEDGER: ledger, + COMMAND_ARGUMENTS_DATA: data, + } + const source = + "raw" in input + ? path.join(import.meta.dir, "fixture/command-arguments-legacy.ts") + : path.resolve(import.meta.dir, `../../src/${input.entry}.ts`) + const command = [ + "run", + "--no-install", + "--no-env-file", + "--preload", + guard, + "--conditions=browser", + source, + ...("raw" in input + ? [] + : ["run", "--format", "json", "--model", "fixture/fixture", "--command", "argumentprobe", ...input.operands]), + ] + const child = spawn(process.execPath, command, { + cwd, + env, + detached: true, + stdio: ["stdin" in input ? "pipe" : "ignore", "pipe", "pipe"], + }) + cleanup.push(async () => { + if (!child.pid) return + try { + process.kill(-child.pid, "SIGKILL") + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return + throw error + } + await pause(150) + try { + process.kill(-child.pid, 0) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return + throw error + } + throw new Error("Fixture process group remains after final cleanup") + }) + let stdout = "", + stderr = "", + deadline = false + const exists = () => { + try { + process.kill(-child.pid!, 0) + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false + throw error + } + } + let staged = false + let firstBytesAt: number | undefined + let eofAt: number | undefined + let requestsBeforeEOF: number | undefined + const started = performance.now() + const stages: ReturnType[] = [] + child.stdin?.on("error", (error: NodeJS.ErrnoException) => { + if (error.code !== "EPIPE") stderr += String(error) + }) + child.stdout!.on("data", (value) => { + stdout += value.toString() + if ("stdin" in input && !staged && stdout.includes('"type":"invocation_start"')) { + staged = true + stages.push( + setTimeout(() => { + firstBytesAt = performance.now() - started + child.stdin!.write(input.stdin ?? "") + stages.push( + setTimeout(() => { + eofAt = performance.now() - started + requestsBeforeEOF = requests.length + child.stdin!.end() + }, 100), + ) + }, 100), + ) + } + }) + child.stderr!.on("data", (value) => { + stderr += value.toString() + }) + const timer = setTimeout(() => { + deadline = true + if (exists()) process.kill(-child.pid!, "SIGTERM") + }, input.deadlineMs ?? 12000) + const escalation = setTimeout( + () => { + if (deadline && exists()) process.kill(-child.pid!, "SIGKILL") }, - agent: { title: { disable: true } }, - command: { argumentprobe: { template: input.template ?? template } }, - }), - ) - const data = path.join(cwd, "arguments.json") - await fs.writeFile(data, JSON.stringify("raw" in input ? { arguments: input.raw } : {})) - const ledger = path.join(cwd, "network.jsonl") - await fs.writeFile(ledger, "") - const env = { - PATH: [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter), - HOME: path.join(cwd, "home"), - AICTRL_TEST_HOME: path.join(cwd, "home"), - AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(cwd, "managed"), - XDG_DATA_HOME: path.join(cwd, "data"), - XDG_CONFIG_HOME: path.join(cwd, "config"), - XDG_CACHE_HOME: path.join(cwd, "cache"), - XDG_STATE_HOME: path.join(cwd, "state"), - BUN_RUNTIME_TRANSPILER_CACHE_PATH: path.join(cwd, "bun-cache"), - GIT_CONFIG_NOSYSTEM: "1", - GIT_CONFIG_GLOBAL: "/dev/null", - TZ: "UTC", - AICTRL_DISABLE_DEFAULT_PLUGINS: "true", - AICTRL_DISABLE_AUTOCOMPACT: "true", - STDIN_FIXTURE_ORIGIN: server.url.origin, - STDIN_FIXTURE_LEDGER: ledger, - COMMAND_ARGUMENTS_DATA: data, - } - const source = - "raw" in input - ? path.join(import.meta.dir, "fixture/command-arguments-legacy.ts") - : path.resolve(import.meta.dir, `../../src/${input.entry}.ts`) - const command = [ - "run", - "--no-install", - "--no-env-file", - "--preload", - guard, - "--conditions=browser", - source, - ...("raw" in input - ? [] - : ["run", "--format", "json", "--model", "fixture/fixture", "--command", "argumentprobe", ...input.operands]), - ] - const child = spawn(process.execPath, command, { - cwd, - env, - detached: true, - stdio: ["stdin" in input ? "pipe" : "ignore", "pipe", "pipe"], - }) - let stdout = "", - stderr = "", - deadline = false - const exists = () => { + (input.deadlineMs ?? 12000) + 200, + ) + cleanup.push(() => { + for (const stage of stages) clearTimeout(stage) + clearTimeout(timer) + clearTimeout(escalation) + child.stdin?.destroy() + }) + let code: number | null = null try { - process.kill(-child.pid!, 0) - return true + code = await new Promise((resolve, reject) => { + child.once("error", reject) + child.once("close", (value) => resolve(value)) + }) } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ESRCH") return false + failed = true throw error + } finally { + try { + for (const stage of stages) clearTimeout(stage) + child.stdin?.destroy() + clearTimeout(timer) + clearTimeout(escalation) + if (exists()) { + process.kill(-child.pid!, "SIGTERM") + await pause(150) + if (exists()) process.kill(-child.pid!, "SIGKILL") + } + } catch (error) { + console.error("Fixture child cleanup failed", { directory: cwd, error }) + if (!failed) throw error + } } - } - let staged = false - let firstBytesAt: number | undefined - let eofAt: number | undefined - let requestsBeforeEOF: number | undefined - const started = performance.now() - const stages: ReturnType[] = [] - child.stdin?.on("error", (error: NodeJS.ErrnoException) => { - if (error.code !== "EPIPE") stderr += String(error) - }) - child.stdout!.on("data", (value) => { - stdout += value.toString() - if ("stdin" in input && !staged && stdout.includes('"type":"invocation_start"')) { - staged = true - stages.push( - setTimeout(() => { - firstBytesAt = performance.now() - started - child.stdin!.write(input.stdin ?? "") - stages.push( - setTimeout(() => { - eofAt = performance.now() - started - requestsBeforeEOF = requests.length - child.stdin!.end() - }, 100), - ) - }, 100), - ) + // Partial fetch/install observations only; coverage is documented in fixture/README.md. + const network = (await fs.readFile(ledger, "utf8")) + .split("\n") + .filter(Boolean) + .map((value) => JSON.parse(value)) + const events = stdout + .split("\n") + .filter((value) => value.startsWith("{")) + .map((value) => JSON.parse(value)) + const result = { + input, + template: input.template ?? template, + firstBytesAt, + eofAt, + requestsBeforeEOF, + command, + cwd, + code, + deadline, + groupGone: !exists(), + stdout, + stderr, + requests, + events, + network, } - }) - child.stderr!.on("data", (value) => { - stderr += value.toString() - }) - const timer = setTimeout(() => { - deadline = true - if (exists()) process.kill(-child.pid!, "SIGTERM") - }, input.deadlineMs ?? 12000) - const escalation = setTimeout( - () => { - if (deadline && exists()) process.kill(-child.pid!, "SIGKILL") - }, - (input.deadlineMs ?? 12000) + 200, - ) - let code: number | null = null - try { - code = await new Promise((resolve, reject) => { - child.once("error", reject) - child.once("close", (value) => resolve(value)) - }) + if (evidence) { + await fs.mkdir(evidence, { recursive: true }) + const prefix = "entry" in input ? input.entry : "legacy" + await fs.writeFile(path.join(evidence, `${prefix}-${input.name}.json`), JSON.stringify(result, null, 2) + "\n") + } + if (deadline) throw new Error("Command argument fixture deadline fired: " + JSON.stringify(result)) + if (!result.groupGone) throw new Error("Command argument fixture process group remains") + const denied = network + .filter((row: { allowed?: boolean }) => row.allowed === false) + .map((row: { kind: string; origin?: string; path?: string }) => ({ + kind: row.kind, + origin: row.origin, + path: row.path, + })) + if (denied.length) + throw new Error( + "Command argument fixture blocked external attempt: " + + JSON.stringify({ code, requestCount: requests.length, denied }), + ) + expect(code, stderr + stdout).toBe(0) + expect(requests).toHaveLength(1) + return result + } catch (error) { + failed = true + throw error } finally { - for (const stage of stages) clearTimeout(stage) - child.stdin?.destroy() - clearTimeout(timer) - clearTimeout(escalation) - if (exists()) { - process.kill(-child.pid!, "SIGTERM") - await pause(150) - if (exists()) process.kill(-child.pid!, "SIGKILL") + const errors: unknown[] = [] + for (const dispose of cleanup.reverse()) { + try { + await dispose() + } catch (error) { + errors.push(error) + } + } + if (errors.length) { + console.error("Fixture cleanup failed", { directory: cwd, errors }) + if (!failed) throw new AggregateError(errors, "Fixture cleanup failed") } - server.stop(true) - } - const network = (await fs.readFile(ledger, "utf8")) - .split("\n") - .filter(Boolean) - .map((value) => JSON.parse(value)) - const events = stdout - .split("\n") - .filter((value) => value.startsWith("{")) - .map((value) => JSON.parse(value)) - const result = { - input, - template: input.template ?? template, - firstBytesAt, - eofAt, - requestsBeforeEOF, - command, - cwd, - code, - deadline, - groupGone: !exists(), - stdout, - stderr, - requests, - events, - network, - } - if (process.env.COMMAND_ARGUMENTS_EVIDENCE) { - await fs.mkdir(process.env.COMMAND_ARGUMENTS_EVIDENCE, { recursive: true }) - const prefix = "entry" in input ? input.entry : "legacy" - await fs.writeFile( - path.join(process.env.COMMAND_ARGUMENTS_EVIDENCE, `${prefix}-${input.name}.json`), - JSON.stringify(result, null, 2) + "\n", - ) } - if (deadline) throw new Error("Command argument fixture deadline fired: " + JSON.stringify(result)) - if (!result.groupGone) throw new Error("Command argument fixture process group remains") - if (network.some((row) => row.allowed === false)) throw new Error("Command argument fixture blocked external attempt") - expect(code, stderr + stdout).toBe(0) - expect(requests).toHaveLength(1) - return result } function userText(body: Record) { const messages = body.messages as { role: string; content: string | { type: string; text?: string }[] }[] @@ -219,6 +286,99 @@ function userText(body: Record) { .join("\n") } +// These controls fail during real fixture-file setup, after the real listener starts. +for (const fixture of [ + { name: "arguments", invoke: () => capture({ entry: "index", name: "setup-failure", operands: ["inert"] }) }, + { name: "template", invoke: () => templateCapture({ name: "setup-failure", arguments: "inert" }) }, +]) { + for (const secondary of [undefined, new Error("listener cleanup failed after close")]) { + test(`${fixture.name}: setup failure cleans owned resources${secondary ? " despite a cleanup error" : ""}`, async () => { + const primary = new Error("fixture setup write failed") + const listener = spyOn(Bun, "serve") + const logging = secondary ? spyOn(console, "error").mockImplementation(() => {}) : undefined + let directory: string | undefined + let stop: { mockRestore: () => void; mock: { calls: unknown[][] } } | undefined + const write = spyOn(fs, "writeFile").mockImplementationOnce(async (file) => { + directory = path.dirname(String(file)) + const server = listener.mock.results[0]?.value as ReturnType | undefined + if (!server) throw new Error("Expected real fixture listener before setup write") + const close = server.stop.bind(server) + const dispose = spyOn(server, "stop") + stop = dispose + if (secondary) + dispose.mockImplementation(async (active) => { + await close(active) + throw secondary + }) + throw primary + }) + try { + await expect(fixture.invoke()).rejects.toBe(primary) + expect(stop?.mock.calls).toEqual([[true]]) + expect(directory).toBeDefined() + if (secondary) + expect(logging).toHaveBeenCalledWith("Fixture cleanup failed", { directory, errors: [secondary] }) + if (!process.env.COMMAND_ARGUMENTS_EVIDENCE) + await expect(fs.stat(directory!)).rejects.toMatchObject({ code: "ENOENT" }) + } finally { + write.mockRestore() + stop?.mockRestore() + listener.mockRestore() + logging?.mockRestore() + } + }) + } +} + +test("template fixture removes its owned directory after reading final evidence", async () => { + const result = await templateCapture({ name: "directory-cleanup", arguments: "alpha beta rest" }) + expect(userText(result.requests[0])).toBe("FIRST\nSECOND\nREST") + if (!process.env.COMMAND_ARGUMENTS_EVIDENCE) + await expect(fs.stat(result.cwd)).rejects.toMatchObject({ code: "ENOENT" }) +}, 15000) + +test("successful capture rejects a listener cleanup failure and still removes its directory", async () => { + const secondary = new Error("listener cleanup failed after successful capture") + const listener = spyOn(Bun, "serve") + const logging = spyOn(console, "error").mockImplementation(() => {}) + const writeFile = fs.writeFile + let directory: string | undefined + let restore: (() => void) | undefined + const write = spyOn(fs, "writeFile").mockImplementationOnce(async (file, data, options) => { + directory = path.dirname(String(file)) + const server = listener.mock.results[0]?.value as ReturnType | undefined + if (!server) throw new Error("Expected real fixture listener before setup write") + const close = server.stop.bind(server) + const stop = spyOn(server, "stop").mockImplementation(async (active) => { + await close(active) + throw secondary + }) + restore = () => stop.mockRestore() + return writeFile(file, data, options) + }) + try { + await expect(templateCapture({ name: "cleanup-failure", arguments: "alpha beta rest" })).rejects.toBeInstanceOf( + AggregateError, + ) + expect(logging).toHaveBeenCalledWith("Fixture cleanup failed", { directory, errors: [secondary] }) + expect(directory).toBeDefined() + if (!process.env.COMMAND_ARGUMENTS_EVIDENCE) + await expect(fs.stat(directory!)).rejects.toMatchObject({ code: "ENOENT" }) + } finally { + write.mockRestore() + restore?.() + listener.mockRestore() + logging.mockRestore() + } +}, 15000) + +test("public SDK v2 command types match the canonical structured schema", async () => { + const { SessionPrompt } = await import("../../src/session/prompt") + const argumentsData: CommandArguments = { argv: ["alpha beta", ""], stdin: "late λ\n" } + const input: CommandInput = { sessionID: "ses_fixture", command: "argumentprobe", arguments: argumentsData } + expect(SessionPrompt.CommandInput.parse(input).arguments).toEqual(argumentsData) +}) + // Literal expected templates are ground truth from original operand boundaries, not a copy of the decoder. const cases = [ { name: "plain-control", operands: ["alpha", "beta", "rest"], expected: "FIRST\nSECOND\nREST" }, @@ -282,6 +442,8 @@ describe("actual local command argument grouping", () => { test(`${entry}: ${item.name}`, async () => { const result = await capture({ entry, name: item.name, operands: item.operands }) expect(userText(result.requests[0])).toBe(item.expected) + if (!process.env.COMMAND_ARGUMENTS_EVIDENCE) + await expect(fs.stat(result.cwd)).rejects.toMatchObject({ code: "ENOENT" }) const emitted = result.events.find((event) => event.type === "user_message") expect( emitted?.parts diff --git a/packages/cli/test/cli/run-stdin.test.ts b/packages/cli/test/cli/run-stdin.test.ts index e19c78da..eeaa7499 100644 --- a/packages/cli/test/cli/run-stdin.test.ts +++ b/packages/cli/test/cli/run-stdin.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "bun:test" +import { describe, expect, spyOn, test } from "bun:test" import { spawn } from "node:child_process" import fs from "node:fs/promises" import os from "node:os" @@ -19,197 +19,276 @@ async function run(input: { deadlineMs?: number }) { const directory = await fs.mkdtemp(path.join(os.tmpdir(), "aictrl-stdin-")) - for (const [name, text] of Object.entries(input.files ?? {})) await fs.writeFile(path.join(directory, name), text) - const ledger = path.join(directory, "network.jsonl") - await fs.writeFile(ledger, "") - const requests: { method: string; path: string; body: Record | undefined }[] = [] - let stream: ReadableStreamDefaultController | undefined - const sessionID = "ses_stdin_fixture" - const server = Bun.serve({ - hostname: "127.0.0.1", - port: 0, - async fetch(request) { - const url = new URL(request.url) - const body = request.method === "POST" ? ((await request.json()) as Record) : undefined - requests.push({ method: request.method, path: url.pathname, body }) - if (url.pathname === "/event") - return new Response( - new ReadableStream({ - start(controller) { - stream = controller - controller.enqueue(new TextEncoder().encode(": ready\n\n")) - }, - }), - { headers: { "content-type": "text/event-stream" } }, - ) - if (url.pathname === "/session") return Response.json({ id: sessionID }) - if (url.pathname === `/session/${sessionID}/message` || url.pathname === `/session/${sessionID}/command`) { - stream?.enqueue( - new TextEncoder().encode( - `data: ${JSON.stringify({ type: "session.status", properties: { sessionID, status: { type: "idle" } } })}\n\n`, - ), - ) - return Response.json({}) - } - return Response.json({ unexpected: url.pathname }, { status: 500 }) + const evidence = process.env.STDIN_TEST_EVIDENCE + const cleanup: (() => void | Promise)[] = [ + async () => { + if (!evidence) await fs.rm(directory, { recursive: true, force: true }) }, - }) - const env = { - PATH: [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter), - SHELL: "/bin/bash", - TZ: "UTC", - HOME: path.join(directory, "home"), - AICTRL_TEST_HOME: path.join(directory, "home"), - AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(directory, "managed"), - XDG_CONFIG_HOME: path.join(directory, "config"), - XDG_DATA_HOME: path.join(directory, "data"), - XDG_STATE_HOME: path.join(directory, "state"), - XDG_CACHE_HOME: path.join(directory, "cache"), - BUN_RUNTIME_TRANSPILER_CACHE_PATH: path.join(directory, "bun-cache"), - GIT_CONFIG_NOSYSTEM: "1", - GIT_CONFIG_GLOBAL: "/dev/null", - AICTRL_DISABLE_PROJECT_CONFIG: "true", - AICTRL_DISABLE_DEFAULT_PLUGINS: "true", - STDIN_FIXTURE_ORIGIN: server.url.origin, - STDIN_FIXTURE_LEDGER: ledger, - } - const command = [ - "run", - "--no-install", - "--no-env-file", - "--preload", - guard, - "--conditions=browser", - path.resolve(import.meta.dir, `../../src/${input.entry}.ts`), - ...(input.rootArgs ?? ["run", "--format", "json", "--attach", server.url.origin, ...input.args]), ] - const child = spawn(process.execPath, command, { - cwd: directory, - env, - detached: true, - stdio: ["pipe", "pipe", "pipe"], - }) - let stdout = "" - let stderr = "" - let deadline = false - let kill = false - let signal: NodeJS.Signals | null = null - const started = performance.now() - const exists = () => { + let failed = false + try { + for (const [name, text] of Object.entries(input.files ?? {})) await fs.writeFile(path.join(directory, name), text) + const ledger = path.join(directory, "network.jsonl") + await fs.writeFile(ledger, "") + const requests: { method: string; path: string; body: Record | undefined }[] = [] + let stream: ReadableStreamDefaultController | undefined + const sessionID = "ses_stdin_fixture" + let requestArrivals = 0 + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + requestArrivals++ + const url = new URL(request.url) + const body = request.method === "POST" ? ((await request.json()) as Record) : undefined + requests.push({ method: request.method, path: url.pathname, body }) + if (url.pathname === "/event") + return new Response( + new ReadableStream({ + start(controller) { + stream = controller + controller.enqueue(new TextEncoder().encode(": ready\n\n")) + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ) + if (url.pathname === "/session") return Response.json({ id: sessionID }) + if (url.pathname === `/session/${sessionID}/message` || url.pathname === `/session/${sessionID}/command`) { + stream?.enqueue( + new TextEncoder().encode( + `data: ${JSON.stringify({ type: "session.status", properties: { sessionID, status: { type: "idle" } } })}\n\n`, + ), + ) + return Response.json({}) + } + return Response.json({ unexpected: url.pathname }, { status: 500 }) + }, + }) + cleanup.push(() => server.stop(true)) + const env = { + PATH: [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter), + SHELL: "/bin/bash", + TZ: "UTC", + HOME: path.join(directory, "home"), + AICTRL_TEST_HOME: path.join(directory, "home"), + AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(directory, "managed"), + XDG_CONFIG_HOME: path.join(directory, "config"), + XDG_DATA_HOME: path.join(directory, "data"), + XDG_STATE_HOME: path.join(directory, "state"), + XDG_CACHE_HOME: path.join(directory, "cache"), + BUN_RUNTIME_TRANSPILER_CACHE_PATH: path.join(directory, "bun-cache"), + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + AICTRL_DISABLE_PROJECT_CONFIG: "true", + AICTRL_DISABLE_DEFAULT_PLUGINS: "true", + STDIN_FIXTURE_ORIGIN: server.url.origin, + STDIN_FIXTURE_LEDGER: ledger, + } + const command = [ + "run", + "--no-install", + "--no-env-file", + "--preload", + guard, + "--conditions=browser", + path.resolve(import.meta.dir, `../../src/${input.entry}.ts`), + ...(input.rootArgs ?? ["run", "--format", "json", "--attach", server.url.origin, ...input.args]), + ] + const child = spawn(process.execPath, command, { + cwd: directory, + env, + detached: true, + stdio: ["pipe", "pipe", "pipe"], + }) + cleanup.push(async () => { + if (!child.pid) return + try { + process.kill(-child.pid, "SIGKILL") + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return + throw error + } + await delay(150) + try { + process.kill(-child.pid, 0) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return + throw error + } + throw new Error("Fixture process group remains after final cleanup") + }) + let stdout = "" + let stderr = "" + let deadline = false + let kill = false + let signal: NodeJS.Signals | null = null + const started = performance.now() + const exists = () => { + try { + process.kill(-child.pid!, 0) + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false + throw error + } + } + const staged: ReturnType[] = [] + let firstBytesAt: number | undefined + let eofAt: number | undefined + let requestsBeforeEOF: number | undefined + let stagedInput = false + child.stdout.on("data", (data) => { + stdout += data.toString() + if (input.stdin === "late" && !stagedInput && stdout.includes('"type":"invocation_start"')) { + stagedInput = true + staged.push( + setTimeout(() => { + firstBytesAt = performance.now() - started + child.stdin.write(input.payload ?? "") + staged.push( + setTimeout(() => { + eofAt = performance.now() - started + requestsBeforeEOF = requestArrivals + child.stdin.end() + }, 150), + ) + }, 150), + ) + } + }) + child.stderr.on("data", (data) => { + stderr += data.toString() + }) + child.stdin.on("error", (error: NodeJS.ErrnoException) => { + if (error.code !== "EPIPE") stderr += String(error) + }) + if (input.payload && input.stdin !== "late") child.stdin.write(input.payload) + if (input.stdin === "closed") child.stdin.end() + const timer = setTimeout(() => { + deadline = true + if (exists()) process.kill(-child.pid!, "SIGTERM") + }, input.deadlineMs ?? 3000) + const escalation = setTimeout( + () => { + if (deadline && exists()) { + kill = true + process.kill(-child.pid!, "SIGKILL") + } + }, + (input.deadlineMs ?? 3000) + 200, + ) + cleanup.push(() => { + for (const stage of staged) clearTimeout(stage) + clearTimeout(timer) + clearTimeout(escalation) + child.stdin.destroy() + }) + let code: number | null = null try { - process.kill(-child.pid!, 0) - return true + code = await new Promise((resolve, reject) => { + child.once("error", reject) + child.once("close", (status, reason) => { + signal = reason + resolve(status) + }) + }) } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ESRCH") return false + failed = true throw error + } finally { + try { + for (const timer of staged) clearTimeout(timer) + clearTimeout(timer) + clearTimeout(escalation) + child.stdin.destroy() + if (exists()) { + process.kill(-child.pid!, "SIGTERM") + await delay(150) + if (exists()) { + kill = true + process.kill(-child.pid!, "SIGKILL") + } + } + } catch (error) { + console.error("Fixture child cleanup failed", { directory, error }) + if (!failed) throw error + } } - } - const staged: ReturnType[] = [] - let firstBytesAt: number | undefined - let eofAt: number | undefined - let requestsBeforeEOF: number | undefined - let stagedInput = false - child.stdout.on("data", (data) => { - stdout += data.toString() - if (input.stdin === "late" && !stagedInput && stdout.includes('"type":"invocation_start"')) { - stagedInput = true - staged.push( - setTimeout(() => { - firstBytesAt = performance.now() - started - child.stdin.write(input.payload ?? "") - staged.push( - setTimeout(() => { - eofAt = performance.now() - started - requestsBeforeEOF = requests.length - child.stdin.end() - }, 150), - ) - }, 150), + const groupGone = !exists() + // Partial fetch/install observations only; coverage is documented in fixture/README.md. + const network = (await fs.readFile(ledger, "utf8")) + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)) + const result = { + ...input, + directory, + command, + code, + signal, + deadline, + kill, + groupGone, + milliseconds: performance.now() - started, + firstBytesAt, + eofAt, + requestsBeforeEOF, + requestArrivals, + stdout, + stderr, + requests, + network, + } + if (evidence) { + await fs.mkdir(evidence, { recursive: true }) + await fs.writeFile( + path.join(evidence, `${input.entry}-${input.name}.json`), + JSON.stringify(result, null, 2) + "\n", ) } - }) - child.stderr.on("data", (data) => { - stderr += data.toString() - }) - child.stdin.on("error", (error: NodeJS.ErrnoException) => { - if (error.code !== "EPIPE") stderr += String(error) - }) - if (input.payload && input.stdin !== "late") child.stdin.write(input.payload) - if (input.stdin === "closed") child.stdin.end() - const timer = setTimeout(() => { - deadline = true - if (exists()) process.kill(-child.pid!, "SIGTERM") - }, input.deadlineMs ?? 3000) - const escalation = setTimeout( - () => { - if (deadline && exists()) { - kill = true - process.kill(-child.pid!, "SIGKILL") - } - }, - (input.deadlineMs ?? 3000) + 200, - ) - let code: number | null = null - try { - code = await new Promise((resolve, reject) => { - child.once("error", reject) - child.once("close", (status, reason) => { - signal = reason - resolve(status) - }) - }) + if (deadline) throw new Error(`CLI deadline fired (never a successful rejection): ${JSON.stringify(result)}`) + if (!groupGone) throw new Error("Stdin test child process group remains") + if (network.some((row) => row.allowed === false)) + throw new Error("Stdin test attempted forbidden network: " + JSON.stringify(network)) + return result + } catch (error) { + failed = true + throw error } finally { - for (const timer of staged) clearTimeout(timer) - clearTimeout(timer) - clearTimeout(escalation) - child.stdin.destroy() - if (exists()) { - process.kill(-child.pid!, "SIGTERM") - await delay(150) - if (exists()) { - kill = true - process.kill(-child.pid!, "SIGKILL") + const errors: unknown[] = [] + for (const dispose of cleanup.reverse()) { + try { + await dispose() + } catch (error) { + errors.push(error) } } - server.stop(true) - } - const groupGone = !exists() - const network = (await fs.readFile(ledger, "utf8")) - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line)) - const result = { - ...input, - directory, - command, - code, - signal, - deadline, - kill, - groupGone, - milliseconds: performance.now() - started, - firstBytesAt, - eofAt, - requestsBeforeEOF, - stdout, - stderr, - requests, - network, - } - if (process.env.STDIN_TEST_EVIDENCE) { - await fs.mkdir(process.env.STDIN_TEST_EVIDENCE, { recursive: true }) - await fs.writeFile( - path.join(process.env.STDIN_TEST_EVIDENCE, `${input.entry}-${input.name}.json`), - JSON.stringify(result, null, 2) + "\n", - ) + if (errors.length) { + console.error("Fixture cleanup failed", { directory, errors }) + if (!failed) throw new AggregateError(errors, "Fixture cleanup failed") + } } - if (deadline) throw new Error(`CLI deadline fired (never a successful rejection): ${JSON.stringify(result)}`) - if (!groupGone) throw new Error("Stdin test child process group remains") - if (network.some((row) => row.allowed === false)) - throw new Error("Stdin test attempted forbidden network: " + JSON.stringify(network)) - return result } +test("stdin fixture setup failure removes its owned directory before listener creation", async () => { + const primary = new Error("stdin fixture setup write failed") + const listener = spyOn(Bun, "serve") + let directory: string | undefined + const write = spyOn(fs, "writeFile").mockImplementationOnce(async (file) => { + directory = path.dirname(String(file)) + throw primary + }) + try { + await expect(run({ entry: "index", name: "setup-failure", args: ["inert"], stdin: "closed" })).rejects.toBe(primary) + expect(listener).not.toHaveBeenCalled() + expect(directory).toBeDefined() + if (!process.env.STDIN_TEST_EVIDENCE) await expect(fs.stat(directory!)).rejects.toMatchObject({ code: "ENOENT" }) + } finally { + write.mockRestore() + listener.mockRestore() + } +}) + function message(result: Awaited>) { const body = result.requests.find((request) => request.path.endsWith("/message"))?.body return (body?.parts as { type: string; text?: string }[])?.find((part) => part.type === "text")?.text @@ -253,6 +332,8 @@ describe("explicit stdin contract (#210)", () => { payload: "context λ\n", }) expect(result.code, result.stderr + result.stdout).toBe(0) + if (!process.env.STDIN_TEST_EVIDENCE) + await expect(fs.stat(result.directory)).rejects.toMatchObject({ code: "ENOENT" }) // Structured operands preserve raw decoded prompt text; quotes are not added. expect(message(result)).toBe("summarize this\ncontext λ\n") expect(title(result)).toBe("summarize this") diff --git a/packages/cli/test/cli/run-tool-selection.test.ts b/packages/cli/test/cli/run-tool-selection.test.ts index 4889619c..39da8682 100644 --- a/packages/cli/test/cli/run-tool-selection.test.ts +++ b/packages/cli/test/cli/run-tool-selection.test.ts @@ -146,126 +146,242 @@ async function run(input: Input) { ) }, }) - const config = { - enabled_providers: [input.codex ? "openai" : "fixture"], - provider: { - [input.codex ? "openai" : "fixture"]: { - npm: input.codex ? "@ai-sdk/openai" : "@ai-sdk/openai-compatible", - options: { - apiKey: "fixture", - baseURL: `${server.url.origin}/v1`, - ...(input.proxy ? { litellmProxy: true } : {}), + const cleanup: (() => unknown | Promise)[] = [() => server.stop(true)] + let primaryFailure: { error: unknown } | undefined + try { + const config = { + enabled_providers: [input.codex ? "openai" : "fixture"], + provider: { + [input.codex ? "openai" : "fixture"]: { + npm: input.codex ? "@ai-sdk/openai" : "@ai-sdk/openai-compatible", + options: { + apiKey: "fixture", + baseURL: `${server.url.origin}/v1`, + ...(input.proxy ? { litellmProxy: true } : {}), + }, + ...(input.codex + ? {} + : { + models: Object.fromEntries( + ["fixture", "gpt-5"].map((id) => [ + id, + { name: id, tool_call: true, limit: { context: 200000, output: 1000 } }, + ]), + ), + }), }, - ...(input.codex - ? {} - : { - models: Object.fromEntries( - ["fixture", "gpt-5"].map((id) => [ - id, - { name: id, tool_call: true, limit: { context: 200000, output: 1000 } }, - ]), - ), - }), }, - }, - agent: { title: { disable: true } }, - permission: { "*": "allow" }, - ...input.config, - ...(extra ?? {}), - } - const home = path.join(tmp.path, "home") - const preload = path.join(tmp.path, "network-guard.ts") - const ledger = path.join(tmp.path, "network.jsonl") - await fs.mkdir(home) - await Bun.write(preload, guard) - await Bun.write(ledger, "") - const codexLog = path.join(tmp.path, "codex-captures.jsonl") - await Bun.write(codexLog, "") - if (input.codex) - await Bun.write( - path.join(tmp.path, "data/aictrl/auth.json"), - JSON.stringify({ - openai: { type: "oauth", access: "fixture-access", refresh: "fixture-refresh", expires: Date.now() + 60000 }, - }), - ) - const selected = input.entry ?? entry - const program = path.join(tmp.path, "record-only.ts") - if (input.noReply) - await Bun.write( - program, - [ - "const root = process.env.SELECTOR_RUNTIME_ROOT", - 'const [{ Instance }, { Session }, { SessionPrompt }] = await Promise.all([import(root + "/project/instance.ts"), import(root + "/session/index.ts"), import(root + "/session/prompt.ts")])', - "try { await Instance.provide({ directory: process.cwd(), fn: async () => {", - ' const session = await Session.create({ title: "Record unavailable model" })', - ' const message = await SessionPrompt.prompt({ sessionID: session.id, noReply: true, model: { providerID: "fixture", modelID: "unavailable-model" }, parts: [{ type: "text", text: "Record only; do not run a model." }] })', - " console.log(JSON.stringify({ role: message.info.role, model: message.info.model, saved: (await Session.messages({ sessionID: session.id })).length }))", - "} }) } catch (error) { console.error(String(error)); process.exitCode = 1 }", - "await Instance.disposeAll()", - ].join("\n"), - ) - const command = [ - process.execPath, - "run", - "--preload", - preload, - "--conditions=browser", - ...(input.runtime - ? [path.resolve(import.meta.dir, "../fixture/tool-selection-runtime.ts")] - : input.noReply - ? [program] - : [ - selected, - "run", - "--format", - "json", - "--model", - input.codex ? "openai/gpt-5.2" : "fixture/" + (input.model ?? "fixture"), - ...(input.args ?? []), - ...(input.noMessage ? [] : ["Say fixture complete without tools."]), - ]), - ] - const process_ = Bun.spawn(command, { - cwd: tmp.path, - env: { - PATH: fixturePath, - SHELL: "/bin/bash", - HOME: home, - AICTRL_TEST_HOME: home, - AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(tmp.path, "managed"), - XDG_DATA_HOME: path.join(tmp.path, "data"), - XDG_CONFIG_HOME: path.join(tmp.path, "config"), - XDG_STATE_HOME: path.join(tmp.path, "state"), - XDG_CACHE_HOME: path.join(tmp.path, "cache"), - AICTRL_CONFIG_CONTENT: JSON.stringify(config), - AICTRL_DISABLE_DEFAULT_PLUGINS: "true", - AICTRL_DISABLE_AUTOCOMPACT: "true", - AICTRL_MODEL_STREAM_IDLE_TIMEOUT_MS: "5000", - SELECTOR_FIXTURE_ORIGIN: server.url.origin, - SELECTOR_FIXTURE_GUARD_LOG: ledger, - SELECTOR_FIXTURE_ZOD: import.meta.resolve("zod"), - SELECTOR_RUNTIME_ROOT: path.dirname(selected), - SELECTOR_RUNTIME_MODE: input.runtime ?? "", - SELECTOR_RUNTIME_PRELOAD: preload, - SELECTOR_FIXTURE_CODEX: String(!!input.codex), - SELECTOR_FIXTURE_CODEX_LOG: codexLog, - }, - stdin: "ignore", - stdout: "pipe", - stderr: "pipe", - }) - const deadlineMs = input.deadlineMs ?? 15000 - let deadlineFired = false - const timeout = setTimeout(() => { - deadlineFired = true - process_.kill("SIGKILL") - }, deadlineMs) - try { - const [stdout, stderr, exit] = await Promise.all([ - new Response(process_.stdout).text(), - new Response(process_.stderr).text(), - process_.exited, - ]) + agent: { title: { disable: true } }, + permission: { "*": "allow" }, + ...input.config, + ...(extra ?? {}), + } + const home = path.join(tmp.path, "home") + const preload = path.join(tmp.path, "network-guard.ts") + const ledger = path.join(tmp.path, "network.jsonl") + await fs.mkdir(home) + await Bun.write(preload, guard) + await Bun.write(ledger, "") + const codexLog = path.join(tmp.path, "codex-captures.jsonl") + await Bun.write(codexLog, "") + if (input.codex) + await Bun.write( + path.join(tmp.path, "data/aictrl/auth.json"), + JSON.stringify({ + openai: { type: "oauth", access: "fixture-access", refresh: "fixture-refresh", expires: Date.now() + 60000 }, + }), + ) + const selected = input.entry ?? entry + const program = path.join(tmp.path, "record-only.ts") + if (input.noReply) + await Bun.write( + program, + [ + "const root = process.env.SELECTOR_RUNTIME_ROOT", + 'const [{ Instance }, { Session }, { SessionPrompt }] = await Promise.all([import(root + "/project/instance.ts"), import(root + "/session/index.ts"), import(root + "/session/prompt.ts")])', + "try { await Instance.provide({ directory: process.cwd(), fn: async () => {", + ' const session = await Session.create({ title: "Record unavailable model" })', + ' const message = await SessionPrompt.prompt({ sessionID: session.id, noReply: true, model: { providerID: "fixture", modelID: "unavailable-model" }, parts: [{ type: "text", text: "Record only; do not run a model." }] })', + " console.log(JSON.stringify({ role: message.info.role, model: message.info.model, saved: (await Session.messages({ sessionID: session.id })).length }))", + "} }) } catch (error) { console.error(String(error)); process.exitCode = 1 }", + "await Instance.disposeAll()", + ].join("\n"), + ) + const command = [ + process.execPath, + "run", + "--preload", + preload, + "--conditions=browser", + ...(input.runtime + ? [path.resolve(import.meta.dir, "../fixture/tool-selection-runtime.ts")] + : input.noReply + ? [program] + : [ + selected, + "run", + "--format", + "json", + "--model", + input.codex ? "openai/gpt-5.2" : "fixture/" + (input.model ?? "fixture"), + ...(input.args ?? []), + ...(input.noMessage ? [] : ["Say fixture complete without tools."]), + ]), + ] + const process_ = Bun.spawn(command, { + cwd: tmp.path, + env: { + PATH: fixturePath, + SHELL: "/bin/bash", + HOME: home, + AICTRL_TEST_HOME: home, + AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(tmp.path, "managed"), + XDG_DATA_HOME: path.join(tmp.path, "data"), + XDG_CONFIG_HOME: path.join(tmp.path, "config"), + XDG_STATE_HOME: path.join(tmp.path, "state"), + XDG_CACHE_HOME: path.join(tmp.path, "cache"), + AICTRL_CONFIG_CONTENT: JSON.stringify(config), + AICTRL_DISABLE_DEFAULT_PLUGINS: "true", + AICTRL_DISABLE_AUTOCOMPACT: "true", + AICTRL_MODEL_STREAM_IDLE_TIMEOUT_MS: "5000", + SELECTOR_FIXTURE_ORIGIN: server.url.origin, + SELECTOR_FIXTURE_GUARD_LOG: ledger, + SELECTOR_FIXTURE_ZOD: import.meta.resolve("zod"), + SELECTOR_RUNTIME_ROOT: path.dirname(selected), + SELECTOR_RUNTIME_MODE: input.runtime ?? "", + SELECTOR_RUNTIME_PRELOAD: preload, + SELECTOR_FIXTURE_CODEX: String(!!input.codex), + SELECTOR_FIXTURE_CODEX_LOG: codexLog, + }, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + detached: true, + }) + const deadlineMs = input.deadlineMs ?? 15000 + let deadlineFired = false + let timeout: ReturnType | undefined + const output = { stdout: "", stderr: "" } + const readers: ReadableStreamDefaultReader[] = [] + async function read(reader: (typeof readers)[number], name: keyof typeof output) { + const decoder = new TextDecoder() + while (true) { + const chunk = await reader.read() + if (chunk.done) break + output[name] += decoder.decode(chunk.value, { stream: true }) + } + output[name] += decoder.decode() + return output[name] + } + function signalGroup(signal: NodeJS.Signals | 0) { + try { + process.kill(-process_.pid, signal) + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false + throw error + } + } + async function waitGroupGone(milliseconds: number) { + const until = Date.now() + milliseconds + while (signalGroup(0)) { + if (Date.now() >= until) return false + await Bun.sleep(10) + } + return true + } + let stopping: Promise | undefined + function stopGroup() { + return (stopping ??= (async () => { + const errors: unknown[] = [] + let gone = false + try { + gone = !signalGroup("SIGTERM") || (await waitGroupGone(200)) + } catch (error) { + errors.push(error) + } + if (!gone) { + try { + signalGroup("SIGKILL") + } catch (error) { + errors.push(error) + } + try { + if (!(await waitGroupGone(2000))) throw new Error("Owned selector process group remains after SIGKILL") + } catch (error) { + errors.push(error) + } + } + if (errors.length) throw new AggregateError(errors, "Owned selector group termination failed") + })()) + } + async function bounded(task: Promise, milliseconds: number, message: string): Promise { + let timer: ReturnType | undefined + try { + return await Promise.race([ + task, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(message)), milliseconds) + }), + ]) + } finally { + clearTimeout(timer) + } + } + const drains: Promise[] = [] + cleanup.push(async () => { + clearTimeout(timeout) + const errors: unknown[] = [] + for (const operation of [ + () => stopGroup(), + () => bounded(process_.exited, 1000, "Owned selector child was not reaped after termination"), + ...readers.map((reader) => () => bounded(reader.cancel(), 500, "Owned selector pipe cancellation timed out")), + () => bounded(Promise.allSettled(drains), 500, "Owned selector output did not settle after cancellation"), + ]) { + try { + await operation() + } catch (error) { + errors.push(error) + } + } + for (const reader of readers) { + try { + reader.releaseLock() + } catch (error) { + errors.push(error) + } + } + if (errors.length) throw new AggregateError(errors, "Owned selector process cleanup failed") + }) + readers.push(process_.stdout.getReader()) + readers.push(process_.stderr.getReader()) + drains.push(read(readers[0], "stdout"), read(readers[1], "stderr")) + const exited = process_.exited.then(async (exit) => { + if (signalGroup(0)) { + try { + await stopGroup() + } catch (error) { + if (!deadlineFired) throw error + } + if (!deadlineFired) throw new Error("Owned selector descendants outlived the fixture child") + } + return exit + }) + const deadlineFailure = () => + new Error( + `${input.label}: child deadline exceeded after ${deadlineMs}ms; providerRequests=${requests.length}; stdout=${output.stdout.slice(-2000)}; stderr=${output.stderr.slice(-2000)}`, + ) + const deadline = new Promise((_, reject) => { + timeout = setTimeout(() => { + deadlineFired = true + // The deadline also interrupts collection when descendants retain the pipes. + void stopGroup().then( + () => reject(deadlineFailure()), + () => reject(deadlineFailure()), + ) + }, deadlineMs) + }) + const [stdout, stderr, exit] = await Promise.race([Promise.all([drains[0], drains[1], exited]), deadline]) clearTimeout(timeout) const network = (await Bun.file(ledger).text()) .split("\n") @@ -315,10 +431,27 @@ async function run(input: Input) { ) expect(network.filter((request) => !request.allowed)).toEqual([]) return result + } catch (error) { + primaryFailure = { error } + throw error } finally { - clearTimeout(timeout) - process_.kill("SIGKILL") - server.stop(true) + const errors: unknown[] = [] + for (const dispose of cleanup.reverse()) { + try { + await dispose() + } catch (error) { + errors.push(error) + } + } + if (errors.length) { + throw new AggregateError( + [...(primaryFailure ? [primaryFailure.error] : []), ...errors], + primaryFailure + ? `${input.label}: ${String(primaryFailure.error)}; fixture cleanup also failed` + : `${input.label}: fixture cleanup failed`, + primaryFailure ? { cause: primaryFailure.error } : undefined, + ) + } } } @@ -1660,7 +1793,16 @@ export default {description:"Actual permitted runtime failure",args:{},execute:a }) test("forced CLI fixture deadline cannot pass as an expected negative exit", async () => { - await expect(run({ label: "forced-cli-deadline", runtime: "deadline-hang", deadlineMs: 20 })).rejects.toThrow( - /child deadline exceeded after 20ms; providerRequests=0/, + const failure: unknown = await run({ label: "forced-cli-deadline", runtime: "deadline-hang", deadlineMs: 20 }).catch( + (error: unknown) => error, + ) + expect(failure).toBeInstanceOf(Error) + expect(failure).not.toBeInstanceOf(AggregateError) + expect(failure).toHaveProperty( + "message", + expect.stringMatching(/^forced-cli-deadline: child deadline exceeded after 20ms; providerRequests=0;/), ) + expect(failure).not.toHaveProperty("message", expect.stringContaining("fixture cleanup")) + expect(failure).not.toHaveProperty("cause") + expect(failure).not.toHaveProperty("errors") }, 20000) diff --git a/packages/cli/test/session/tool-guidance.test.ts b/packages/cli/test/session/tool-guidance.test.ts index 1c771d32..ea284b45 100644 --- a/packages/cli/test/session/tool-guidance.test.ts +++ b/packages/cli/test/session/tool-guidance.test.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, describe, expect, spyOn, test } from "bun:test" +import { afterAll, beforeAll, describe, expect, spyOn, test, type Mock } from "bun:test" import path from "node:path" // Install before runtime imports: every model request is local or an intercepted @@ -149,66 +149,83 @@ async function fixture( } = {}, fn: (context: Fixture) => Promise, ) { - transport.captures = [] - transport.codex = options.codex ?? false const providerID = options.gpt || options.codex ? "openai" : "fixture" const modelID = providerID === "openai" ? "gpt-5.2" : "fixture" - const server = Bun.serve({ - hostname: "127.0.0.1", - port: 0, - async fetch(request) { - transport.captures.push((await request.json()) as Wire) - if (options.call && transport.captures.length === 1) { - const call = options.call() - return sse([ - { - id: "fixture-call", - object: "chat.completion.chunk", - choices: [ - { - index: 0, - delta: { - role: "assistant", - tool_calls: [ - { - index: 0, - id: "runtime-call", - type: "function", - function: { name: call.name, arguments: JSON.stringify(call.arguments) }, - }, - ], - }, - }, - ], - }, - { - id: "fixture-call", - object: "chat.completion.chunk", - choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], - }, - ]) - } - return providerID === "openai" ? responses() : chat() - }, - }) - transport.origin = server.url.origin - const previousAuth = options.codex ? await Auth.get("openai") : undefined - if (options.codex) - await Auth.set("openai", { - type: "oauth", - access: "fixture-access", - refresh: "fixture-refresh", - expires: Date.now() + 60_000, - }) + let server: Bun.Server | undefined + let previousAuth: { value: Awaited> } | undefined + let authSwapAttempted = false const original = LLM.stream let last: StreamInput | undefined let persisted: unknown const persistedCheckpoints: unknown[] = [] - const spy = spyOn(LLM, "stream").mockImplementation(async (input) => { - last = input - return original(input) - }) + let spy: Mock | undefined + let primaryFailure: { error: unknown } | undefined + const cleanupErrors: Error[] = [] + const diagnosticErrors: Error[] = [] + async function attempt(errors: Error[], operation: string, fn: () => unknown | Promise) { + try { + await fn() + } catch (cause) { + errors.push(new Error(operation, { cause })) + } + } try { + transport.captures = [] + transport.codex = options.codex ?? false + server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + transport.captures.push((await request.json()) as Wire) + if (options.call && transport.captures.length === 1) { + const call = options.call() + return sse([ + { + id: "fixture-call", + object: "chat.completion.chunk", + choices: [ + { + index: 0, + delta: { + role: "assistant", + tool_calls: [ + { + index: 0, + id: "runtime-call", + type: "function", + function: { name: call.name, arguments: JSON.stringify(call.arguments) }, + }, + ], + }, + }, + ], + }, + { + id: "fixture-call", + object: "chat.completion.chunk", + choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], + }, + ]) + } + return providerID === "openai" ? responses() : chat() + }, + }) + transport.origin = server.url.origin + if (options.codex) { + previousAuth = { value: await Auth.get("openai") } + authSwapAttempted = true + await Auth.set("openai", { + type: "oauth", + access: "fixture-access", + refresh: "fixture-refresh", + expires: Date.now() + 60_000, + }) + } + spy = spyOn(LLM, "stream") + spy.mockImplementation(async (input) => { + last = input + return original(input) + }) await using tmp = await tmpdir({ init: options.init, config: { @@ -305,37 +322,62 @@ async function fixture( try { await fn(context) } finally { - persisted = await context.messages() + await attempt(diagnosticErrors, "Failed to capture persisted fixture messages", async () => { + persisted = await context.messages() + }) } }, }) + } catch (error) { + primaryFailure = { error } } finally { + const acquiredSpy = spy + if (acquiredSpy) await attempt(cleanupErrors, "Failed to restore LLM.stream", () => acquiredSpy.mockRestore()) + const acquiredServer = server + if (acquiredServer) await attempt(cleanupErrors, "Failed to stop fixture server", () => acquiredServer.stop(true)) + await attempt(cleanupErrors, "Failed to reset fixture transport", () => { + transport.origin = "" + transport.codex = false + }) + if (authSwapAttempted && previousAuth) { + const snapshot = previousAuth.value + await attempt(cleanupErrors, "Failed to restore fixture auth", async () => { + if (snapshot) await Auth.set("openai", snapshot) + else await Auth.remove("openai") + }) + } const output = process.env.GUIDANCE_CAPTURE_DIR if (output) - await Bun.write( - path.join(output, `${label}.json`), - JSON.stringify( - { - label, - fixtureOptions: options, - codex: options.codex ?? false, - requests: transport.captures, - persisted, - persistedCheckpoints, - }, - null, - 2, + await attempt(diagnosticErrors, "Failed to write fixture capture", () => + Bun.write( + path.join(output, `${label}.json`), + JSON.stringify( + { + label, + fixtureOptions: options, + codex: options.codex ?? false, + requests: transport.captures, + persisted, + persistedCheckpoints, + }, + null, + 2, + ), ), ) - spy.mockRestore() - server.stop(true) - transport.origin = "" - transport.codex = false - if (options.codex) { - if (previousAuth) await Auth.set("openai", previousAuth) - else await Auth.remove("openai") - } } + if (cleanupErrors.length || diagnosticErrors.length) { + throw new AggregateError( + [ + ...(primaryFailure ? [primaryFailure.error] : []), + ...(cleanupErrors.length ? [new AggregateError(cleanupErrors, "Fixture cleanup failed")] : []), + ...(diagnosticErrors.length ? [new AggregateError(diagnosticErrors, "Fixture diagnostics failed")] : []), + ], + `Tool guidance fixture ${label} failed`, + primaryFailure ? { cause: primaryFailure.error } : undefined, + ) + } + if (primaryFailure) throw primaryFailure.error } beforeAll(() => HeadlessRun.enable()) @@ -344,6 +386,73 @@ afterAll(() => { globalThis.fetch = originalFetch }) +describe("guidance fixture teardown", () => { + for (const callbackFails of [false, true]) { + test(`capture failure restores fixture resources${callbackFails ? " and preserves the callback error" : ""}`, async () => { + await using tmp = await tmpdir() + const output = path.join(tmp.path, "capture-path-conflict") + await Bun.write(output, "owned regular file, not a capture directory") + const previousCapture = process.env.GUIDANCE_CAPTURE_DIR + const previousStream = LLM.stream + const previousAuth = await Auth.get("openai") + const primary = new Error("Owned callback failure") + let origin = "" + let failure: unknown + process.env.GUIDANCE_CAPTURE_DIR = output + try { + await fixture("teardown-capture-conflict", { codex: true }, async () => { + origin = transport.origin + if (callbackFails) throw primary + }) + } catch (error) { + failure = error + } finally { + if (previousCapture === undefined) delete process.env.GUIDANCE_CAPTURE_DIR + else process.env.GUIDANCE_CAPTURE_DIR = previousCapture + } + expect(failure).toBeInstanceOf(AggregateError) + const aggregate = failure as AggregateError + if (callbackFails) { + expect(aggregate.cause).toBe(primary) + expect(aggregate.errors[0]).toBe(primary) + } else expect(aggregate.cause).toBeUndefined() + const diagnostic = aggregate.errors.find( + (error) => error instanceof AggregateError && error.message === "Fixture diagnostics failed", + ) as AggregateError + expect(diagnostic).toBeInstanceOf(AggregateError) + expect(diagnostic.errors.some((error) => error.message === "Failed to write fixture capture")).toBe(true) + expect(LLM.stream).toBe(previousStream) + expect(transport.origin).toBe("") + expect(transport.codex).toBe(false) + expect(await Auth.get("openai")).toEqual(previousAuth) + expect(origin).toStartWith("http://127.0.0.1:") + await expect(originalFetch(origin)).rejects.toThrow() + }) + } + + test("successful capture is written after fixture resources are restored", async () => { + await using tmp = await tmpdir() + const previousCapture = process.env.GUIDANCE_CAPTURE_DIR + const previousStream = LLM.stream + const previousAuth = await Auth.get("openai") + process.env.GUIDANCE_CAPTURE_DIR = tmp.path + try { + await fixture("teardown-capture-success", { codex: true }, async () => {}) + const capture = await Bun.file(path.join(tmp.path, "teardown-capture-success.json")).json() + expect(capture.label).toBe("teardown-capture-success") + expect(capture.codex).toBe(true) + expect(capture.requests).toEqual([]) + expect(LLM.stream).toBe(previousStream) + expect(transport.origin).toBe("") + expect(transport.codex).toBe(false) + expect(await Auth.get("openai")).toEqual(previousAuth) + } finally { + if (previousCapture === undefined) delete process.env.GUIDANCE_CAPTURE_DIR + else process.env.GUIDANCE_CAPTURE_DIR = previousCapture + } + }) +}) + describe("actual request tool guidance (#244)", () => { test("full built-in catalog retains useful peer guidance", async () => { await fixture("full", {}, async (context) => { diff --git a/packages/plugin/src/tool.ts b/packages/plugin/src/tool.ts index 9a9eb786..dc150e4c 100644 --- a/packages/plugin/src/tool.ts +++ b/packages/plugin/src/tool.ts @@ -1,3 +1,4 @@ +import type { CommandInput } from "@aictrl/sdk/v2" import { z } from "zod" export type ToolContext = { @@ -21,7 +22,15 @@ export type ToolContext = { readonly names: readonly string[] readonly restricted: boolean assert(name: string): void - execute(name: string, args: unknown): Promise<{ title: string; output: string; metadata: Record }> + execute( + name: string, + args: unknown, + ): Promise<{ + title: string + output: string + metadata: Record + attachments?: Omit[number], { type: "file" }>, "id">[] + }> } abort: AbortSignal metadata(input: { title?: string; metadata?: { [key: string]: any } }): void diff --git a/packages/sdk/script/build.ts b/packages/sdk/script/build.ts index 29b6c447..258b14a3 100755 --- a/packages/sdk/script/build.ts +++ b/packages/sdk/script/build.ts @@ -95,6 +95,6 @@ await legacy.write(source) await $`bun prettier --write src/gen` await $`bun prettier --write src/v2` await $`rm -rf dist` -// dist was removed deliberately; a surviving composite cache must not skip emit. +// Dist was removed above; force emission even when the composite build cache is current. await $`bun tsc --build --force` await $`rm openapi.json` diff --git a/packages/sdk/src/v2/index.ts b/packages/sdk/src/v2/index.ts index 1dde1664..443411d3 100644 --- a/packages/sdk/src/v2/index.ts +++ b/packages/sdk/src/v2/index.ts @@ -2,4 +2,4 @@ export { createAictrlClient } from "../client.js" export * from "./manual-types.js" export * from "./gen/client/index.js" export * from "./gen/client/types.gen.js" -export type { StepFinishPart, ProviderTermination } from "./gen/types.gen.js" +export type { CommandArguments, CommandInput, StepFinishPart, ProviderTermination } from "./gen/types.gen.js" diff --git a/script/token-budget/README.md b/script/token-budget/README.md index 88db0714..d5a7c934 100644 --- a/script/token-budget/README.md +++ b/script/token-budget/README.md @@ -16,9 +16,13 @@ python3.12 -I -B script/token-budget/setup.py --directory /tmp/my-token-environm The guard uses an exclusive nonblocking lock in the owner-only `/tmp/aictrl-token-budget` directory. Concurrent invocations fail before changing the fixed workspace. Each cold child runs in the literal `/tmp/aictrl-token-budget/workspace`, newly initialized as a Git repository, with UTC and Date fixed before production imports to 21 October 2026. This controls all three raw cwd occurrences, including the two in Bash descriptions/schema. No text normalization is applied. Existing unowned, shared, or symlink fixture directories are rejected. The directory/lock remain for safe reuse; raw captures remain in the requested evidence directory. -Only loopback fixture HTTP is allowed by the before-import fetch/preconnect/install guard reused from the selector fixture. Child environments are explicitly allowlisted, without provider credentials or inherited project configuration. The real CLI has a mandatory 15-second deadline; the outer owned process group has 25 seconds and rejects surviving descendants even if its parent exits successfully. These fixture controls are not an OS network sandbox. +The before-import guard observes `globalThis.fetch` calls in the CLI process and rejects origins other than the loopback fixture. It rejects `fetch.preconnect` and direct `Bun.spawn` calls whose executable is `bun` and whose next argument is exactly `add` or `install`. This ledger is partial: other network APIs, `Bun.spawnSync`, Node child-process APIs, shell-launched programs, and package-manager aliases are outside those controls. A clean ledger establishes no rejected operation on the instrumented paths; it does not establish complete network or installation accounting. Child environments are explicitly allowlisted, without provider credentials or inherited project configuration. These trusted fixture controls are not an OS network sandbox. -`count.py` verifies the pinned tokenizer version, exact rank SHA256, raw environment, catalog, single request and clean process/network evidence. `reference.py` denies socket/DNS use during accounting. The enforced metric is `o200k_base` encoding of compact sorted Unicode JSON containing actual SDK system-role messages, serialized tools, and instructions when present. The fixture user text is excluded. The full first model input is separately reported using `instructions/messages/input/tools/response_format/text`; transport controls are excluded. System/tool diagnostic counts use separate tokenizations and are not additive BPE contributions. +The real CLI has a mandatory 15-second deadline; each outer capture has 25 seconds. A fresh forked Linux subreaper owns only that invocation's capture and adopted descendants, including descendants that create a new session. After the capture parent exits, already-dead children are reaped without being classified as live survivors. Live adopted children are terminated and reaped, and the capture is rejected even if its parent exited successfully. Cleanup repeatedly handles newly adopted children and fails if it cannot verify completion within its cleanup deadline. The caller's other children are outside this ownership scope. The result records the supervisor and descendant cleanup evidence; the legacy `remaining_group_killed` field now indicates any observed live adopted descendant, not just the original process group. + +Evidence output must be a new directory under an existing parent. Its root and per-run directories are owner-only (`0700`), so raw fixture requests and allowlisted environment records are not exposed by an ambient permissive umask. Existing output directories are never overwritten. The fixed workspace directory uses the same ownership/private-directory validation in production and lock controls. + +`count.py` verifies the pinned tokenizer version, exact rank SHA256, raw environment, catalog, single request and the recorded process/partial-fetch-ledger evidence. `reference.py` denies socket/DNS use during accounting. The enforced metric is `o200k_base` encoding of compact sorted Unicode JSON containing actual SDK system-role messages, serialized tools, and instructions when present. The fixture user text is excluded. The full first model input is separately reported using `instructions/messages/input/tools/response_format/text`; transport controls are excluded. System/tool diagnostic counts use separate tokenizations and are not additive BPE contributions. `budgets.json` contains two fixed explicit ceilings. Raising either is a reviewed source edit requiring raw evidence and an explanation. `--observe` records an evaluation without enforcing ceilings and is never used by CI. `--source` selects an isolated source copy for QA source-growth controls; CI always uses the repository entry. The default two cold repetitions must produce identical canonical prefix bytes for each arm. diff --git a/script/token-budget/release-count.py b/script/token-budget/release-count.py index 2cdcd19f..a5059889 100644 --- a/script/token-budget/release-count.py +++ b/script/token-budget/release-count.py @@ -17,7 +17,12 @@ def analyze(campaign_path): raise ValueError('Refuse overwriting accounting results') if campaign['protocolVersion'] != 3 or campaign['failures'] or campaign['REPS'] < 2: raise ValueError('Incomplete or failed release campaign') - final = campaign['cleanupChecks'][-1]['finalCampaignCheck'] + checks = campaign.get('cleanupChecks') + if not isinstance(checks, list) or not checks or not isinstance(checks[-1], dict): + raise ValueError('Incomplete release campaign: final cleanup evidence missing') + final = checks[-1].get('finalCampaignCheck') + if not isinstance(final, dict) or not {'exit', 'timedOut', 'stdout'} <= final.keys(): + raise ValueError('Incomplete release campaign: final cleanup evidence missing') if final['exit'] != 0 or final['timedOut'] or final['stdout'].strip(): raise ValueError('Final campaign container cleanup not verified') encoding = load_encoding() diff --git a/script/token-budget/release-fixtures/token-stub.test.ts b/script/token-budget/release-fixtures/token-stub.test.ts new file mode 100644 index 00000000..a3d04292 --- /dev/null +++ b/script/token-budget/release-fixtures/token-stub.test.ts @@ -0,0 +1,89 @@ +import { expect, test } from "bun:test" +import { MARKER, Stub } from "./token-stub" + +for (const api of ["chat", "responses"] as const) { + const body = (main: boolean) => ({ + model: "fixture", + ...(api === "chat" + ? { + messages: [ + { role: "user", content: main ? `${MARKER} short` : "Generate a title" }, + { role: "tool", tool_call_id: "foreign_history_id", content: "Earlier unrelated result" }, + ], + tools: main + ? [ + { + type: "function", + function: { + name: "read", + parameters: { + type: "object", + properties: { filePath: { type: "string" } }, + required: ["filePath"], + }, + }, + }, + ] + : [], + } + : { + input: [ + { role: "user", content: main ? `${MARKER} short` : "Generate a title" }, + { type: "function_call_output", call_id: "foreign_history_id", output: "Earlier unrelated result" }, + ], + tools: main + ? [ + { + type: "function", + name: "read", + parameters: { type: "object", properties: { filePath: { type: "string" } }, required: ["filePath"] }, + }, + ] + : [], + }), + }) + + test(`${api}: auxiliary foreign history does not advance or reject the main script`, async () => { + const stub = new Stub(0) + stub.reset("short") + try { + const response = await fetch( + new URL(api === "chat" ? "/v1/chat/completions" : "/v1/responses", stub.server.url), + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body(false)), + }, + ) + expect(response.status).toBe(200) + expect(await response.text()).toContain("Bench title") + expect(stub.log).toHaveLength(1) + expect(stub.log[0]).toMatchObject({ kind: "aux", step: 0 }) + expect(stub.log[0].toolResults?.[0]?.callId).toBe("foreign_history_id") + expect(stub.log.some((row) => row.error)).toBe(false) + } finally { + await stub.server.stop(true) + } + }) + + test(`${api}: main foreign history still fails explicit scripted-ID validation`, async () => { + const stub = new Stub(0) + stub.reset("short") + try { + const response = await fetch( + new URL(api === "chat" ? "/v1/chat/completions" : "/v1/responses", stub.server.url), + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body(true)), + }, + ) + expect(response.status).toBe(400) + expect(await response.text()).toContain("Tool results lost all scripted call IDs") + expect(stub.log.some((row) => row.error?.includes("Tool results lost all scripted call IDs"))).toBe(true) + expect(stub.log.some((row) => row.kind === "main")).toBe(false) + } finally { + await stub.server.stop(true) + } + }) +} diff --git a/script/token-budget/release-fixtures/token-stub.ts b/script/token-budget/release-fixtures/token-stub.ts index bb62a9cd..71d1a901 100644 --- a/script/token-budget/release-fixtures/token-stub.ts +++ b/script/token-budget/release-fixtures/token-stub.ts @@ -151,7 +151,7 @@ export class Stub { const tools = toolList(json, api) const main = tools.length > 0 && body.includes(MARKER) const resultEvidence = fixtureEvidence(json, api, this.scenario) - const done = scriptProgress(resultEvidence) + const done = main ? scriptProgress(resultEvidence) : 0 const furthest = Math.max(0, ...this.log.filter((r) => r.kind === "main").map((r) => r.step)) if (main && done < furthest) throw new Error(`Tool history regressed from scripted step ${furthest} to ${done}`) const turn = main ? this.next(done, tools) : { text: "Bench title" } diff --git a/script/token-budget/release.ts b/script/token-budget/release.ts index d2a0606b..c0a0959f 100644 --- a/script/token-budget/release.ts +++ b/script/token-budget/release.ts @@ -75,6 +75,17 @@ let proc: ReturnType | undefined const campaignId = `release248-${process.pid}-${Date.now()}` const failures: unknown[] = [] const cleanupChecks: unknown[] = [] +const errors: unknown[] = [] +const cleanups: { name: typeof ownedContainer; child: typeof proc; promise: Promise }[] = [] +let interruption: { signal: "SIGINT" | "SIGTERM"; error: Error } | undefined +function recordFailure(error: unknown, field: "campaignError" | "cleanupError" = "campaignError") { + if (errors.includes(error)) return + errors.push(error) + failures.push({ [field]: error instanceof Error ? error.message : String(error) }) +} +function checkInterrupted() { + if (interruption) throw interruption.error +} const arms: string[] = ["current-default", "previous-default"] const support: Record = {} const raw: Record> = { short: Object.fromEntries(arms.map((x) => [x, []])) } @@ -128,7 +139,9 @@ function checkpoint() { ) } async function docker(args: string[]) { + checkInterrupted() const child = Bun.spawn(["docker", ...args], { env: hostEnv, stdin: "ignore", stdout: "pipe", stderr: "pipe" }) + proc = child let timeout = false const timer = setTimeout(() => { timeout = true @@ -141,12 +154,16 @@ async function docker(args: string[]) { child.exited, ]) if (code || timeout) { + const error = new Error(`Docker preflight failed (${code}, timeout=${timeout}): ${err}`) + recordFailure(error) await cleanup() - throw new Error(`Docker preflight failed (${code}, timeout=${timeout}): ${err}`) + throw error } + checkInterrupted() return out } finally { clearTimeout(timer) + if (proc === child) proc = undefined } } async function cleanupCommand(args: string[]) { @@ -167,30 +184,70 @@ async function cleanupCommand(args: string[]) { clearTimeout(timer) } } -async function cleanup() { - try { - if (ownedContainer) { - const name = ownedContainer - const removed = await cleanupCommand(["rm", "-f", name]) - const remaining = await cleanupCommand(["ps", "-aq", "--filter", `name=^/${name}$`]) - cleanupChecks.push({ name, removed, remaining }) +function cleanup() { + const name = ownedContainer + const child = proc + const running = cleanups.find((item) => item.name === name && item.child === child) + if (running) return running.promise + const promise = (async () => { + const failed: unknown[] = [] + try { + if (child && child.exitCode === null) child.kill("SIGKILL") + } catch (error) { + failed.push(error) + } + if (name) { + let removed: Awaited> | undefined + let remaining: Awaited> | undefined + try { + removed = await cleanupCommand(["rm", "-f", name]) + } catch (error) { + failed.push(error) + } + try { + remaining = await cleanupCommand(["ps", "-aq", "--filter", `name=^/${name}$`]) + } catch (error) { + failed.push(error) + } + cleanupChecks.push({ name, removed, remaining, errors: failed.map(String) }) if ( + !removed || + !remaining || removed.timedOut || remaining.timedOut || remaining.exit !== 0 || remaining.stdout.trim() || (removed.exit !== 0 && !removed.stderr.includes("No such container")) ) - throw new Error(`Unverified container cleanup: ${name}`) + failed.push(new Error(`Unverified container cleanup: ${name}`)) } - } finally { - if (proc && proc.exitCode === null) proc.kill("SIGKILL") - } -} -for (const signal of ["SIGINT", "SIGTERM"] as const) - process.on(signal, () => { - void cleanup().finally(() => process.exit(128 + (signal === "SIGINT" ? 2 : 15))) + try { + if (child) await child.exited + } catch (error) { + failed.push(error) + } + if (failed.length) { + const error = new AggregateError(failed, failed.map(String).join("; "), { cause: failed[0] }) + recordFailure(error, "cleanupError") + throw error + } + })().finally(() => { + const index = cleanups.findIndex((item) => item.promise === promise) + cleanups.splice(index, 1) }) + cleanups.push({ name, child, promise }) + return promise +} +const handlers = (["SIGINT", "SIGTERM"] as const).map((signal) => { + const handler = () => { + if (interruption) return + interruption = { signal, error: new Error(`Campaign interrupted by ${signal}`) } + recordFailure(interruption.error) + void cleanup().catch((error) => recordFailure(error, "cleanupError")) + } + process.on(signal, handler) + return { signal, handler } +}) try { const image = JSON.parse(await docker(["image", "inspect", input.fixtureImage]))[0] if ( @@ -201,6 +258,7 @@ try { ) throw new Error("Fixture image identity/platform mismatch") for (const name of ["current", "previous"] as const) { + checkInterrupted() const binary = input[name] const script = `const fs=require('fs'),cp=require('child_process'),crypto=require('crypto'); const binary='/release/aictrl'; console.log(JSON.stringify({uid:process.getuid(),home:fs.readdirSync(process.env.HOME),binarySha256:crypto.createHash('sha256').update(fs.readFileSync(binary)).digest('hex'),version:cp.execFileSync(binary,['--version'],{encoding:'utf8'}).trim(),help:cp.execFileSync(binary,['run','--help'],{encoding:'utf8'}),fixtures:Object.fromEntries(['notes.txt','big.txt',...Array.from({length:3},(_,i)=>'notes-'+i+'.txt'),...Array.from({length:50},(_,i)=>'big-'+i+'.txt')].map(name=>{const b=fs.readFileSync('/workspace/'+name);return[name,{bytes:b.length,sha256:crypto.createHash('sha256').update(b).digest('hex')}]}))}));` ownedContainer = `${campaignId}-preflight-${name}` @@ -227,6 +285,7 @@ try { ]), ) await cleanup() + checkInterrupted() ownedContainer = undefined fs.writeFileSync(path.join(output, `${name}-preflight.json`), JSON.stringify(preflight, null, 2) + "\n") if ( @@ -256,10 +315,12 @@ try { raw.short[`${name}-selected`] = [] } } + checkInterrupted() stub = new Stub(50) stub.keepBodies = true for (let rep = 0; rep < input.repeats; rep++) { for (const arm of rep % 2 ? [...arms].reverse() : arms) { + checkInterrupted() stub.reset("short") const runId = `${String(rep + 1).padStart(2, "0")}-${arm}-short` ownedContainer = `${campaignId}-${runId}` @@ -318,12 +379,14 @@ try { ) let timeout = false let cleanupFailure: string | undefined + checkInterrupted() const child = Bun.spawn(argv, { env: hostEnv, stdin: "ignore", stdout: "pipe", stderr: "pipe" }) proc = child const timer = setTimeout(() => { timeout = true + recordFailure(new Error(`Capture ${runId} timed out`)) void cleanup().catch((error) => { - cleanupFailure = String(error) + cleanupFailure ??= String(error) }) }, TIMEOUT) let stdout = "", @@ -340,7 +403,7 @@ try { try { await cleanup() } catch (error) { - cleanupFailure = String(error) + cleanupFailure ??= String(error) } if (!cleanupFailure) ownedContainer = undefined proc = undefined @@ -432,25 +495,44 @@ try { } } } catch (error) { - failures.push({ campaignError: error instanceof Error ? error.message : String(error) }) - checkpoint() - throw error + recordFailure(error) } finally { try { await cleanup() + } catch (error) { + recordFailure(error, "cleanupError") + } + try { const remaining = await cleanupCommand(["ps", "-aq", "--filter", `label=aictrl.harness-benchmark=${campaignId}`]) cleanupChecks.push({ finalCampaignCheck: remaining }) if (remaining.timedOut || remaining.exit !== 0 || remaining.stdout.trim()) throw new Error("Campaign-owned containers remain or cleanup state could not be inspected") - checkpoint() } catch (error) { - failures.push({ cleanupError: String(error) }) - checkpoint() - throw error - } finally { - stub?.server.stop(true) + recordFailure(error, "cleanupError") + } + try { + await stub?.server.stop(true) + } catch (error) { + recordFailure(error, "cleanupError") + } + await Promise.allSettled(cleanups.map((item) => item.promise)) + try { fs.rmSync(hostHome, { recursive: true, force: true }) + } catch (error) { + recordFailure(error, "cleanupError") + } + try { + checkpoint() + } catch (error) { + recordFailure(error) } + for (const { signal, handler } of handlers) process.off(signal, handler) +} +if (errors.length) { + const error = new AggregateError(errors, errors.map(String).join("; "), { cause: errors[0] }) + if (!interruption) throw error + console.error(error) + process.exit(128 + (interruption.signal === "SIGINT" ? 2 : 15)) } console.log( `Captured ${input.repeats} sessions per supported arm; containers verified absent in ${output}; run release-count.py next`, diff --git a/script/token-budget/run.py b/script/token-budget/run.py index 49b49d57..8f424768 100644 --- a/script/token-budget/run.py +++ b/script/token-budget/run.py @@ -2,6 +2,7 @@ """Linux-only public CLI SDK capture; no downloads or model traffic.""" import argparse import ctypes +import errno import fcntl import json import os @@ -12,57 +13,220 @@ import subprocess import sys import time +import traceback ROOT = Path(__file__).resolve().parent FIXTURE = Path('/tmp/aictrl-token-budget') WORKSPACE = FIXTURE / 'workspace' -def execute(command, cwd, env, limit=25): - child = subprocess.Popen(command, cwd=cwd, env=env, stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) - timed = False - try: - stdout, stderr = child.communicate(timeout=limit) - except subprocess.TimeoutExpired: - timed = True - os.killpg(child.pid, signal.SIGKILL) - stdout, stderr = child.communicate(timeout=5) +def _reap_descendants(limit=2): + # Called only in a fresh forked subreaper: it has no caller-owned children. + # waitid(WNOWAIT) proves each PID is our unreaped direct child. Until we + # explicitly waitpid it, even an exiting child cannot release/reuse its PID. + end = time.monotonic() + limit surviving = False - try: - os.killpg(child.pid, 0) - surviving = True - os.killpg(child.pid, signal.SIGKILL) - except ProcessLookupError: - pass - # As a Linux subreaper, reap orphaned members after terminating our owned group. - end = time.monotonic() + 2 + records = [] while True: - try: - pid, _ = os.waitpid(-child.pid, os.WNOHANG) - if not pid and time.monotonic() < end: - time.sleep(0.01) + children = Path(f'/proc/self/task/{os.getpid()}/children').read_text().split() + if not children: + try: + os.waitpid(-1, os.WNOHANG) + except ChildProcessError: + return surviving, records + raise RuntimeError('Owned child inventory incomplete; refusing unproven cleanup') + for value in children: + pid = int(value) + state = os.waitid(os.P_PID, pid, os.WEXITED | os.WNOHANG | os.WNOWAIT) + if state is not None: + os.waitpid(pid, 0) + records.append({'pid': pid, 'state': 'reaped', 'owner': os.getpid()}) continue - if not pid: - break - except ChildProcessError: - break + surviving = True + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + # Still unreaped: this identity cannot have become a foreign PID. + pass + records.append({'pid': pid, 'state': 'live-killed', 'owner': os.getpid()}) + if time.monotonic() >= end: + raise RuntimeError('Owned descendants remain after cleanup deadline: ' + ','.join(children)) + time.sleep(0.005) + + +def _capture(command, cwd, env, limit, cancelled): + child = None + primary = None + timed = False + surviving = False + records = [] + stdout = stderr = '' try: - os.killpg(child.pid, 0) - raise RuntimeError('Owned process group remains after cleanup') - except ProcessLookupError: - pass + child = subprocess.Popen(command, cwd=cwd, env=env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) + end = time.monotonic() + limit + while child.poll() is None: + if cancelled(): + raise InterruptedError('Capture supervisor interrupted') + remaining = end - time.monotonic() + if remaining <= 0: + timed = True + break + try: + stdout, stderr = child.communicate(timeout=min(0.05, remaining)) + except subprocess.TimeoutExpired: + # Recheck the actual parent, even while a descendant holds a pipe. + pass + except BaseException as error: + primary = error + raise + finally: + errors = [] + if child is not None and child.poll() is None: + try: + child.kill() + except ProcessLookupError: + pass + except Exception as error: + errors.append(error) + try: + child.wait(timeout=5) + except Exception as error: + errors.append(error) + try: + surviving, records = _reap_descendants() + except Exception as error: + errors.append(error) + if child is not None: + try: + stdout, stderr = child.communicate(timeout=5) + except Exception as error: + errors.append(error) + finally: + for stream in (child.stdout, child.stderr): + if stream is not None: + try: + stream.close() + except Exception as error: + errors.append(error) + if errors: + detail = 'Owned capture cleanup failed: ' + '; '.join(str(error) for error in errors) + if primary is not None: + primary.add_note(detail) + else: + raise RuntimeError(detail) from errors[0] return {'exit': child.returncode, 'timedOut': timed, 'remaining_group_killed': surviving, - 'process_group': child.pid, 'command': command}, stdout, stderr + 'process_group': child.pid, 'supervisor': os.getpid(), + 'descendant_cleanup': records, 'command': command}, stdout, stderr + + +def execute(command, cwd, env, limit=25): + # A separate fork is an ownership boundary, not a network/security sandbox. + # Its only children are the capture and descendants adopted from that capture. + read, write = os.pipe() + try: + supervisor = os.fork() + except BaseException: + os.close(read) + os.close(write) + raise + if supervisor == 0: + status = 1 + try: + os.close(read) + interrupted = False + + def cancel(number, frame): + nonlocal interrupted + interrupted = True + + try: + # Do not retain the caller's unrelated pipes/files in this fork. + for value in os.listdir('/proc/self/fd'): + descriptor = int(value) + if descriptor > 2 and descriptor != write: + try: + os.close(descriptor) + except OSError as error: + if error.errno != errno.EBADF: + raise + signal.signal(signal.SIGTERM, cancel) + signal.signal(signal.SIGINT, cancel) + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), 'Unable to establish owned Linux child subreaper') + result, stdout, stderr = _capture(command, cwd, env, limit, lambda: interrupted) + payload = {'result': result, 'stdout': stdout, 'stderr': stderr} + except BaseException as error: + payload = {'error': {'type': type(error).__name__, 'message': str(error), + 'errno': getattr(error, 'errno', None), 'strerror': getattr(error, 'strerror', None), + 'filename': getattr(error, 'filename', None), + 'traceback': ''.join(traceback.format_exception(error))}} + with os.fdopen(write, 'w') as stream: + json.dump(payload, stream) + status = 0 if 'result' in payload else 1 + finally: + os._exit(status) + os.close(write) + with os.fdopen(read) as stream: + primary = None + try: + # Drain before waitpid: stdout/stderr can exceed the result pipe buffer. + payload = json.load(stream) + except BaseException as error: + primary = error + try: + os.kill(supervisor, signal.SIGTERM) + except ProcessLookupError: + pass + except Exception as cleanup_error: + primary.add_note('Supervisor cancellation failed: ' + str(cleanup_error)) + try: + # Finish owned cleanup and drain its reply before waiting. + stream.read() + except Exception as cleanup_error: + primary.add_note('Supervisor reply drain failed: ' + str(cleanup_error)) + raise + finally: + try: + _, status = os.waitpid(supervisor, 0) + except Exception as cleanup_error: + if primary is None: + raise + primary.add_note('Supervisor reap failed: ' + str(cleanup_error)) + if 'error' in payload: + error = payload['error'] + if error['errno'] is not None: + failure = OSError(error['errno'], error['strerror'], error['filename']) + else: + failure = RuntimeError('Capture supervisor failed: ' + error['type'] + ': ' + error['message']) + failure.add_note(error['traceback']) + raise failure + if not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0: + raise RuntimeError('Capture supervisor exited without successful cleanup: ' + str(status)) + return payload['result'], payload['stdout'], payload['stderr'] def verify(result): if result['remaining_group_killed']: - raise RuntimeError('Owned process group survived controller exit; capture rejected after cleanup') + raise RuntimeError('Owned descendant survived controller exit; capture rejected after cleanup') if result['exit'] or result['timedOut']: raise RuntimeError('Capture failed or exceeded deadline: ' + json.dumps(result)) +def private_directory(path, reuse=False): + try: + path.mkdir(mode=0o700) + except FileExistsError: + if not reuse: + raise + info = path.lstat() + if (not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077 + or (not reuse and stat.S_IMODE(info.st_mode) != 0o700)): + raise RuntimeError('Fixture/evidence directory must be an owned private directory, never a symlink') + return path + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--output', type=Path, required=True, help='New evidence directory; never overwritten') @@ -78,29 +242,24 @@ def main(): bun = shutil.which('bun') if not bun or not shutil.which('rg'): raise RuntimeError('Require real bun and ripgrep on PATH; install prerequisites before offline capture') - if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0: - raise OSError('Unable to establish Linux child subreaper') - try: - FIXTURE.mkdir(mode=0o700) - except FileExistsError: - info = FIXTURE.lstat() - if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077: - raise RuntimeError('Fixed fixture must be an owned private directory, never a symlink') + private_directory(FIXTURE, reuse=True) fd = os.open(FIXTURE / 'lock', os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) with os.fdopen(fd, 'a') as lock: try: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: raise RuntimeError('Exclusive fixed workspace is already in use') + if args.output.is_symlink(): + raise ValueError('Evidence output must be a new directory, never a symlink') output = args.output.resolve() if output.is_relative_to(FIXTURE): raise ValueError('Evidence output must be outside the fixed fixture directory') - output.mkdir(parents=True, exist_ok=False) + private_directory(output) records = [] for arm in ['default', 'four']: for repeat in range(1, args.repeats + 1): out = output / f'{arm}-{repeat}' - out.mkdir() + private_directory(out) if WORKSPACE.is_symlink(): raise RuntimeError('Refusing a symlink fixed workspace') if WORKSPACE.exists(): diff --git a/script/token-budget/test_protocol.py b/script/token-budget/test_protocol.py index a5199930..e7de55e8 100644 --- a/script/token-budget/test_protocol.py +++ b/script/token-budget/test_protocol.py @@ -11,6 +11,7 @@ import sys import tempfile import unittest +from unittest import mock ROOT = Path(__file__).resolve().parent BUN = shutil.which('bun') @@ -61,6 +62,134 @@ def test_surviving_group_rejected_even_after_parent_exit_zero(self): with self.assertRaises(ProcessLookupError): os.killpg(result['process_group'], 0) + def test_setsid_child_with_inherited_output_is_owned_rejected_and_reaped(self): + source = """import os,time +read,write=os.pipe() +pid=os.fork() +if pid==0: + os.close(read);os.setsid();os.write(write,b'ready');os.close(write);time.sleep(10);os._exit(0) +os.close(write);os.read(read,5);os.close(read);print(pid,flush=True) +""" + result, stdout, _ = RUN.execute([sys.executable, '-I', '-c', source], self.directory, self.env, 2) + self.assertEqual(result['exit'], 0) + self.assertFalse(result['timedOut']) + self.assertTrue(result['remaining_group_killed']) + self.assertNotEqual(result['supervisor'], os.getpid()) + pid = int(stdout.strip()) + self.assertTrue(any(row['pid'] == pid and row['state'] == 'live-killed' + and row['owner'] == result['supervisor'] for row in result['descendant_cleanup'])) + with self.assertRaisesRegex(RuntimeError, 'survived controller exit'): + RUN.verify(result) + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + + def test_setsid_child_and_grandchild_are_repeatedly_adopted_and_reaped(self): + source = """import os,time,json +read,write=os.pipe() +pid=os.fork() +if pid==0: + os.close(read);os.setsid() + grand=os.fork() + if grand==0: + os.setsid();os.write(write,(str(os.getpid())+'\\n').encode());os.close(write);time.sleep(10);os._exit(0) + os.close(write);time.sleep(10);os._exit(0) +os.close(write) +with os.fdopen(read) as stream: grand=int(stream.readline()) +print(json.dumps([pid,grand]),flush=True) +""" + result, stdout, _ = RUN.execute([sys.executable, '-I', '-c', source], self.directory, self.env, 2) + self.assertFalse(result['timedOut']) + self.assertTrue(result['remaining_group_killed']) + pids = json.loads(stdout) + killed = {row['pid'] for row in result['descendant_cleanup'] if row['state'] == 'live-killed'} + self.assertTrue(set(pids) <= killed) + for pid in pids: + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + + def test_zombie_only_adoptee_does_not_reject_a_clean_capture(self): + source = """import os +pid=os.fork() +if pid==0: os._exit(0) +os.waitid(os.P_PID,pid,os.WEXITED|os.WNOWAIT) +print(pid,flush=True) +""" + result, stdout, _ = RUN.execute([sys.executable, '-I', '-c', source], self.directory, self.env, 2) + RUN.verify(result) + self.assertFalse(result['remaining_group_killed']) + pid = int(stdout.strip()) + self.assertIn({'pid': pid, 'state': 'reaped', 'owner': result['supervisor']}, result['descendant_cleanup']) + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + + def test_caller_unrelated_live_child_is_not_signalled_reaped_or_waited(self): + foreign = subprocess.Popen([sys.executable, '-I', '-c', 'import time;time.sleep(10)'], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + identity = Path(f'/proc/{foreign.pid}/stat').read_text().rsplit(')', 1)[1].split()[19] + result, stdout, _ = RUN.execute([sys.executable, '-I', '-c', 'print("owned")'], + self.directory, self.env, 2) + RUN.verify(result) + self.assertEqual(stdout.strip(), 'owned') + self.assertIsNone(foreign.poll()) + self.assertEqual(Path(f'/proc/{foreign.pid}/stat').read_text().rsplit(')', 1)[1].split()[19], identity) + self.assertNotIn(foreign.pid, [row['pid'] for row in result['descendant_cleanup']]) + finally: + foreign.kill() + foreign.wait(timeout=3) + + def test_result_pipe_drains_output_larger_than_pipe_capacity(self): + source = 'import sys;sys.stdout.write("λ"*262144);sys.stderr.write("x"*262144)' + result, stdout, stderr = RUN.execute([sys.executable, '-I', '-c', source], self.directory, self.env, 2) + RUN.verify(result) + self.assertEqual(stdout, 'λ' * 262144) + self.assertEqual(stderr, 'x' * 262144) + + def test_root_spawn_error_remains_actionable(self): + with self.assertRaises(FileNotFoundError): + RUN.execute([str(self.directory / 'missing-owned-command')], self.directory, self.env, 2) + + def test_cleanup_failure_preserves_primary_spawn_error_with_diagnostic(self): + with mock.patch.object(RUN, '_reap_descendants', side_effect=RuntimeError('owned cleanup probe failed')): + with self.assertRaises(FileNotFoundError) as failure: + RUN.execute([str(self.directory / 'missing-owned-command')], self.directory, self.env, 2) + self.assertIn('owned cleanup probe failed', '\n'.join(failure.exception.__notes__)) + + def test_cleanup_failure_rejects_an_otherwise_successful_capture(self): + with mock.patch.object(RUN, '_reap_descendants', side_effect=RuntimeError('owned cleanup probe failed')): + with self.assertRaisesRegex(RuntimeError, 'Owned capture cleanup failed'): + RUN.execute([sys.executable, '-I', '-c', 'print("clean")'], self.directory, self.env, 2) + + def test_private_evidence_directory_ignores_permissive_umask_and_refuses_overwrite(self): + previous = os.umask(0) + try: + output = RUN.private_directory(self.directory / 'evidence') + arm = RUN.private_directory(output / 'default-1') + finally: + os.umask(previous) + self.assertEqual(output.stat().st_mode & 0o777, 0o700) + self.assertEqual(arm.stat().st_mode & 0o777, 0o700) + marker = output / 'accepted.json' + marker.write_text('retained') + with self.assertRaises(FileExistsError): + RUN.private_directory(output) + self.assertEqual(marker.read_text(), 'retained') + + def test_reused_fixture_directory_rejects_shared_mode_and_symlinks(self): + shared = self.directory / 'shared' + shared.mkdir(mode=0o700) + shared.chmod(0o755) + with self.assertRaisesRegex(RuntimeError, 'owned private directory'): + RUN.private_directory(shared, reuse=True) + target = self.directory / 'owned' + RUN.private_directory(target) + link = self.directory / 'link' + link.symlink_to(target, target_is_directory=True) + with self.assertRaisesRegex(RuntimeError, 'never a symlink'): + RUN.private_directory(link, reuse=True) + with self.assertRaises(FileExistsError): + RUN.private_directory(link) + def test_before_import_guard_rejects_network_preconnect_and_install(self): guard = self.directory / 'guard.ts' env = {**self.env, 'GUARD_MODULE': str(ROOT / 'guard.ts'), 'GUARD_FILE': str(guard), @@ -91,7 +220,7 @@ def test_before_import_guard_rejects_network_preconnect_and_install(self): self.assertEqual(ledger, [{'origin': 'https://example.invalid', 'path': '/offline-probe', 'allowed': False}]) def test_fixed_workspace_lock_rejects_parallel_campaign_before_output(self): - RUN.FIXTURE.mkdir(mode=0o700, exist_ok=True) + RUN.private_directory(RUN.FIXTURE, reuse=True) fd = os.open(RUN.FIXTURE / 'lock', os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) output = self.directory / 'must-not-exist' with os.fdopen(fd, 'a') as lock: @@ -111,5 +240,21 @@ def test_clock_preload_is_fixed_before_imports(self): self.assertEqual(p.returncode, 0, p.stderr) self.assertEqual(json.loads(p.stdout), {'iso': '2026-10-21T12:00:00.000Z', 'date': 'Wed Oct 21 2026', 'now': 1792584000000}) +class ReleaseManifest(unittest.TestCase): + def test_missing_final_cleanup_is_explicit_incomplete_campaign(self): + counter = module('release_count', ROOT / 'release-count.py') + for checks in [None, [], [{'name': 'last-run', 'removed': {}, 'remaining': {}}], + [{'finalCampaignCheck': {}}], ['not-an-object']]: + with self.subTest(checks=checks), tempfile.TemporaryDirectory(prefix='release-incomplete-') as directory: + campaign = Path(directory) / 'campaign.json' + data = {'protocolVersion': 3, 'failures': [], 'REPS': 2} + if checks is not None: + data['cleanupChecks'] = checks + campaign.write_text(json.dumps(data)) + with self.assertRaisesRegex(ValueError, 'Incomplete release campaign: final cleanup evidence missing'): + counter.analyze(campaign) + self.assertFalse(campaign.with_name('release-comparison.json').exists()) + + if __name__ == '__main__': unittest.main() From b45f6f3da7341e8e1ad1fd84b30461b65a1513e1 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 21 Oct 2026 15:26:36 +0100 Subject: [PATCH 6/7] ci: run checks for stacked pull requests --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f1b08c7..dbe2dff0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,6 @@ on: push: branches: [main] pull_request: - branches: [main] permissions: contents: read From 2999d0f7b7bf3bbca9ed93d00e73fbd931279732 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Wed, 21 Oct 2026 16:52:54 +0100 Subject: [PATCH 7/7] docs(cli): clarify scalar option hyphen values --- README.md | 2 +- packages/cli/test/cli/fixture/integrated-capture.ts | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 5edb862e..bdb3d4c8 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Stdin is read only when a standalone `-` is a message operand, or when no messag The marker can appear before or after `--`; repeated markers still read stdin once. Markers are removed from the model message, command arguments and derived session title. `--` ends option parsing, so `run -- -` also selects stdin. An operand such as `"literal -"` remains ordinary text. An explicit empty operand (`run ""`) does not select implicit stdin. Selected stdin is read to EOF and appended after one actual newline. Ordinary prompt operands are joined with spaces without adding artificial quotes or backslash escapes; quotes, backslashes and whitespace inside an operand remain its text. -Required-value options own a separated hyphen: `--model -` and `--command -` do not select stdin, nor do their `=` forms. Optional `--port` leaves a separated `-` as the stdin operand (`--port -`). Use `--file=-` to attach a file literally named `-`; `--file -` is missing its required file value. Multiple file values and repeated `--skill` flags retain their existing behavior. Required option values are checked by the argument parser; explicit `--title ""` remains supported. +Scalar required-value options own a separated hyphen: `--model -` and `--command -` do not select stdin, nor do their `=` forms. Optional `--port` leaves a separated `-` as the stdin operand (`--port -`). Use `--file=-` to attach a file literally named `-`; `--file -` is missing its required file value. Multiple file values and repeated `--skill` flags retain their existing behavior. Required option values are checked by the argument parser; explicit `--title ""` remains supported. An explicit marker reads stdin even from a terminal. Any selected stream can wait for EOF indefinitely; this change adds no stdin timeout. Callers must close selected input or apply their own supervisor deadline. diff --git a/packages/cli/test/cli/fixture/integrated-capture.ts b/packages/cli/test/cli/fixture/integrated-capture.ts index 57be5b11..ca91b001 100644 --- a/packages/cli/test/cli/fixture/integrated-capture.ts +++ b/packages/cli/test/cli/fixture/integrated-capture.ts @@ -1,4 +1,3 @@ -import { expect } from "bun:test" import { spawn, spawnSync } from "node:child_process" import fs from "node:fs/promises" import { existsSync, readFileSync } from "node:fs" @@ -6,7 +5,6 @@ import os from "node:os" import { Database } from "bun:sqlite" import path from "node:path" -const template = "FIRST<$1>\nSECOND<$2>\nREST<$3>" const guard = path.join(import.meta.dir, "stdin-guard.ts") const pause = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))