From 50b56b2ca487812b1888e299aac24c912bdd4b97 Mon Sep 17 00:00:00 2001 From: zzwong <6979793+zzwong@users.noreply.github.com> Date: Thu, 8 Oct 2026 06:32:33 -0400 Subject: [PATCH] test(integration): validate combined draft stack Integration-validation-only. DO NOT MERGE. Exact source tree 7fd8e55a2b263b912b9103a3ab213c46b929a59b. No combined execution result is inherited. --- .github/workflows/ci.yml | 30 + BENCHMARKING.md | 36 +- Makefile | 26 +- README.md | 34 +- docs/development.md | 76 ++ docs/llm-task-artifacts.md | 192 +++- docs/model-catalog-plan.md | 52 + internal/app/runtime_test.go | 2 +- internal/benchmark/discussion_test.go | 69 ++ internal/benchmark/suite.go | 48 +- internal/cmd/benchmarkcmd/benchmarkcmd.go | 9 +- .../cmd/benchmarkcmd/benchmarkcmd_test.go | 21 +- internal/cmd/benchmarkcmd/compare.go | 139 +-- .../discussion_relocation_test.go | 76 ++ internal/cmd/benchmarkcmd/discussion_test.go | 252 +++++ internal/cmd/benchmarkcmd/executor.go | 1 + internal/cmd/benchmarkcmd/executor_test.go | 8 +- internal/cmd/benchmarkcmd/run.go | 164 ++-- internal/cmd/catalogcmd/catalogcmd.go | 3 +- internal/cmd/catalogcmd/catalogcmd_test.go | 47 + internal/cmd/reviewcmd/reviewcmd.go | 14 + internal/cmd/reviewcmd/reviewcmd_test.go | 67 +- internal/cmd/root/root_test.go | 47 +- internal/dossier/dossier.go | 27 +- internal/llm/adapter.go | 56 +- internal/llm/request_cost_evidence.go | 570 +++++++++++ internal/llm/request_cost_evidence_test.go | 361 +++++++ internal/llm/subprocess.go | 3 +- .../request-cost-evidence-v1.golden.json | 1 + internal/llm/tool_evidence.go | 95 ++ internal/llm/tool_evidence_test.go | 30 + internal/llmadapters/api.go | 102 +- internal/llmadapters/api_test.go | 107 +++ internal/llmadapters/pi_rpc.go | 331 +++++-- .../llmadapters/pi_rpc_extension_unix_test.go | 50 + .../pi_rpc_integration_settlement_test.go | 142 +++ internal/llmadapters/pi_rpc_runtime_test.go | 890 ++++++++++++++++++ .../llmadapters/pi_rpc_runtime_unix_test.go | 168 ++++ internal/llmadapters/pi_rpc_test.go | 637 ++++++++++++- internal/llmadapters/pi_rpc_tool_evidence.go | 287 ++++++ .../llmadapters/pi_rpc_tool_evidence_test.go | 303 ++++++ internal/llmadapters/request_cost_evidence.go | 192 ++++ .../llmadapters/request_cost_evidence_test.go | 360 +++++++ internal/llmadapters/subprocess.go | 82 +- .../subprocess_cleanup_order_test.go | 151 +++ internal/llmadapters/subprocess_test.go | 192 +++- internal/llmlifecycle/lifecycle.go | 162 +++- internal/llmlifecycle/lifecycle_test.go | 4 +- .../llmlifecycle/request_cost_evidence.go | 665 +++++++++++++ .../request_cost_evidence_test.go | 814 ++++++++++++++++ .../request-cost-checkpoint-v1.golden.json | 1 + internal/llmlifecycle/tool_evidence_test.go | 160 ++++ internal/modelcatalog/catalog.go | 83 +- internal/modelcatalog/data/manifest.json | 68 +- internal/modelcatalog/data/pricing.csv | 82 +- internal/modelcatalog/request_pricing.go | 271 ++++++ internal/modelcatalog/request_pricing_test.go | 636 +++++++++++++ .../modelcatalog/testdata/v1/defaults.csv | 10 + .../modelcatalog/testdata/v1/manifest.json | 28 + internal/modelcatalog/testdata/v1/models.csv | 37 + internal/modelcatalog/testdata/v1/pricing.csv | 41 + .../modelcatalog/testdata/v1/runtimes.csv | 6 + internal/pipeline/pipeline.go | 197 ++-- internal/pipeline/prompts.go | 37 +- .../pipeline/request_cost_evidence_test.go | 50 + .../pipeline/thread_analysis_failure_test.go | 159 ++++ .../pipeline/thread_analysis_prompt_test.go | 126 +++ .../without_discussion_relocation_test.go | 368 ++++++++ .../without_discussion_symlink_test.go | 112 +++ internal/pipeline/without_discussion_test.go | 368 ++++++++ internal/pricing/pricing.go | 3 +- internal/pricing/pricing_test.go | 151 ++- .../reviewplan/relocation_coverage_test.go | 2 +- internal/reviewplan/reviewplan.go | 26 +- internal/reviewplan/summary.go | 98 +- .../thread_analysis_failure_test.go | 139 +++ .../reviewrun/superseded_approval_test.go | 228 +++++ internal/runartifact/runartifact.go | 21 +- internal/threadanalysis/isolation_test.go | 239 +++++ internal/threadanalysis/threadanalysis.go | 53 +- .../threadanalysis/threadanalysis_test.go | 16 +- internal/threadrespond/threadrespond.go | 2 +- internal/view/review.go | 37 +- .../review_thread_analysis_failure_test.go | 54 ++ .../without_discussion_relocation_test.go | 48 + internal/workbench/offline_symlink_test.go | 83 ++ internal/workbench/workbench.go | 60 +- internal/workbench/workbench_test.go | 41 + 88 files changed, 11753 insertions(+), 583 deletions(-) create mode 100644 internal/benchmark/discussion_test.go create mode 100644 internal/cmd/benchmarkcmd/discussion_relocation_test.go create mode 100644 internal/cmd/benchmarkcmd/discussion_test.go create mode 100644 internal/cmd/catalogcmd/catalogcmd_test.go create mode 100644 internal/llm/request_cost_evidence.go create mode 100644 internal/llm/request_cost_evidence_test.go create mode 100644 internal/llm/testdata/request-cost-evidence-v1.golden.json create mode 100644 internal/llm/tool_evidence.go create mode 100644 internal/llm/tool_evidence_test.go create mode 100644 internal/llmadapters/pi_rpc_integration_settlement_test.go create mode 100644 internal/llmadapters/pi_rpc_runtime_test.go create mode 100644 internal/llmadapters/pi_rpc_runtime_unix_test.go create mode 100644 internal/llmadapters/pi_rpc_tool_evidence.go create mode 100644 internal/llmadapters/pi_rpc_tool_evidence_test.go create mode 100644 internal/llmadapters/request_cost_evidence.go create mode 100644 internal/llmadapters/request_cost_evidence_test.go create mode 100644 internal/llmadapters/subprocess_cleanup_order_test.go create mode 100644 internal/llmlifecycle/request_cost_evidence.go create mode 100644 internal/llmlifecycle/request_cost_evidence_test.go create mode 100644 internal/llmlifecycle/testdata/request-cost-checkpoint-v1.golden.json create mode 100644 internal/llmlifecycle/tool_evidence_test.go create mode 100644 internal/modelcatalog/request_pricing.go create mode 100644 internal/modelcatalog/request_pricing_test.go create mode 100644 internal/modelcatalog/testdata/v1/defaults.csv create mode 100644 internal/modelcatalog/testdata/v1/manifest.json create mode 100644 internal/modelcatalog/testdata/v1/models.csv create mode 100644 internal/modelcatalog/testdata/v1/pricing.csv create mode 100644 internal/modelcatalog/testdata/v1/runtimes.csv create mode 100644 internal/pipeline/request_cost_evidence_test.go create mode 100644 internal/pipeline/thread_analysis_failure_test.go create mode 100644 internal/pipeline/thread_analysis_prompt_test.go create mode 100644 internal/pipeline/without_discussion_relocation_test.go create mode 100644 internal/pipeline/without_discussion_symlink_test.go create mode 100644 internal/pipeline/without_discussion_test.go create mode 100644 internal/reviewplan/thread_analysis_failure_test.go create mode 100644 internal/reviewrun/superseded_approval_test.go create mode 100644 internal/threadanalysis/isolation_test.go create mode 100644 internal/view/review_thread_analysis_failure_test.go create mode 100644 internal/view/without_discussion_relocation_test.go create mode 100644 internal/workbench/offline_symlink_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d10ae486..aa72a1d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,18 @@ jobs: with: go-version-file: go.mod + test-race: + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + CGO_ENABLED: "1" # Required by Go's race detector. + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - uses: open-cli-collective/.github/actions/go-test@b6382514b809d960c6fbdc85745a60c14cde04f1 # shared .github + with: + go-version-file: go.mod + target: test-race + test-no1password-optout: runs-on: ubuntu-latest env: @@ -70,6 +82,24 @@ jobs: go-version-file: go.mod target: test-static-smoke + pi-runtime: # installed Pi RPC runtime gate — not a required check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22.22.3 # Pi 1.0 requires Node >= 22.19.0 + # The Makefile owns the Pi version; test-pi-runtime fails if it differs. + # Read it first so a failed lookup stops the step instead of installing + # an unpinned (latest) Pi. + - run: | + pi_version="$(make -s pi-runtime-version)" + npm install -g --ignore-scripts "@earendil-works/pi-coding-agent@${pi_version}" + - uses: open-cli-collective/.github/actions/go-test@b6382514b809d960c6fbdc85745a60c14cde04f1 # shared .github + with: + go-version-file: go.mod + target: test-pi-runtime + lint: runs-on: ubuntu-latest steps: diff --git a/BENCHMARKING.md b/BENCHMARKING.md index fa3a3af1..8078bec4 100644 --- a/BENCHMARKING.md +++ b/BENCHMARKING.md @@ -130,6 +130,7 @@ cases: pr: https://github.com/OWNER/REPO/pull/123 review_base_sha: 1111111 review_head_sha: 2222222 + without_discussion: true # default for pinned cases; false opts into saved sessions expected_base_sha: abc1234 expected_head_sha: def5678 anchors: @@ -148,6 +149,34 @@ metadata for downstream reports and graders; they do not change what gets reviewed. All SHA fields must be non-empty 7 to 64 character hexadecimal SHAs when present. +`without_discussion` is an optional per-case boolean for full-pipeline +`benchmark run`. It defaults to `true` when both review SHAs are pinned, and +`false` for unpinned cases. Setting it to `true` requires both review SHAs. +Discussion-free runs do not resume or overwrite saved orchestrator/reviewer +sessions or the PR's reviewer cohort, so candidates do not inherit earlier +reviews. Both subprocess and `--in-process` runs apply the same setting. +Explicit `false` opts into the normal saved-session behavior, including for +pinned cases. Expected SHA metadata alone does not enable discussion-free mode. + +For cross-version comparisons with `--cr-bin`, isolated cases require a binary +that supports `cr review --without-discussion` and reports +`run.without_discussion: true` in review JSON. An older binary may reject the +flag; a successful child exit without that marker is recorded as +`discussion_isolation_unverified` and counted as a failed benchmark cell. +The harness preserves the child's actual exit code and raw output and never +retries without the flag. To deliberately benchmark an older binary, set +`without_discussion: false` and account for possible saved-session contamination. +Doctor and selected-case artifacts record the effective setting. Per-run +metrics, manifests, summaries, and comparisons distinguish +`requested_without_discussion` from `without_discussion_verified`; a missing +marker is never treated as proof of isolation. `benchmark compare` rechecks +the current review artifact; replacing or corrupting it invalidates prior +verification and fails a cell that requested discussion-free execution. + +Selector-only `benchmark select` already uses a caller-owned selection run with +no saved review sessions. It preserves this case setting in its recipe but does +not need the full-review flag. + Candidate `profile` must reference a configured profile. Candidate PR hosts must match the candidate profile's Git host. For the current full-pipeline `validate`, `doctor`, and `run` commands, candidates must declare explicit @@ -270,8 +299,9 @@ When set on the case, `run` also passes: |------------|-------------| | `review_base_sha` | `--review-base-sha ` | | `review_head_sha` | `--review-head-sha ` | +| `without_discussion` | `--without-discussion` when effectively true; defaults to true for pinned cases | -Unset fields are omitted. Benchmark candidate artifacts record reviewer +Other unset fields are omitted. Benchmark candidate artifacts record reviewer `effort_source` as `inherited` or `override`, so reports can distinguish the two execution recipes even when the inherited effort is resolved later per agent. Effort values are `low`, `medium`, `high`, `xhigh`, or `max`; validation @@ -280,8 +310,8 @@ rejects levels unsupported by the candidate profile's runtime. `claude_cli`, `anthropic_api` supports through `high`. Effective extended-effort support depends on the selected model and, for CLI adapters, the installed CLI. -Posting, retry, approval, thread-resolution, session, -and live-review flags are never taken from the suite. +Apart from the per-case `without_discussion` setting, posting, retry, approval, +thread-resolution, session, and live-review flags are never taken from the suite. `--cr-bin ` selects the binary used for child review runs. If omitted, `run` uses the current `cr` binary. `doctor` reports the binary it would use. diff --git a/Makefile b/Makefile index b572a772..f851f2a1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all build test test-cover test-static-smoke lint fmt tidy deps check install snapshot package-render-check release clean +.PHONY: all build test test-race test-cover test-static-smoke test-pi-runtime pi-runtime-version check-pi-runtime-version lint fmt tidy deps check install snapshot package-render-check release clean # Standard keyring tags: enable 1Password support, keep passage disabled. GOFLAGS ?= -tags=keyring_nopassage @@ -12,12 +12,36 @@ build: test: go test -v ./... +# Exercise SQLite startup/migration contention repeatedly without racing the +# entire suite. Keep package execution and test scheduling bounded in CI. +test-race: + go test -race -count=25 -cpu=4 -parallel=4 -p=1 -timeout=5m ./internal/ledger/... ./internal/dbmig/... + test-cover: go test -coverprofile=coverage.out ./... test-static-smoke: go test -v ./internal/... ./cmd/... -count=1 +# Exact Pi version the installed-runtime gate requires. CI installs this +# version; test-pi-runtime fails, rather than skips, when `pi --version` on PATH +# reports anything else or Pi is missing. +PI_RUNTIME_VERSION ?= 1.0.0 + +# A blank pin would make the runtime tests skip and let npm install the latest +# Pi, so both targets below require exactly one version. +check-pi-runtime-version: + @if [ "$(words $(PI_RUNTIME_VERSION))" != 1 ]; then \ + echo "PI_RUNTIME_VERSION must be exactly one Pi version" >&2; \ + exit 1; \ + fi + +test-pi-runtime: check-pi-runtime-version + CR_PI_RUNTIME_VERSION=$(strip $(PI_RUNTIME_VERSION)) go test -v -race -count=1 -run 'TestPiRPCRuntime|TestPiRPCReviewerExtensionLoadsInInstalledPi' ./internal/llmadapters + +pi-runtime-version: check-pi-runtime-version + @echo $(strip $(PI_RUNTIME_VERSION)) + lint: golangci-lint run diff --git a/README.md b/README.md index 9b391c71..70e7d63a 100644 --- a/README.md +++ b/README.md @@ -119,16 +119,33 @@ cr init --non-interactive \ ``` Setup with Pi's local RPC runtime. Install Pi's coding agent and make sure the -`pi` binary is available on `PATH` before running `cr review`. New installs -should use the current npm package (`@earendil-works/pi-coding-agent`); existing -installs from the previous npm scope can also work when CR's compatibility -preflight confirms the reviewer controls it requires: RPC/system-prompt mode; -`--no-builtin-tools` with an exact `--tools` allowlist; explicit `--extension` loading while -`--no-extensions` disables discovery; and `--no-context-files`, `--no-approve`, -`--no-skills`, `--no-prompt-templates`, `--no-themes`, and `--no-session`. -CR preflights these capabilities before starting a Pi reviewer and returns an +`pi` binary is available on `PATH` before running `cr review`. Use Pi 1.0 or +newer from the current npm package (`@earendil-works/pi-coding-agent`). CR +relies on Pi 1.0's RPC completion contract: it keeps reading events through +automatic retries and compaction recovery until `agent_settled`, takes the +answer only from the final completed assistant message, and sums usage from +each completed assistant message and compaction summary. A final provider +error or abort fails the task with Pi's diagnostic. Older runtimes, including +installs from the previous npm scope, that end a run without `agent_settled` +fail with an incompatible-runtime error after a short grace period instead of +returning a result. + +CR also preflights the reviewer controls it requires before starting a Pi +reviewer: RPC/system-prompt mode; `--no-builtin-tools` with an exact `--tools` +allowlist; explicit `--extension` loading while `--no-extensions` disables +discovery; and `--no-context-files`, `--no-approve`, `--no-skills`, +`--no-prompt-templates`, `--no-themes`, and `--no-session`. It returns an incompatible-runtime error when any control is unavailable. +Every CR-owned Pi request disables `AGENTS.md`/`CLAUDE.md` context discovery, +supplies its own system prompt, and passes an explicit empty +`--append-system-prompt` source. This also suppresses ambient `SYSTEM.md` and +`APPEND_SYSTEM.md` inputs, including those in the selected agent directory. +CR leaves Pi's selected agent directory and authentication storage in place; +credentials are never copied into disposable scratch state, so Pi can persist +OAuth refresh updates to its original store. Resource discovery remains +disabled, and only reviewers load the CR-owned read-only tool extension. + ```bash cr init --non-interactive \ --llm-provider pi \ @@ -1235,6 +1252,7 @@ Review selection and execution flags: | `--reviewer-effort ` | Override reviewer-stage effort with `low`, `medium`, `high`, `xhigh`, or `max`, subject to runtime support. Available for dry-run, no-post, and live reviews. | | `--review-base-sha ` | Review this base commit SHA instead of the PR's current base SHA. Requires `--review-head-sha` and `--dry-run` or `--no-post`. | | `--review-head-sha ` | Review this head commit SHA instead of the PR's current head SHA. Requires `--review-base-sha` and `--dry-run` or `--no-post`. | +| `--without-discussion` | Replay the pinned review as a first pass. cr reads no review threads, thread outcomes, issue comments, or prior reviews, and neither resumes nor updates the PR's orchestrator session or reviewer cohort sessions, so the discussion is not in any selection, reviewer, or rollup prompt; the PR title, description, and diff still are. Reviewers also get a narrower tool surface: the review checkouts have no git remotes, `claude_cli` reviewers are denied WebFetch, WebSearch, and Bash commands such as `gh`, `curl`, `wget`, and `git fetch`/`pull`/`ls-remote`, and `codex_cli` reviewers run with web search and sandbox network access off. That blocks the obvious lookups but is not a network sandbox, so a reviewer with a shell could still reach the network another way. The run marker (`review-run.json`), dossier discussion artifacts, and `--json` run output record `without_discussion: true`, and such a run never resumes an incomplete run made with discussion (or the reverse). Requires `--review-base-sha`, `--review-head-sha`, and `--dry-run` or `--no-post`; cannot be combined with `--fresh-session`. | | `--session ` | Override the PR's default orchestrator session with a named live-review session. Reviewer cohorts remain PR-scoped. Not allowed with `--dry-run`, `--no-post`, or `--retry-posts`. | Review progress on stderr reports the merged reviewer catalog, final selected diff --git a/docs/development.md b/docs/development.md index cc0c2027..a76e742b 100644 --- a/docs/development.md +++ b/docs/development.md @@ -90,15 +90,91 @@ progress interface rather than importing Cobra or root options. ```bash make build # compile the binary make test # go test ./... +make test-race # repeated race checks for ledger/dbmig make lint # golangci-lint run make tidy # go mod tidy and verify go.mod is unchanged make deps # download and verify Go modules make check # tidy + fmt + lint + test + build make clean # remove build artifacts +make test-pi-runtime # installed Pi runtime gate (see Pi Runtime Gate) ``` `make snapshot` runs a local GoReleaser snapshot build without publishing. +### Concurrency checks + +`make test-race` runs all tests in `internal/ledger` and `internal/dbmig` under +the Go race detector with 25 repetitions. This includes concurrent ledger +startup and allocation, the immediate transaction-lock DSN contract, and +migration version re-checks. Repeating the existing contention tests makes +timing-dependent failures more likely to surface without applying race +detection to the entire suite. + +The target sets test `GOMAXPROCS` to four with `-cpu=4` and caps tests using +`t.Parallel` at four with `-parallel=4`. Contention-test worker goroutines +remain bounded by their own fixture counts. Packages run serially, with a +five-minute timeout per package. The target requires +`CGO_ENABLED=1` and a C compiler, even though the SQLite driver itself is +pure Go. CI runs the same target in the separate `test-race` job on pull +requests and pushes to `main`, with a 20-minute job timeout. The job uses the +same pinned shared test action as `test`, including Go version selection from +`go.mod` and the action's default dependency/build caching. Existing required +check names and branch-protection settings are unchanged; requiring the new +check is a separate maintainer decision after it has run successfully. + +## Pi Runtime Gate + +`make test-pi-runtime` runs the `pi_rpc` adapter against the installed Pi on +`PATH` instead of the fake Pi the ordinary adapter tests use. It sets +`CR_PI_RUNTIME_VERSION` to the Makefile's `PI_RUNTIME_VERSION` (currently +`1.0.0`, printed by `make -s pi-runtime-version`). Both targets fail when +`PI_RUNTIME_VERSION` is blank or holds more than one version. When +`CR_PI_RUNTIME_VERSION` is set, the runtime tests fail if it is blank, if Pi is +missing, or if `pi --version` reports any other version. When it is unset, as in +`make test`, they skip. To install the pinned runtime: + +```bash +pi_version="$(make -s pi-runtime-version)" +npm install -g --ignore-scripts "@earendil-works/pi-coding-agent@${pi_version}" +make test-pi-runtime +``` + +The tests make no paid or external calls: + +- Each test gives Pi a fresh `PI_CODING_AGENT_DIR` and `HOME` with + `PI_OFFLINE=1`. +- The only model is a scripted OpenAI-compatible endpoint on loopback, with a + dummy key stored in the selected directory's `auth.json`. +- Reviewer tool calls run the real `__pi-review-tool` helper, dispatched to the + test binary by `TestMain`. +- Hostile instructions, extensions, and MCP servers are planted in the + actually selected agent directory as well as under the fixture `HOME`'s + default `.pi/agent` location and in the reviewer checkout. Hostile skills + are also planted under the fixture home and checkout. The tests fail if any + resource reaches a provider request or runs. +- Both structured and reviewer cases plant each selected `AGENTS.md`, + `CLAUDE.md`, `AGENTS.override.md`, `SYSTEM.md`, and `APPEND_SYSTEM.md` + independently, so context-file precedence cannot hide an uncovered input. + +The Pi 1.0 prompt-resource contract is explicit: `--no-context-files` disables +context discovery, CR's `--system-prompt` suppresses `SYSTEM.md` discovery, +and `--append-system-prompt` with an empty argument supplies an explicit CLI +source list that suppresses `APPEND_SYSTEM.md` discovery while contributing no +text. See Pi's pinned [argument parser](https://github.com/earendil-works/pi/blob/v1.0.0/packages/coding-agent/src/cli/args.ts) +and [resource loader](https://github.com/earendil-works/pi/blob/v1.0.0/packages/coding-agent/src/core/resource-loader.ts). + +CR does not redirect `PI_CODING_AGENT_DIR`, `HOME`, or Pi's credential store to +achieve isolation. Authentication, model configuration, and OAuth refresh +persistence continue to use the selected directory. The ordinary subprocess +test verifies that a synthetic auth-state update remains in the selected store +after scratch cleanup; it does not exercise a real OAuth provider or refresh. +All installed-runtime evidence is local/mock-provider evidence, not deployed +validation or an operating-system sandbox claim. + +The CI `pi-runtime` job installs the Makefile version on the Node release it +pins and runs the same target. It is not a required check. To move to a new Pi +release, change `PI_RUNTIME_VERSION` and run the target locally first. + ## Repo-Local Shape - Module: `github.com/open-cli-collective/codereview-cli` diff --git a/docs/llm-task-artifacts.md b/docs/llm-task-artifacts.md index ff7fc9b3..2110ed17 100644 --- a/docs/llm-task-artifacts.md +++ b/docs/llm-task-artifacts.md @@ -42,12 +42,16 @@ trust the final `metadata.json` name, never a temporary metadata file. ## Schema Version -`schema_version` is currently `3`. Version 2 added the relocation/context -coverage contract. Version 3 adds the pinned symlink metadata digest and +`schema_version` is currently `4`. Version 2 added the relocation/context +coverage contract. Version 3 added the pinned symlink metadata digest and inspection-contract version to reviewer, coverage-repair, and rollup resume -inputs. Schema versions 1 and 2 are rejected for reuse and require rerunning -the task; old metadata is never treated as safe merely because the prompt or -assignment count appears unchanged. +inputs. Version 4 adds bounded per-call reviewer tool telemetry, including the +explicit distinction between regular-file and symlink-metadata read views. +Schema versions 1, 2, and 3 are rejected for reuse and require rerunning the task, +so old successful Pi tasks cannot silently reuse output without this collection +contract. The schema version participates in the input fingerprint. Old +metadata is never treated as safe merely because the prompt or assignment +count appears unchanged. Bump it when changing any load-bearing field, status value, fingerprint input, task identity, or resume rule in a way that could make an in-flight run unsafe @@ -112,8 +116,9 @@ differ. reuse the output only when the metadata schema and input fingerprint still match the current task. -`failed_isolated` is for reviewer-local LLM failures while the caller context is -still valid. This includes structured validation failures and provider failures +`failed_isolated` is for reviewer-local LLM failures and review-time thread +analysis failures while the caller context is still valid. This includes +structured validation failures and provider failures after a task provider session has started. Provider start failures with no session are treated as blocking because they can indicate auth, quota, or other systemic adapter problems. The failed reviewer is treated as @@ -122,6 +127,37 @@ diagnostic. Sibling reviewers continue to run. A review with any isolated reviewer failure must not approve; the final event is clamped to at least `comment`. +Review-time thread analysis uses the same isolation boundary. An isolated thread +failure leaves that thread untouched, emits a warning, and appears under +**Unanalyzed Threads** in the posted rollup. Public diagnostics report total +counts and at most five deterministic examples, with each ID and diagnostic +sampled to 500 Unicode runes. The complete `thread_analysis_failures` collection +remains in the dry-run JSON summary, and every failure remains in the local +per-thread task artifacts. Successful sibling analyses still produce replies and +resolutions, and reviewer execution continues. Proposed selection actions for a +failed thread are suppressed. Any unanalyzed thread withholds an otherwise +approving review, even when the reviewers inspected every changed file. + +Matching isolated thread failures are reused when resuming an interrupted run; +resume does not spend another provider call on them. Use `--rerun` to retry in a +fresh run. Existing `failed_blocking` thread tasks still retry on resume, and +matching successful analyses remain cached. Public diagnostics do not echo +rejected model values or provider error text; detailed errors remain in the local +task artifacts. Cancellation, provider failures without execution evidence, +stale or corrupt artifacts, and persistence or checkpoint errors remain +blocking. Response-only `cr respond` retains its blocking thread-analysis +behavior. + +The synthesis prompt includes bounded failure counts and examples plus a digest +of the complete typed thread failure evidence when present. Its fingerprint +therefore changes even when an omitted failure changes. The existing coverage +artifact, relocation manifest, context contract, and pinned symlink metadata +inspection dependencies remain intact. Prompts without thread failures remain +unchanged, preserving reuse of existing successful synthesis tasks. This change +requires no additional task-metadata schema bump: existing blocking tasks retain +their retry semantics, and new isolated thread tasks use the existing status +contract. + `failed_blocking` means the task prevents dependent phases from safely running. Selection and rollup failures are blocking. `reviewrun` is the sole owner of live planner-error outcomes: blocking LLM failures leave fresh and resumed runs @@ -174,13 +210,81 @@ The lifecycle persists this evidence in metadata and restores it when loading a cached task, so reusing successful output preserves the tool state used to assess coverage and approval. -In schema version `2`, an absent `reviewer_tool_evidence` field means no +In schema version `4`, an absent `reviewer_tool_evidence` field means no adapter-provided evidence is available. It is distinct from explicit `not_invoked` evidence: absence does not trigger the tool-evidence coverage check, but the normal coverage checks for skipped, missing, and unassigned -files still apply. Version 1 metadata fails the schema check before any task +files still apply. Version 1, 2, or 3 metadata fails the schema check before any task output can be reused. +### Bounded execution trace + +Pi reviewer invocations also persist `reviewer_tool_evidence.trace`, version +`1`, source `pi_rpc`. This trace records only actual `tool_execution_start` and +`tool_execution_end` events for `cr_read` and `cr_diff`. Model tool-call text, +`inspected_files` claims, searches, and directory listings do not create file +read evidence. Other adapters, including subprocess adapters, leave evidence +absent; an absent trace means this per-call contract is unsupported/unavailable, +not that no calls happened. A present empty `calls` array means no supported +execution events were observed during that invocation. + +Each call retains its exact bounded `call_id`, tool, observed start/end flags, +requested byte offset/limit, and `succeeded`, `failed`, or `incomplete` outcome. +Success requires matching start/end call identity and an explicit non-error +execution outcome. An unmatched end, missing end/identity/outcome, duplicate +identity, or mismatched tool cannot establish success. A stable +`provenance_issue` explains missing or ambiguous observations without copying +raw arguments, tool output, or failure text into the trace. + +For `cr_read`, `path` is the unchanged canonical repository-relative argument. +Absolute, traversing, VCS metadata, noncanonical, or unavailable paths are +omitted with a provenance issue rather than normalized into another file's +evidence. This collection is independent of reviewer assignments: context +reads can be recorded without receiving assignment coverage. `read_view` is +`file` for an ordinary read or `symlink` for an explicit `view="symlink"` +request. Unavailable arguments or an invalid explicit view leave `read_view` +absent rather than assuming a file-body read. Symlink-view ranges cover pinned +metadata only: they do not read a destination body or a head regular-file body for a base-only +symlink transition. The trace does not duplicate pinned link payloads, digests, +target resolution, or payload-omission evidence from the symlink artifact. +Tool success alone does not establish that a non-omitted link payload was read. +`cr_diff` reads the fixed pinned diff and has no path parameter, so its entries are always +unattributed to individual files. Its success never fabricates a per-file read. + +`output`, when present, is CR-extension-reported transport byte count and +truncation state. Absence means unknown. `output.truncated: false` does not mean +a complete file/diff was returned: the helper may have selected a bounded +range. Range headers in returned text are not parsed as provenance, because +repository content itself can contain those strings. Zero requested offset and +limit select the helper's default bounded read, not unbounded/full inspection. + +The collector retains at most 256 calls and 64 KiB of combined identity/path +bytes. IDs are limited to 256 bytes and paths to 4096 bytes. Identity/path +strings are omitted instead of shortened into misleading identities. +`truncated` records collection limits, and `dropped_events` counts events +discarded at the record/identity budget. End events for already retained calls +can still complete those calls after the cap. `stream_complete` requires a +successful `agent_settled` result, clean parse/process exit, and an uncanceled +task context; a complete stream can still contain incomplete calls. The trace +has its own bound and survives RPC/stderr log caps. + +Lifecycle metadata preserves the terminal invocation's trace for success, +isolated failure, and cache reload. Each provider invocation/validation retry +owns a separate trace; a fresh or resumed call does not inherit or union tool +evidence from an earlier invocation. Pi itself still does not support provider +session resume. Mutable trace snapshots are copied at response and lifecycle +boundaries. + +This is collection-only telemetry. A successful call is not proof that the +reviewer read the entire file, understood it, or meaningfully inspected it. +The existing diff gate, symlink inspection contract, and coverage/approval +behavior are unchanged. The separate structured model-output schema remains +version `1`; only durable task metadata advances to version `4`. Per-file +reconciliation remains a separate opt-in follow-up; it must account for missing +or truncated evidence and distinguish assigned body inspection, relocation +impact assessment, and supporting context. `coverage.json` remains the existing +coverage artifact; the trace does not create a competing coverage format. + ## Relocation and Context Coverage Reviewer structured output may include `context_files` and an optional @@ -362,3 +466,73 @@ no-run lifecycle mode. Classifier failures are non-blocking: the gate warns and continues with normal review. Successful and failed classifier task metadata still lives under the prospective run artifact root so provider-session resume and local artifact inspection use the same lifecycle shape. + +## Request-Cost Observations (Collection Only) + +Optional `request_cost_evidence` and `request_cost_checkpoint` fields supplement +schema 4 without changing output reuse or reviewer-tool coverage semantics. Only +the concrete direct OpenAI Responses adapter produces v1 request observations. +Unknown or malformed optional cost data is isolated in both metadata readers; +valid output and reviewer-tool evidence remain available. Schemas 1/2/3 retain +the existing rejection behavior. This change does not enable cost estimation. + +Each observed adapter attempt, including transient retries and structured-output +corrections, gets a distinct ordinal. Usage and observed response identity are +captured independently of accepted output, including no-text, incomplete, +non-2xx and cancellation paths. Missing/null values remain unknown. Requested +and observed model/service tier are separate. Invalid counters are diagnostic +only. No prompt, raw output, refusal/error body, URL, header or credential is +copied into these observations. + +The adapter-owned OpenAI client uses a fresh pinned standard HTTP transport, +disables redirects and clears the replay body. Caller-supplied clients remain +opaque. Only the actual canonical global Responses destination qualifies as +`official_global`; gateway/custom routes remain unverified. Entering `Client.Do` +means `possibly_dispatched`, including network errors; it does not prove a free +request. A pre-dispatch failure is recorded separately. + +The lifecycle holds the same fail-fast advisory task lock across cache checking, +checkpoint updates, invocation and metadata, including no-run mode. Reset uses +that lock too. The bounded sidecar lives at +`llm-tasks/.request-cost/.json`, outside the task directory removed +by fingerprint resets and inside the same root covered by retention/purge. +Before dispatch it atomically records a pending generation. After return it +writes the final checkpoint before ledger insertion or task output/metadata. +If the pending write fails, no adapter call starts. A failed final write leaves +the on-disk pending marker as unknown. Cache loads never create generations or +ledger rows. Reset reconciles metadata observations with the sidecar before +removing metadata, retaining a known prefix even if the sidecar was lost. + +Histories retain at most 64 attempts and 64 finalized generation descriptors, +16 fixed gap reasons and 256 KiB encoded checkpoints. A 4 KiB reserve keeps room +for pending/unknown state even at capacity. Prefixes, saturating omission counts +and gaps survive later success. The whole legacy metadata read is still +unbounded; only its isolated cost envelope and the sidecar reads are bounded. + +Task-history completeness remains false: this slice does not establish a fresh +artifact-root accounting origin. An absent directory, empty ledger, new session +ID or successful latest request cannot establish it. Interrupted, legacy, +opaque, conflicting, missing and overflowing observations remain unknown. +Primary and repair histories remain separate detached values. Direct OpenAI +costs are not passed to the old aggregate estimator; Codex cannot borrow direct +API pricing. Existing compatibility token counters and reported-cost semantics +are not redefined as sums of the new request history. + +Evidence and checkpoints have fixed-order, map-free compact JSON SHA-256 +projections with explicit nulls, normalized empty arrays, sorted gap enums and +separate versioned digest domains. They bind scope, ordered attempts, generation +fingerprints, pending state and omission counts. Prices are excluded. + +Two writes per generation do not preserve every pre-crash observation: a crash +during an invocation may lose that generation's already observed prefix. Its +pending marker makes the missing history explicit. The repository's atomic +writer uses rename without file/directory fsync; no power-loss durability or +provider-invoice completeness is claimed. Per-attempt durable checkpoints, +fresh-origin initialization, request-local pricing and pricing receipts require +separate implementation and verification before estimate activation. + +Cost-bearing metadata uses compact JSON so its durable nested-envelope size +matches the collector/reader admission budget; unrelated legacy metadata keeps +its existing formatting. Recovered metadata-only generations contribute a known +lower-bound count, with missing finalized descriptors recorded as omissions and +gaps. Their fingerprints are not reconstructed by guessing. diff --git a/docs/model-catalog-plan.md b/docs/model-catalog-plan.md index ce7f0629..8afe3c2a 100644 --- a/docs/model-catalog-plan.md +++ b/docs/model-catalog-plan.md @@ -56,3 +56,55 @@ build's own test hook passed. OpenAI context-banded estimates remain unknown when usage metadata cannot identify the applicable band; the published rows are still visible through `catalog show`. Fast and ultrafast rows describe verified capabilities, while only adapter mappings can consume them. + +## Request-pricing schema and activation gate + +Catalog schema 2 appends five pricing CSV columns, in order: `runtime_id`, +`observed_service_tiers`, `input_tokens_min_inclusive`, +`input_tokens_max_exclusive`, and `rate_application`. The schema-1 reader keeps +its original exact 11-column header and produces no request bindings. Schema 2 +requires the exact extended header, with no leading, trailing or quoted name +whitespace. Schema-1 historical header/cell handling is preserved. Data cells +keep their established whitespace trimming in either schema; header whitespace +must never erase populated bindings or bounds. Old binaries reject schema 2; coordinate +published update-catalog rollout with binary rollout. A failed explicit update +keeps the previously installed snapshot and does not silently fall back. + +All five metadata cells blank or null means legacy/display-only. A complete +binding requires a supported runtime, exact observed tier spellings, a +nonnegative decimal int64 minimum, and `whole_request`; an absent maximum means +unbounded only within that complete binding. The short and long rows must form +matching `[0,B)` and `[B,unbounded)` intervals, or an all row must span +`[0,unbounded)`. Every supported raw-tier identity must be unambiguous. Bounds +and rates are row data, including for explicit custom catalogs; the loader does +not impose a provider-wide threshold or rate multiplier. + +The reviewed initial bindings cover 24 existing rows for GPT-6.1 Sol, GPT-6 +Astra/Sol/Luna, and GPT-5.6 Sol/Terra/Luna. Their short band includes 272000 +inclusive input tokens and their long band starts at 272001. Rates are consumed +literally from the selected row. Standard binds only observed `default`, Fast +binds observed `fast` or `priority`, and Astra Ultrafast binds `ultrafast`. +Unrecognized, omitted, requested-only or differently capitalized tiers do not +match. GPT-5.4/5.5 remain display-only pending clarification of their documented +full-session billing unit. Claude all-context rows remain legacy records. + +`Catalog.PriceForRequest` uses the exact observed model and raw service tier, +runtime, and one request's inclusive input count. It returns a detached row and +catalog schema/revision/digest, never an aggregate estimate. `Catalog.Digest` +is the full SHA-256 of the exact five-file snapshot, including local snapshots +without manifest checksums; the human revision alone is not sufficient identity. +All price accessors deep-copy bindings, tiers, numeric bounds and rate pointers. +`catalog show --json` exposes those binding semantics and the full digest. + +No production estimator is activated by metadata. Legacy aggregate Usage cannot +consume request-bound rows even when `context_band=all`. A later request-cost +integration must independently establish official global OpenAI API endpoint +scope, known input/cache/output categories, every dispatch (including retries), +complete history and a persisted evidence/catalog basis. Custom gateways, +regional endpoints, subscription runtimes and aggregate-only old records cannot +inherit these bindings. Selection does not validate model context capacity or +include non-token tool charges. A future result is a public list-price token +estimate, not an invoice total. + +Source and test changes for this schema are subject to separate runtime +verification; the earlier executed evidence above does not cover this slice. diff --git a/internal/app/runtime_test.go b/internal/app/runtime_test.go index 14ba4306..5d2a8880 100644 --- a/internal/app/runtime_test.go +++ b/internal/app/runtime_test.go @@ -1176,7 +1176,7 @@ func runtimeCatalogFixture(t *testing.T) (*modelcatalog.Catalog, string) { } files["models.csv"] = append(files["models.csv"], []byte("openai-api-key,custom-review-model,low|medium,Y,,,https://example.invalid/custom-review-model,2026-10-03\n")...) files["defaults.csv"] = []byte(strings.Replace(string(files["defaults.csv"]), "openai-api-key,medium,gpt-6.1-sol,low,", "openai-api-key,medium,custom-review-model,medium,", 1)) - files["pricing.csv"] = append(files["pricing.csv"], []byte("custom-review-model,standard,all,3,7,1,0.5,,,https://example.invalid/custom-review-model,2026-10-03\ncustom-review-model,fast,all,4,8,1,0.5,,,https://example.invalid/custom-review-model,2026-10-03\n")...) + files["pricing.csv"] = append(files["pricing.csv"], []byte("custom-review-model,standard,all,3,7,1,0.5,,,https://example.invalid/custom-review-model,2026-10-03,,,,,\ncustom-review-model,fast,all,4,8,1,0.5,,,https://example.invalid/custom-review-model,2026-10-03,,,,,\n")...) for name, body := range files { if err := os.WriteFile(filepath.Join(source, name), body, 0o600); err != nil { // #nosec G703 -- source is under t.TempDir. t.Fatalf("write catalog %s: %v", name, err) diff --git a/internal/benchmark/discussion_test.go b/internal/benchmark/discussion_test.go new file mode 100644 index 00000000..f26a4034 --- /dev/null +++ b/internal/benchmark/discussion_test.go @@ -0,0 +1,69 @@ +package benchmark + +import ( + "errors" + "strings" + "testing" +) + +func TestCaseEffectiveWithoutDiscussion(t *testing.T) { + for _, tt := range []struct { + name string + setting string + pinned bool + want bool + }{ + {name: "pinned defaults isolated", pinned: true, want: true}, + {name: "unpinned defaults normal"}, + {name: "pinned explicit isolated", pinned: true, setting: "true", want: true}, + {name: "pinned explicit opt out", pinned: true, setting: "false"}, + {name: "unpinned explicit normal", setting: "false"}, + } { + t.Run(tt.name, func(t *testing.T) { + body := validSuiteYAML() + if !tt.pinned { + body = strings.Replace(body, " review_base_sha: 1111111\n review_head_sha: 2222222\n", "", 1) + } + if tt.setting != "" { + body = strings.Replace(body, " - id: case1\n", " - id: case1\n without_discussion: "+tt.setting+"\n", 1) + } + suite := loadSuite(t, body) + if err := Validate(suite, testConfig()); err != nil { + t.Fatalf("Validate: %v", err) + } + if got := suite.Cases[0].EffectiveWithoutDiscussion(); got != tt.want { + t.Fatalf("EffectiveWithoutDiscussion = %t, want %t", got, tt.want) + } + if (suite.Cases[0].WithoutDiscussion == nil) != (tt.setting == "") { + t.Fatal("setting presence did not survive YAML loading") + } + }) + } +} + +func TestValidateRejectsWithoutDiscussionForUnpinnedCase(t *testing.T) { + body := strings.Replace(validSuiteYAML(), " review_base_sha: 1111111\n review_head_sha: 2222222\n", " without_discussion: true\n", 1) + err := Validate(loadSuite(t, body), testConfig()) + if !errors.Is(err, ErrInvalid) || !strings.Contains(err.Error(), "without_discussion requires review_base_sha and review_head_sha") { + t.Fatalf("Validate error = %v, want pinned requirement", err) + } +} + +func TestLoadRejectsNonBooleanWithoutDiscussion(t *testing.T) { + for _, value := range []string{"null", "", "\"true\"", "1", "[]", "{}"} { + t.Run(value, func(t *testing.T) { + body := strings.Replace(validSuiteYAML(), " - id: case1\n", " - id: case1\n without_discussion: "+value+"\n", 1) + _, err := Load([]byte(body)) + if !errors.Is(err, ErrInvalid) || !strings.Contains(err.Error(), "without_discussion") && !strings.Contains(err.Error(), "bool") { + t.Fatalf("Load error = %v, want invalid boolean", err) + } + }) + } +} + +func TestExpectedSHAsDoNotEnableWithoutDiscussion(t *testing.T) { + benchCase := Case{ExpectedBaseSHA: "1111111", ExpectedHeadSHA: "2222222"} + if benchCase.EffectiveWithoutDiscussion() { + t.Fatal("expected SHA metadata enabled discussion isolation") + } +} diff --git a/internal/benchmark/suite.go b/internal/benchmark/suite.go index b4805115..5ad3db29 100644 --- a/internal/benchmark/suite.go +++ b/internal/benchmark/suite.go @@ -198,13 +198,14 @@ func (s *ReviewerStage) UnmarshalYAML(value *yaml.Node) error { // Case is one pull request to review during a benchmark. type Case struct { - ID string `yaml:"id" json:"id"` - PR string `yaml:"pr" json:"pr"` - ReviewBaseSHA string `yaml:"review_base_sha,omitempty" json:"review_base_sha,omitempty"` - ReviewHeadSHA string `yaml:"review_head_sha,omitempty" json:"review_head_sha,omitempty"` - ExpectedBaseSHA string `yaml:"expected_base_sha,omitempty" json:"expected_base_sha,omitempty"` - ExpectedHeadSHA string `yaml:"expected_head_sha,omitempty" json:"expected_head_sha,omitempty"` - Anchors []Anchor `yaml:"anchors,omitempty" json:"anchors,omitempty"` + ID string `yaml:"id" json:"id"` + PR string `yaml:"pr" json:"pr"` + ReviewBaseSHA string `yaml:"review_base_sha,omitempty" json:"review_base_sha,omitempty"` + ReviewHeadSHA string `yaml:"review_head_sha,omitempty" json:"review_head_sha,omitempty"` + WithoutDiscussion *bool `yaml:"without_discussion,omitempty" json:"without_discussion,omitempty"` + ExpectedBaseSHA string `yaml:"expected_base_sha,omitempty" json:"expected_base_sha,omitempty"` + ExpectedHeadSHA string `yaml:"expected_head_sha,omitempty" json:"expected_head_sha,omitempty"` + Anchors []Anchor `yaml:"anchors,omitempty" json:"anchors,omitempty"` reviewBaseSHASet bool reviewHeadSHASet bool @@ -212,27 +213,41 @@ type Case struct { expectedHeadSHASet bool } +// EffectiveWithoutDiscussion defaults pinned cases to discussion-free reviews. +// A pointer preserves the distinction between an omitted setting and false. +func (c Case) EffectiveWithoutDiscussion() bool { + if c.WithoutDiscussion != nil { + return *c.WithoutDiscussion + } + return c.ReviewBaseSHA != "" && c.ReviewHeadSHA != "" +} + // UnmarshalYAML tracks optional SHA field presence so explicitly blank values // can be rejected after normalization. func (c *Case) UnmarshalYAML(value *yaml.Node) error { type rawCase struct { - ID string `yaml:"id"` - PR string `yaml:"pr"` - ReviewBaseSHA string `yaml:"review_base_sha"` - ReviewHeadSHA string `yaml:"review_head_sha"` - ExpectedBaseSHA string `yaml:"expected_base_sha"` - ExpectedHeadSHA string `yaml:"expected_head_sha"` - Anchors []Anchor `yaml:"anchors"` + ID string `yaml:"id"` + PR string `yaml:"pr"` + ReviewBaseSHA string `yaml:"review_base_sha"` + ReviewHeadSHA string `yaml:"review_head_sha"` + WithoutDiscussion *bool `yaml:"without_discussion"` + ExpectedBaseSHA string `yaml:"expected_base_sha"` + ExpectedHeadSHA string `yaml:"expected_head_sha"` + Anchors []Anchor `yaml:"anchors"` } var raw rawCase if err := value.Decode(&raw); err != nil { return fmt.Errorf("%w: decode case: %w", ErrInvalid, err) } + if mappingHasKey(value, "without_discussion") && raw.WithoutDiscussion == nil { + return fmt.Errorf("%w: case without_discussion must be a boolean when present", ErrInvalid) + } *c = Case{ ID: raw.ID, PR: raw.PR, ReviewBaseSHA: raw.ReviewBaseSHA, ReviewHeadSHA: raw.ReviewHeadSHA, + WithoutDiscussion: raw.WithoutDiscussion, ExpectedBaseSHA: raw.ExpectedBaseSHA, ExpectedHeadSHA: raw.ExpectedHeadSHA, Anchors: raw.Anchors, @@ -648,6 +663,9 @@ func validateCases(cases []Case) error { if (benchCase.ReviewBaseSHA == "") != (benchCase.ReviewHeadSHA == "") { return fmt.Errorf("%w: case %q review_base_sha and review_head_sha must be set together", ErrInvalid, benchCase.ID) } + if benchCase.EffectiveWithoutDiscussion() && benchCase.ReviewBaseSHA == "" { + return fmt.Errorf("%w: case %q without_discussion requires review_base_sha and review_head_sha", ErrInvalid, benchCase.ID) + } if err := validateOptionalSHA("expected_base_sha", benchCase.ID, benchCase.ExpectedBaseSHA, benchCase.expectedBaseSHASet); err != nil { return err } @@ -820,7 +838,7 @@ func validateKnownFields(doc *yaml.Node) error { for i, benchCase := range node.Content { if err := validateMappingKeys(benchCase, fmt.Sprintf("case[%d]", i), map[string]bool{ "id": true, "pr": true, "review_base_sha": true, "review_head_sha": true, - "expected_base_sha": true, "expected_head_sha": true, "anchors": true, + "expected_base_sha": true, "expected_head_sha": true, "without_discussion": true, "anchors": true, }); err != nil { return err } diff --git a/internal/cmd/benchmarkcmd/benchmarkcmd.go b/internal/cmd/benchmarkcmd/benchmarkcmd.go index 88c9886d..4e7a9c5d 100644 --- a/internal/cmd/benchmarkcmd/benchmarkcmd.go +++ b/internal/cmd/benchmarkcmd/benchmarkcmd.go @@ -77,8 +77,9 @@ type doctorAgentDir struct { } type doctorCase struct { - ID string `json:"id"` - PR string `json:"pr"` + ID string `json:"id"` + PR string `json:"pr"` + WithoutDiscussion bool `json:"without_discussion"` } type stickyWriter struct { @@ -240,7 +241,7 @@ func buildDoctorReport(suite benchmark.SuiteFile, cfg config.File, flags doctorF report.Candidates = append(report.Candidates, out) } for _, benchCase := range selectedCases { - report.Cases = append(report.Cases, doctorCase{ID: benchCase.ID, PR: benchCase.PR}) + report.Cases = append(report.Cases, doctorCase{ID: benchCase.ID, PR: benchCase.PR, WithoutDiscussion: benchCase.EffectiveWithoutDiscussion()}) } if report.Warnings == nil { report.Warnings = []string{} @@ -282,7 +283,7 @@ func renderDoctorText(opts *root.Options, report doctorReport) error { ) } for _, benchCase := range report.Cases { - w.printf("- case %s pr=%s\n", benchCase.ID, benchCase.PR) + w.printf("- case %s pr=%s without_discussion=%t\n", benchCase.ID, benchCase.PR, benchCase.WithoutDiscussion) } if len(report.Warnings) == 0 { w.println("Warnings: 0") diff --git a/internal/cmd/benchmarkcmd/benchmarkcmd_test.go b/internal/cmd/benchmarkcmd/benchmarkcmd_test.go index 6d78eccd..fa9fc65f 100644 --- a/internal/cmd/benchmarkcmd/benchmarkcmd_test.go +++ b/internal/cmd/benchmarkcmd/benchmarkcmd_test.go @@ -591,6 +591,7 @@ func TestRunExecutesSelectedMatrixAndWritesArtifacts(t *testing.T) { "--dry-run", "--json", "--review-base-sha", "1111111", "--review-head-sha", "2222222", + "--without-discussion", "--selection-model", "claude-sonnet-4-6", "--selection-effort", "high", "--selection-prompt", got.SelectedCandidates[0].Stages.Selection.Prompt.Resolved, @@ -603,7 +604,7 @@ func TestRunExecutesSelectedMatrixAndWritesArtifacts(t *testing.T) { if strings.Join(invocations[1].args, "\x00") != strings.Join(wantSecondArgs, "\x00") { t.Fatalf("second args = %#v, want %#v", invocations[1].args, wantSecondArgs) } - if len(got.SelectedCases) != 2 || got.SelectedCases[1].ReviewBaseSHA != "1111111" || got.SelectedCases[1].ReviewHeadSHA != "2222222" { + if len(got.SelectedCases) != 2 || got.SelectedCases[0].WithoutDiscussion || !got.SelectedCases[1].WithoutDiscussion || got.SelectedCases[1].ReviewBaseSHA != "1111111" || got.SelectedCases[1].ReviewHeadSHA != "2222222" { t.Fatalf("selected cases = %#v, want review SHAs on case_two", got.SelectedCases) } wantThirdArgs := []string{ @@ -628,6 +629,7 @@ func TestRunExecutesSelectedMatrixAndWritesArtifacts(t *testing.T) { "--dry-run", "--json", "--review-base-sha", "1111111", "--review-head-sha", "2222222", + "--without-discussion", "--selection-model", "kimi", "--selection-effort", "low", "--reviewer-model", "kimi", @@ -1234,14 +1236,15 @@ func reviewDryRunJSONWithArtifact(t *testing.T, runID, artifactPath string, seve } data, err := json.Marshal(view.ReviewDryRun{ Run: view.ReviewRun{ - RunID: runID, - PRURL: "https://github.com/open-cli-collective/codereview-cli/pull/1", - PRKey: "github.com_open-cli-collective_codereview-cli_1", - PostMode: "dry_run", - Outcome: "dry_run", - ArtifactPath: artifactPath, - BaseSHA: "review-base", - HeadSHA: "review-head", + RunID: runID, + PRURL: "https://github.com/open-cli-collective/codereview-cli/pull/1", + PRKey: "github.com_open-cli-collective_codereview-cli_1", + PostMode: "dry_run", + Outcome: "dry_run", + ArtifactPath: artifactPath, + WithoutDiscussion: true, + BaseSHA: "review-base", + HeadSHA: "review-head", }, Findings: findings, Artifacts: view.ReviewArtifacts{ diff --git a/internal/cmd/benchmarkcmd/compare.go b/internal/cmd/benchmarkcmd/compare.go index 8dedbb5e..162baec4 100644 --- a/internal/cmd/benchmarkcmd/compare.go +++ b/internal/cmd/benchmarkcmd/compare.go @@ -24,18 +24,19 @@ const ( runStatusFailed = "failed" runStatusPartial = "partial" - failureNone = "none" - failureMissingArtifact = "missing_artifact" - failureMissingReviewJSON = "missing_review_json" - failureInvalidReviewJSON = "invalid_review_json" - failureInvalidSelectionJSON = "invalid_selection_json" - failureUsageError = "usage_error" - failureAuthConfigError = "auth_config_error" - failureUpstreamError = "upstream_error" - failureChildProcessError = "child_process_error" - failureChildExitNonzero = "child_exit_nonzero" - failureUnavailable = "unavailable" - failureSelectionError = "selection_error" + failureDiscussionIsolationUnverified = "discussion_isolation_unverified" + failureNone = "none" + failureMissingArtifact = "missing_artifact" + failureMissingReviewJSON = "missing_review_json" + failureInvalidReviewJSON = "invalid_review_json" + failureInvalidSelectionJSON = "invalid_selection_json" + failureUsageError = "usage_error" + failureAuthConfigError = "auth_config_error" + failureUpstreamError = "upstream_error" + failureChildProcessError = "child_process_error" + failureChildExitNonzero = "child_exit_nonzero" + failureUnavailable = "unavailable" + failureSelectionError = "selection_error" placementAnchorHit = "anchor_overlap_hit" placementAnchorMiss = "anchor_overlap_miss" @@ -79,33 +80,35 @@ type comparisonArtifacts struct { } type comparisonRun struct { - RunID string `json:"run_id"` - CandidateID string `json:"candidate_id"` - CaseID string `json:"case_id"` - PRURL string `json:"pr_url"` - RequestedReviewBaseSHA string `json:"requested_review_base_sha,omitempty"` - RequestedReviewHeadSHA string `json:"requested_review_head_sha,omitempty"` - ExpectedBaseSHA string `json:"expected_base_sha,omitempty"` - ExpectedHeadSHA string `json:"expected_head_sha,omitempty"` - ReviewBaseSHA string `json:"review_base_sha,omitempty"` - ReviewHeadSHA string `json:"review_head_sha,omitempty"` - CurrentBaseSHA string `json:"current_base_sha,omitempty"` - CurrentHeadSHA string `json:"current_head_sha,omitempty"` - Status string `json:"status"` - ExitCode int `json:"exit_code"` - RetryCount int `json:"retry_count"` - FailureClassification string `json:"failure_classification"` - FindingCount int `json:"finding_count"` - SeverityCounts map[string]int `json:"severity_counts"` - SelectedAgents []benchmarkSelectedAgent `json:"selected_agents,omitempty"` - ThreadActionCount int `json:"thread_action_count,omitempty"` - DurationMS int64 `json:"duration_ms"` - Usage *benchmark.RunMetrics `json:"usage,omitempty"` - Artifacts comparisonRunArtifacts `json:"artifacts"` - AnchorSummary *anchorSummary `json:"anchor_summary,omitempty"` - AnchorResults []anchorResult `json:"anchor_results,omitempty"` - UnmatchedFindings []unmatchedFinding `json:"unmatched_findings,omitempty"` - Warnings []string `json:"warnings"` + RunID string `json:"run_id"` + CandidateID string `json:"candidate_id"` + CaseID string `json:"case_id"` + PRURL string `json:"pr_url"` + RequestedReviewBaseSHA string `json:"requested_review_base_sha,omitempty"` + RequestedReviewHeadSHA string `json:"requested_review_head_sha,omitempty"` + RequestedWithoutDiscussion bool `json:"requested_without_discussion"` + WithoutDiscussionVerified bool `json:"without_discussion_verified"` + ExpectedBaseSHA string `json:"expected_base_sha,omitempty"` + ExpectedHeadSHA string `json:"expected_head_sha,omitempty"` + ReviewBaseSHA string `json:"review_base_sha,omitempty"` + ReviewHeadSHA string `json:"review_head_sha,omitempty"` + CurrentBaseSHA string `json:"current_base_sha,omitempty"` + CurrentHeadSHA string `json:"current_head_sha,omitempty"` + Status string `json:"status"` + ExitCode int `json:"exit_code"` + RetryCount int `json:"retry_count"` + FailureClassification string `json:"failure_classification"` + FindingCount int `json:"finding_count"` + SeverityCounts map[string]int `json:"severity_counts"` + SelectedAgents []benchmarkSelectedAgent `json:"selected_agents,omitempty"` + ThreadActionCount int `json:"thread_action_count,omitempty"` + DurationMS int64 `json:"duration_ms"` + Usage *benchmark.RunMetrics `json:"usage,omitempty"` + Artifacts comparisonRunArtifacts `json:"artifacts"` + AnchorSummary *anchorSummary `json:"anchor_summary,omitempty"` + AnchorResults []anchorResult `json:"anchor_results,omitempty"` + UnmatchedFindings []unmatchedFinding `json:"unmatched_findings,omitempty"` + Warnings []string `json:"warnings"` } type comparisonRunArtifacts struct { @@ -294,27 +297,30 @@ func buildComparison(summary benchmarkSuiteSummary, resultsDir string) compariso for _, run := range summary.Runs { benchCase := casesByID[run.CaseID] read := readReviewArtifactForCompare(resultsDir, run) + reconcileDiscussionIsolationForCompare(&run, read) row := comparisonRun{ - RunID: run.RunID, - CandidateID: run.CandidateID, - CaseID: run.CaseID, - PRURL: run.PRURL, - RequestedReviewBaseSHA: run.RequestedReviewBaseSHA, - RequestedReviewHeadSHA: run.RequestedReviewHeadSHA, - ExpectedBaseSHA: run.ExpectedBaseSHA, - ExpectedHeadSHA: run.ExpectedHeadSHA, - ReviewBaseSHA: run.ReviewBaseSHA, - ReviewHeadSHA: run.ReviewHeadSHA, - CurrentBaseSHA: run.CurrentBaseSHA, - CurrentHeadSHA: run.CurrentHeadSHA, - Status: compareRunStatus(run, read.ok), - ExitCode: run.ExitCode, - RetryCount: run.RetryCount, - FailureClassification: compareFailureClassification(run, read), - FindingCount: run.FindingCount, - SeverityCounts: copySeverityCounts(run.SeverityCounts), - DurationMS: run.DurationMS, - Usage: run.Usage, + RunID: run.RunID, + CandidateID: run.CandidateID, + CaseID: run.CaseID, + PRURL: run.PRURL, + RequestedReviewBaseSHA: run.RequestedReviewBaseSHA, + RequestedReviewHeadSHA: run.RequestedReviewHeadSHA, + RequestedWithoutDiscussion: run.RequestedWithoutDiscussion, + WithoutDiscussionVerified: run.WithoutDiscussionVerified, + ExpectedBaseSHA: run.ExpectedBaseSHA, + ExpectedHeadSHA: run.ExpectedHeadSHA, + ReviewBaseSHA: run.ReviewBaseSHA, + ReviewHeadSHA: run.ReviewHeadSHA, + CurrentBaseSHA: run.CurrentBaseSHA, + CurrentHeadSHA: run.CurrentHeadSHA, + Status: compareRunStatus(run, read.ok), + ExitCode: run.ExitCode, + RetryCount: run.RetryCount, + FailureClassification: compareFailureClassification(run, read), + FindingCount: run.FindingCount, + SeverityCounts: copySeverityCounts(run.SeverityCounts), + DurationMS: run.DurationMS, + Usage: run.Usage, Artifacts: comparisonRunArtifacts{ ReviewJSON: run.Artifacts.ReviewJSON, SelectionJSON: run.Artifacts.SelectionJSON, @@ -327,6 +333,9 @@ func buildComparison(summary benchmarkSuiteSummary, resultsDir string) compariso Warnings: appendUniqueWarnings(nil, run.Warnings...), } row.Warnings = appendUniqueWarnings(row.Warnings, read.warnings...) + if run.RequestedWithoutDiscussion && !run.WithoutDiscussionVerified { + row.Warnings = appendUniqueWarnings(row.Warnings, "discussion isolation could not be verified from the saved execution and current review JSON") + } if len(benchCase.Anchors) > 0 { if read.parsed != nil { row.AnchorSummary, row.AnchorResults, row.UnmatchedFindings = compareAnchors(benchCase.Anchors, read.parsed.Findings) @@ -513,6 +522,16 @@ func benchmarkArtifactPathInResultsDir(resultsDir, artifactPath string) (string, return abs, true, "" } +// Comparison rechecks the current review artifact instead of trusting a saved +// verification bit. A changed artifact can invalidate prior verification, but +// cannot retroactively validate an execution that was originally unverified. +func reconcileDiscussionIsolationForCompare(run *benchmarkRun, read reviewArtifactRead) { + run.WithoutDiscussionVerified = run.RequestedWithoutDiscussion && run.WithoutDiscussionVerified && read.ok && read.parsed != nil && read.parsed.Run.WithoutDiscussion + if run.RequestedWithoutDiscussion && !run.WithoutDiscussionVerified && read.ok && (run.FailureClassification == "" || run.FailureClassification == failureNone) { + run.FailureClassification = failureDiscussionIsolationUnverified + } +} + func compareFailureClassification(run benchmarkRun, read reviewArtifactRead) string { if run.FailureClassification != "" && run.FailureClassification != failureNone { return run.FailureClassification @@ -538,7 +557,7 @@ func compareFailureClassification(run benchmarkRun, read reviewArtifactRead) str } func compareRunStatus(run benchmarkRun, reviewJSONOK bool) string { - if run.ExitCode != exitcode.Success { + if run.ExitCode != exitcode.Success || run.RequestedWithoutDiscussion && !run.WithoutDiscussionVerified { return runStatusFailed } if reviewJSONOK { diff --git a/internal/cmd/benchmarkcmd/discussion_relocation_test.go b/internal/cmd/benchmarkcmd/discussion_relocation_test.go new file mode 100644 index 00000000..75a88e60 --- /dev/null +++ b/internal/cmd/benchmarkcmd/discussion_relocation_test.go @@ -0,0 +1,76 @@ +package benchmarkcmd + +import ( + "context" + "encoding/json" + "path/filepath" + "testing" + + "github.com/open-cli-collective/codereview-cli/internal/app" + "github.com/open-cli-collective/codereview-cli/internal/benchmark" + "github.com/open-cli-collective/codereview-cli/internal/cmd/root" + "github.com/open-cli-collective/codereview-cli/internal/gitprovider" + "github.com/open-cli-collective/codereview-cli/internal/ledger" + "github.com/open-cli-collective/codereview-cli/internal/pipeline" + "github.com/open-cli-collective/codereview-cli/internal/view" +) + +// Exercise the benchmark-to-view boundary with the union of replay and +// relocation artifact fields. The injected runner makes no provider/LLM calls. +func TestPinnedDiscussionFreeBenchmarkPreservesRelocationArtifactPaths(t *testing.T) { + resultsDir := t.TempDir() + pipelineDir := t.TempDir() + paths := pipeline.ArtifactPaths{ + Dir: pipelineDir, + RelocationsJSON: filepath.Join(pipelineDir, "relocations.json"), + CoverageJSON: filepath.Join(pipelineDir, "coverage.json"), + } + candidate := benchmark.Candidate{ID: "candidate", Profile: "home"} + benchCase := benchmark.Case{ + ID: "pinned", + PR: "https://github.com/open-cli-collective/codereview-cli/pull/1", + ReviewBaseSHA: "1111111", + ReviewHeadSHA: "2222222", + } + executor := inProcessExecutor{ + cfg: testConfig(), + open: func(context.Context, app.OpenRequest) (app.Runtime, error) { + return app.Runtime{Runner: benchmarkTestRunner{dryRun: func(_ context.Context, req pipeline.Request) (pipeline.Result, error) { + if !req.WithoutDiscussion || req.ReviewBaseSHA != benchCase.ReviewBaseSHA || req.ReviewHeadSHA != benchCase.ReviewHeadSHA { + t.Fatalf("pipeline request = %#v, want pinned discussion-free recipe", req) + } + return pipeline.Result{ + Run: ledger.Run{RunID: "pinned-relocation", ArtifactPath: pipelineDir}, + PR: gitprovider.PR{URL: req.PRURL}, + Artifacts: paths, + ReviewBaseSHA: req.ReviewBaseSHA, + ReviewHeadSHA: req.ReviewHeadSHA, + WithoutDiscussion: req.WithoutDiscussion, + }, nil + }}}, nil + }, + } + run, err := executeBenchmarkRun(context.Background(), root.NewProgressLogger(nil), resultsDir, resultsDir, benchmarkInProcessCRBin, "run", candidate, benchCase, executor) + if err != nil { + t.Fatalf("executeBenchmarkRun: %v", err) + } + if !run.RequestedWithoutDiscussion || !run.WithoutDiscussionVerified || run.FailureClassification != failureNone { + t.Fatalf("run = %#v, want verified discussion-free execution", run) + } + var review view.ReviewDryRun + if err := json.Unmarshal([]byte(discussionReadFileString(t, resultsDir, run.Artifacts.ReviewJSON)), &review); err != nil { + t.Fatalf("review JSON: %v", err) + } + if !review.Run.WithoutDiscussion || review.Run.BaseSHA != benchCase.ReviewBaseSHA || review.Run.HeadSHA != benchCase.ReviewHeadSHA || review.Artifacts.RelocationsJSON != paths.RelocationsJSON || review.Artifacts.CoverageJSON != paths.CoverageJSON { + t.Fatalf("review = %#v, want pinned replay marker and relocation/coverage paths together", review) + } + comparison := buildComparison(benchmarkSuiteSummary{ + SuiteID: "pinned-relocation", + SelectedCandidates: summarizeCandidates(resultsDir, []benchmark.Candidate{candidate}), + SelectedCases: summarizeCases([]benchmark.Case{benchCase}), + Runs: []benchmarkRun{run}, + }, resultsDir) + if !comparison.Runs[0].WithoutDiscussionVerified || comparison.Runs[0].Status != runStatusCompleted { + t.Fatalf("comparison = %#v, want current artifact to verify discussion isolation", comparison.Runs[0]) + } +} diff --git a/internal/cmd/benchmarkcmd/discussion_test.go b/internal/cmd/benchmarkcmd/discussion_test.go new file mode 100644 index 00000000..04c5af18 --- /dev/null +++ b/internal/cmd/benchmarkcmd/discussion_test.go @@ -0,0 +1,252 @@ +package benchmarkcmd + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/open-cli-collective/codereview-cli/internal/app" + "github.com/open-cli-collective/codereview-cli/internal/benchmark" + "github.com/open-cli-collective/codereview-cli/internal/cmd/exitcode" + "github.com/open-cli-collective/codereview-cli/internal/cmd/root" + "github.com/open-cli-collective/codereview-cli/internal/ledger" + "github.com/open-cli-collective/codereview-cli/internal/pipeline" + "github.com/open-cli-collective/codereview-cli/internal/view" +) + +func TestReviewArgsPassesEffectiveDiscussionIsolation(t *testing.T) { + optOut := false + for _, tt := range []struct { + name string + benchCase benchmark.Case + want bool + }{ + {name: "unpinned"}, + {name: "pinned", benchCase: benchmark.Case{ReviewBaseSHA: "1111111", ReviewHeadSHA: "2222222"}, want: true}, + {name: "pinned opt out", benchCase: benchmark.Case{ReviewBaseSHA: "1111111", ReviewHeadSHA: "2222222", WithoutDiscussion: &optOut}}, + {name: "expected only", benchCase: benchmark.Case{ExpectedBaseSHA: "1111111", ExpectedHeadSHA: "2222222"}}, + } { + t.Run(tt.name, func(t *testing.T) { + args := reviewArgs(t.TempDir(), benchmark.Candidate{Profile: "home"}, tt.benchCase) + if got := stringSliceContains(args, "--without-discussion"); got != tt.want { + t.Fatalf("args = %#v, want --without-discussion=%t", args, tt.want) + } + }) + } +} + +func TestInProcessExecutorPassesExplicitDiscussionOptOut(t *testing.T) { + optOut := false + var gotRequest pipeline.Request + executor := inProcessExecutor{ + cfg: testConfig(), + open: func(context.Context, app.OpenRequest) (app.Runtime, error) { + return app.Runtime{Runner: benchmarkTestRunner{dryRun: func(_ context.Context, req pipeline.Request) (pipeline.Result, error) { + gotRequest = req + return pipeline.Result{Run: ledger.Run{RunID: "opt-out", ArtifactPath: t.TempDir()}}, nil + }}}, nil + }, + } + result := executor.Execute(context.Background(), reviewExecutionRequest{ + Candidate: benchmark.Candidate{Profile: "home"}, + Case: benchmark.Case{PR: "https://github.com/open-cli-collective/codereview-cli/pull/1", ReviewBaseSHA: "1111111", ReviewHeadSHA: "2222222", WithoutDiscussion: &optOut}, + }) + if result.Err != nil || gotRequest.WithoutDiscussion { + t.Fatalf("Execute = %#v, request = %#v, want explicit opt out", result, gotRequest) + } +} + +func TestBenchmarkDiscussionIsolationCompatibility(t *testing.T) { + optOut := false + for _, tt := range []struct { + name string + stdout string + exitCode int + optOut *bool + wantClass string + wantVerified bool + }{ + {name: "supported", stdout: `{"run":{"without_discussion":true}}`, wantClass: failureNone, wantVerified: true}, + {name: "unknown old binary", stdout: `{"run":{"run_id":"old"}}`, wantClass: failureDiscussionIsolationUnverified}, + {name: "false marker", stdout: `{"run":{"without_discussion":false}}`, wantClass: failureDiscussionIsolationUnverified}, + {name: "null marker", stdout: `{"run":{"without_discussion":null}}`, wantClass: failureDiscussionIsolationUnverified}, + {name: "missing JSON", wantClass: failureMissingReviewJSON}, + {name: "invalid JSON", stdout: `{broken`, wantClass: failureInvalidReviewJSON}, + {name: "unsupported flag", exitCode: exitcode.UsageError, wantClass: failureUsageError}, + {name: "explicit old binary opt out", stdout: `{"run":{"run_id":"old"}}`, optOut: &optOut, wantClass: failureNone}, + } { + t.Run(tt.name, func(t *testing.T) { + calls := 0 + benchCase := benchmark.Case{ID: "pinned", PR: "https://github.com/open-cli-collective/codereview-cli/pull/1", ReviewBaseSHA: "1111111", ReviewHeadSHA: "2222222", WithoutDiscussion: tt.optOut} + executor := subprocessExecutor{run: func(_ context.Context, _ string, args []string) reviewCommandResult { + calls++ + if got := stringSliceContains(args, "--without-discussion"); got != benchCase.EffectiveWithoutDiscussion() { + t.Fatalf("args = %#v, want effective discussion flag", args) + } + return reviewCommandResult{Stdout: []byte(tt.stdout), Stderr: []byte("original stderr\n"), ExitCode: tt.exitCode} + }} + resultsDir := t.TempDir() + run, err := executeBenchmarkRun(context.Background(), root.NewProgressLogger(&root.Options{}), resultsDir, resultsDir, "fake-cr", "run", benchmark.Candidate{ID: "candidate", Profile: "home"}, benchCase, executor) + if err != nil { + t.Fatalf("executeBenchmarkRun: %v", err) + } + if calls != 1 || run.ExitCode != tt.exitCode || run.FailureClassification != tt.wantClass || run.WithoutDiscussionVerified != tt.wantVerified || run.RequestedWithoutDiscussion != benchCase.EffectiveWithoutDiscussion() { + t.Fatalf("calls=%d run=%#v, want class=%s verified=%t actual child exit=%d", calls, run, tt.wantClass, tt.wantVerified, tt.exitCode) + } + if tt.wantClass == failureDiscussionIsolationUnverified { + if compareRunStatus(run, true) != runStatusFailed || !strings.Contains(strings.Join(run.Warnings, "\n"), "could not be verified") { + t.Fatalf("run=%#v, want failed comparison and explicit capability warning", run) + } + } + if got := discussionReadFileString(t, resultsDir, run.Artifacts.ReviewJSON); got != tt.stdout { + t.Fatalf("review stdout changed: %q", got) + } + if got := discussionReadFileString(t, resultsDir, run.Artifacts.Stderr); got != "original stderr\n" { + t.Fatalf("review stderr changed: %q", got) + } + }) + } +} + +func TestRunCountsUnverifiedIsolationAsFailureAndPreservesProvenance(t *testing.T) { + cmd, out := newTestCommand(t) + suitePath := writeBenchmarkSuite(t, validBenchmarkSuite(t)) + withBenchmarkRunSeams(t, fixedBenchmarkTime(), func(context.Context, string, []string) reviewCommandResult { + return reviewCommandResult{Stdout: []byte(`{"run":{"run_id":"old"}}`), ExitCode: exitcode.Success} + }) + resultsDir := filepath.Join(t.TempDir(), "results") + if err := root.Execute(cmd, []string{"benchmark", "run", suitePath, "--case", "case_two", "--candidate", "first", "--in-process", "--results-dir", resultsDir, "--json"}); err != nil { + t.Fatalf("Execute: %v", err) + } + var summary benchmarkSuiteSummary + if err := json.Unmarshal(out.Bytes(), &summary); err != nil { + t.Fatalf("summary JSON: %v", err) + } + if summary.SuccessCount != 0 || summary.FailureCount != 1 || !summary.SelectedCases[0].WithoutDiscussion || !summary.Runs[0].RequestedWithoutDiscussion || summary.Runs[0].WithoutDiscussionVerified { + t.Fatalf("summary=%#v, want requested but unverified isolation counted as failure", summary) + } + var manifest benchmarkManifest + if err := json.Unmarshal([]byte(discussionReadFileString(t, resultsDir, summary.Artifacts.Manifest)), &manifest); err != nil { + t.Fatalf("manifest JSON: %v", err) + } + if !manifest.SelectedCases[0].WithoutDiscussion || !manifest.Runs[0].RequestedWithoutDiscussion || manifest.Runs[0].WithoutDiscussionVerified { + t.Fatalf("manifest=%#v, want requested/verified policy", manifest) + } + comparison := buildComparison(summary, summary.ResultsDir) + if !comparison.Runs[0].RequestedWithoutDiscussion || comparison.Runs[0].WithoutDiscussionVerified || comparison.Runs[0].Status != runStatusFailed { + t.Fatalf("comparison=%#v, want unverified isolation failure", comparison.Runs[0]) + } +} + +func discussionReadFileString(t *testing.T, resultsDir, path string) string { + t.Helper() + fixture, err := os.OpenRoot(resultsDir) + if err != nil { + t.Fatalf("OpenRoot %s: %v", resultsDir, err) + } + defer fixture.Close() + relative, err := filepath.Rel(resultsDir, path) + if err != nil { + t.Fatalf("relative fixture path %s: %v", path, err) + } + data, err := fixture.ReadFile(relative) + if err != nil { + t.Fatalf("ReadFile %s: %v", path, err) + } + return string(data) +} + +func TestCompareRechecksDiscussionIsolationAgainstCurrentArtifact(t *testing.T) { + for _, tt := range []struct { + name string + currentJSON string + missing bool + wantClass string + wantVerified bool + wantStatus string + }{ + {name: "unchanged verified artifact", currentJSON: `{"run":{"without_discussion":true}}`, wantClass: failureNone, wantVerified: true, wantStatus: runStatusCompleted}, + {name: "marker removed", currentJSON: `{"run":{"run_id":"old"}}`, wantClass: failureDiscussionIsolationUnverified, wantStatus: runStatusFailed}, + {name: "marker changed to false", currentJSON: `{"run":{"without_discussion":false}}`, wantClass: failureDiscussionIsolationUnverified, wantStatus: runStatusFailed}, + {name: "marker changed to null", currentJSON: `{"run":{"without_discussion":null}}`, wantClass: failureDiscussionIsolationUnverified, wantStatus: runStatusFailed}, + {name: "invalid JSON", currentJSON: `{broken`, wantClass: failureInvalidReviewJSON, wantStatus: runStatusFailed}, + {name: "empty JSON", wantClass: failureMissingReviewJSON, wantStatus: runStatusFailed}, + {name: "missing artifact", missing: true, wantClass: failureMissingArtifact, wantStatus: runStatusFailed}, + } { + t.Run(tt.name, func(t *testing.T) { + resultsDir := t.TempDir() + summary := comparisonFixtureSummary(resultsDir) + summary.Runs[0].RequestedWithoutDiscussion = true + summary.Runs[0].WithoutDiscussionVerified = true + writeComparisonFixture(t, summary) + artifact := summary.Runs[0].Artifacts.ReviewJSON + writeLog(t, artifact, `{"run":{"without_discussion":true}}`) + writeLog(t, summary.Runs[0].Artifacts.Stderr, "original stderr\n") + initial := buildComparison(summary, resultsDir) + if !initial.Runs[0].WithoutDiscussionVerified || initial.Runs[0].Status != runStatusCompleted { + t.Fatalf("initial comparison = %#v, want verified completed run", initial.Runs[0]) + } + if tt.missing { + if err := os.Remove(artifact); err != nil { + t.Fatalf("Remove review JSON: %v", err) + } + } else { + writeLog(t, artifact, tt.currentJSON) + } + summaryBefore := discussionReadFileString(t, resultsDir, summary.Artifacts.SuiteSummary) + comparison, err := writeComparisonArtifactsForResultsDir(resultsDir) + if err != nil { + t.Fatalf("writeComparisonArtifactsForResultsDir: %v", err) + } + row := comparison.Runs[0] + if row.WithoutDiscussionVerified != tt.wantVerified || row.Status != tt.wantStatus || row.FailureClassification != tt.wantClass || row.ExitCode != exitcode.Success || !row.RequestedWithoutDiscussion { + t.Fatalf("row = %#v, want status=%s class=%s verified=%t actual exit=0", row, tt.wantStatus, tt.wantClass, tt.wantVerified) + } + if !tt.wantVerified && !strings.Contains(strings.Join(row.Warnings, "\n"), "discussion isolation could not be verified") { + t.Fatalf("warnings = %#v, want current-artifact isolation warning", row.Warnings) + } + if got := discussionReadFileString(t, resultsDir, summary.Artifacts.SuiteSummary); got != summaryBefore { + t.Fatal("compare changed the saved suite summary") + } + if got := discussionReadFileString(t, resultsDir, summary.Runs[0].Artifacts.Stderr); got != "original stderr\n" { + t.Fatalf("compare changed raw stderr: %q", got) + } + if !tt.missing { + if got := discussionReadFileString(t, resultsDir, artifact); got != tt.currentJSON { + t.Fatalf("compare changed raw review JSON: %q", got) + } + } + }) + } +} + +func TestCompareDiscussionVerificationCannotUpgradeSavedExecution(t *testing.T) { + resultsDir := t.TempDir() + summary := comparisonFixtureSummary(resultsDir) + summary.Runs[0].RequestedWithoutDiscussion = true + summary.Runs[0].WithoutDiscussionVerified = false + summary.Runs[0].FailureClassification = failureDiscussionIsolationUnverified + writeReviewJSON(t, summary.Runs[0].Artifacts.ReviewJSON, view.ReviewDryRun{Run: view.ReviewRun{WithoutDiscussion: true}}) + comparison := buildComparison(summary, resultsDir) + row := comparison.Runs[0] + if row.WithoutDiscussionVerified || row.Status != runStatusFailed || row.FailureClassification != failureDiscussionIsolationUnverified { + t.Fatalf("row = %#v, want original unverified execution to remain failed", row) + } +} + +func TestCompareDiscussionRecheckPreservesChildFailure(t *testing.T) { + resultsDir := t.TempDir() + summary := comparisonFixtureSummary(resultsDir) + summary.Runs[0].RequestedWithoutDiscussion = true + summary.Runs[0].WithoutDiscussionVerified = true + summary.Runs[0].ExitCode = exitcode.UpstreamError + summary.Runs[0].FailureClassification = failureUpstreamError + writeReviewJSON(t, summary.Runs[0].Artifacts.ReviewJSON, view.ReviewDryRun{}) + row := buildComparison(summary, resultsDir).Runs[0] + if row.WithoutDiscussionVerified || row.Status != runStatusFailed || row.ExitCode != exitcode.UpstreamError || row.FailureClassification != failureUpstreamError { + t.Fatalf("row = %#v, want original child failure/exit with invalidated verification", row) + } +} diff --git a/internal/cmd/benchmarkcmd/executor.go b/internal/cmd/benchmarkcmd/executor.go index b2ccfc8c..9797323c 100644 --- a/internal/cmd/benchmarkcmd/executor.go +++ b/internal/cmd/benchmarkcmd/executor.go @@ -191,6 +191,7 @@ func (e inProcessExecutor) Execute(ctx context.Context, req reviewExecutionReque ReviewerEffortOverride: req.Candidate.Stages.Reviewers.Effort, ReviewBaseSHA: req.Case.ReviewBaseSHA, ReviewHeadSHA: req.Case.ReviewHeadSHA, + WithoutDiscussion: req.Case.EffectiveWithoutDiscussion(), ToolVersion: version.Version, }) if err != nil { diff --git a/internal/cmd/benchmarkcmd/executor_test.go b/internal/cmd/benchmarkcmd/executor_test.go index a20caf50..a7d67fa9 100644 --- a/internal/cmd/benchmarkcmd/executor_test.go +++ b/internal/cmd/benchmarkcmd/executor_test.go @@ -105,8 +105,9 @@ func TestInProcessExecutorOpensAndCleansRuntimePerCell(t *testing.T) { Runner: benchmarkTestRunner{dryRun: func(_ context.Context, req pipeline.Request) (pipeline.Result, error) { pipelineRequests = append(pipelineRequests, req) return pipeline.Result{ - Run: ledger.Run{RunID: runID, ArtifactPath: t.TempDir()}, - PR: gitprovider.PR{URL: req.PRURL}, + Run: ledger.Run{RunID: runID, ArtifactPath: t.TempDir()}, + WithoutDiscussion: req.WithoutDiscussion, + PR: gitprovider.PR{URL: req.PRURL}, }, nil }}, Cleanup: func() { cleanupCount++ }, @@ -153,6 +154,9 @@ func TestInProcessExecutorOpensAndCleansRuntimePerCell(t *testing.T) { if pipelineRequests[1].ReviewBaseSHA != "1111111" || pipelineRequests[1].ReviewHeadSHA != "2222222" { t.Fatalf("second pipeline request = %#v, want case SHAs", pipelineRequests[1]) } + if pipelineRequests[0].WithoutDiscussion || !pipelineRequests[1].WithoutDiscussion { + t.Fatalf("discussion policy = %t/%t, want live:false pinned:true", pipelineRequests[0].WithoutDiscussion, pipelineRequests[1].WithoutDiscussion) + } for _, req := range pipelineRequests { if req.ReviewerEffortOverride != "" { t.Fatalf("reviewer effort override = %q, want inherited effort", req.ReviewerEffortOverride) diff --git a/internal/cmd/benchmarkcmd/run.go b/internal/cmd/benchmarkcmd/run.go index 44421c14..78f3eb4d 100644 --- a/internal/cmd/benchmarkcmd/run.go +++ b/internal/cmd/benchmarkcmd/run.go @@ -89,10 +89,12 @@ type benchmarkManifest struct { } type benchmarkManifestRun struct { - RunID string `json:"run_id"` - CandidateID string `json:"candidate_id"` - CaseID string `json:"case_id"` - Artifacts runArtifacts `json:"artifacts"` + RunID string `json:"run_id"` + CandidateID string `json:"candidate_id"` + CaseID string `json:"case_id"` + RequestedWithoutDiscussion bool `json:"requested_without_discussion"` + WithoutDiscussionVerified bool `json:"without_discussion_verified"` + Artifacts runArtifacts `json:"artifacts"` } type benchmarkCandidate struct { @@ -144,41 +146,44 @@ type benchmarkAgentDir struct { } type benchmarkCase struct { - ID string `json:"id"` - PR string `json:"pr"` - ReviewBaseSHA string `json:"review_base_sha,omitempty"` - ReviewHeadSHA string `json:"review_head_sha,omitempty"` - ExpectedBaseSHA string `json:"expected_base_sha,omitempty"` - ExpectedHeadSHA string `json:"expected_head_sha,omitempty"` - Anchors []benchmark.Anchor `json:"anchors,omitempty"` + ID string `json:"id"` + PR string `json:"pr"` + ReviewBaseSHA string `json:"review_base_sha,omitempty"` + ReviewHeadSHA string `json:"review_head_sha,omitempty"` + WithoutDiscussion bool `json:"without_discussion"` + ExpectedBaseSHA string `json:"expected_base_sha,omitempty"` + ExpectedHeadSHA string `json:"expected_head_sha,omitempty"` + Anchors []benchmark.Anchor `json:"anchors,omitempty"` } type benchmarkRun struct { - RunID string `json:"run_id"` - CandidateID string `json:"candidate_id"` - CaseID string `json:"case_id"` - PRURL string `json:"pr_url"` - RequestedReviewBaseSHA string `json:"requested_review_base_sha,omitempty"` - RequestedReviewHeadSHA string `json:"requested_review_head_sha,omitempty"` - ExpectedBaseSHA string `json:"expected_base_sha,omitempty"` - ExpectedHeadSHA string `json:"expected_head_sha,omitempty"` - ReviewRunID string `json:"review_run_id,omitempty"` - ReviewArtifactPath string `json:"review_artifact_path,omitempty"` - ReviewBaseSHA string `json:"review_base_sha,omitempty"` - ReviewHeadSHA string `json:"review_head_sha,omitempty"` - CurrentBaseSHA string `json:"current_base_sha,omitempty"` - CurrentHeadSHA string `json:"current_head_sha,omitempty"` - ExitCode int `json:"exit_code"` - RetryCount int `json:"retry_count"` - FailureClassification string `json:"failure_classification"` - DurationMS int64 `json:"duration_ms"` - FindingCount int `json:"finding_count"` - SeverityCounts map[string]int `json:"severity_counts"` - SelectedAgents []benchmarkSelectedAgent `json:"selected_agents,omitempty"` - ThreadActionCount int `json:"thread_action_count,omitempty"` - Usage *benchmark.RunMetrics `json:"usage,omitempty"` - Warnings []string `json:"warnings"` - Artifacts runArtifacts `json:"artifacts"` + RunID string `json:"run_id"` + CandidateID string `json:"candidate_id"` + CaseID string `json:"case_id"` + PRURL string `json:"pr_url"` + RequestedReviewBaseSHA string `json:"requested_review_base_sha,omitempty"` + RequestedReviewHeadSHA string `json:"requested_review_head_sha,omitempty"` + RequestedWithoutDiscussion bool `json:"requested_without_discussion"` + WithoutDiscussionVerified bool `json:"without_discussion_verified"` + ExpectedBaseSHA string `json:"expected_base_sha,omitempty"` + ExpectedHeadSHA string `json:"expected_head_sha,omitempty"` + ReviewRunID string `json:"review_run_id,omitempty"` + ReviewArtifactPath string `json:"review_artifact_path,omitempty"` + ReviewBaseSHA string `json:"review_base_sha,omitempty"` + ReviewHeadSHA string `json:"review_head_sha,omitempty"` + CurrentBaseSHA string `json:"current_base_sha,omitempty"` + CurrentHeadSHA string `json:"current_head_sha,omitempty"` + ExitCode int `json:"exit_code"` + RetryCount int `json:"retry_count"` + FailureClassification string `json:"failure_classification"` + DurationMS int64 `json:"duration_ms"` + FindingCount int `json:"finding_count"` + SeverityCounts map[string]int `json:"severity_counts"` + SelectedAgents []benchmarkSelectedAgent `json:"selected_agents,omitempty"` + ThreadActionCount int `json:"thread_action_count,omitempty"` + Usage *benchmark.RunMetrics `json:"usage,omitempty"` + Warnings []string `json:"warnings"` + Artifacts runArtifacts `json:"artifacts"` } type runArtifacts struct { @@ -324,7 +329,7 @@ func runBenchmarkSuite(ctx context.Context, cmd *cobra.Command, opts *root.Optio } summary.Runs = append(summary.Runs, runSummary) summary.RunCount++ - if runSummary.ExitCode == exitcode.Success { + if runSummary.ExitCode == exitcode.Success && (!runSummary.RequestedWithoutDiscussion || runSummary.WithoutDiscussionVerified) { summary.SuccessCount++ } else { summary.FailureCount++ @@ -377,23 +382,26 @@ func executeBenchmarkRun(ctx context.Context, logger *progress.Logger, suiteDir, Candidate: candidate, Case: benchCase, }) + verifyDiscussionIsolation(benchCase, &execution) _ = reviewSpan.End(execution.Err) runSummary := benchmarkRun{ - RunID: runID, - CandidateID: candidate.ID, - CaseID: benchCase.ID, - PRURL: benchCase.PR, - RequestedReviewBaseSHA: benchCase.ReviewBaseSHA, - RequestedReviewHeadSHA: benchCase.ReviewHeadSHA, - ExpectedBaseSHA: benchCase.ExpectedBaseSHA, - ExpectedHeadSHA: benchCase.ExpectedHeadSHA, - ExitCode: execution.ExitCode, - RetryCount: 0, - DurationMS: durationMS(execution.Duration), - SeverityCounts: map[string]int{}, - Warnings: []string{}, - Artifacts: artifacts, + RunID: runID, + CandidateID: candidate.ID, + CaseID: benchCase.ID, + PRURL: benchCase.PR, + RequestedReviewBaseSHA: benchCase.ReviewBaseSHA, + RequestedReviewHeadSHA: benchCase.ReviewHeadSHA, + RequestedWithoutDiscussion: benchCase.EffectiveWithoutDiscussion(), + WithoutDiscussionVerified: benchCase.EffectiveWithoutDiscussion() && execution.Review != nil && execution.Review.Run.WithoutDiscussion, + ExpectedBaseSHA: benchCase.ExpectedBaseSHA, + ExpectedHeadSHA: benchCase.ExpectedHeadSHA, + ExitCode: execution.ExitCode, + RetryCount: 0, + DurationMS: durationMS(execution.Duration), + SeverityCounts: map[string]int{}, + Warnings: []string{}, + Artifacts: artifacts, } if execution.Err != nil && execution.ExitCode < 0 { runSummary.Warnings = append(runSummary.Warnings, fmt.Sprintf("review command failed to start or complete: %s", execution.Err.Error())) @@ -414,9 +422,8 @@ func executeBenchmarkRun(ctx context.Context, logger *progress.Logger, suiteDir, } else if usage.HasData() { runSummary.Usage = &usage } - } else { - runSummary.Warnings = append(runSummary.Warnings, execution.Warnings...) } + runSummary.Warnings = append(runSummary.Warnings, execution.Warnings...) if err := writeArtifactFile(artifacts.ReviewJSON, execution.Stdout); err != nil { return benchmarkRun{}, runSpan.End(err) @@ -431,6 +438,29 @@ func executeBenchmarkRun(ctx context.Context, logger *progress.Logger, suiteDir, return runSummary, nil } +// A successful child exit is not evidence that an external binary honored the +// isolation flag. Require the #627 response marker, retaining raw output and the +// real child exit code when the capability cannot be verified. +func verifyDiscussionIsolation(benchCase benchmark.Case, execution *reviewExecutionResult) { + if !benchCase.EffectiveWithoutDiscussion() { + return + } + if execution.ExitCode != exitcode.Success { + execution.Warnings = append(execution.Warnings, "discussion-free benchmark requires cr review --without-discussion; no fallback to saved sessions was attempted") + return + } + if execution.Review != nil && execution.Review.Run.WithoutDiscussion { + return + } + verificationErr := fmt.Errorf("discussion isolation could not be verified: review JSON must report run.without_discussion=true; use a compatible cr binary or explicitly set without_discussion: false to opt out") + execution.Warnings = append(execution.Warnings, verificationErr.Error()) + if execution.FailureClassification != failureNone { + return // Preserve a more specific missing/invalid JSON failure. + } + execution.Err = verificationErr + execution.FailureClassification = failureDiscussionIsolationUnverified +} + func reviewArgs(suiteDir string, candidate benchmark.Candidate, benchCase benchmark.Case) []string { args := []string{"--profile", candidate.Profile, "--quiet", "review", benchCase.PR, "--dry-run", "--json"} if benchCase.ReviewBaseSHA != "" { @@ -439,6 +469,9 @@ func reviewArgs(suiteDir string, candidate benchmark.Candidate, benchCase benchm if benchCase.ReviewHeadSHA != "" { args = append(args, "--review-head-sha", benchCase.ReviewHeadSHA) } + if benchCase.EffectiveWithoutDiscussion() { + args = append(args, "--without-discussion") + } if candidate.Stages.Selection.Model != "" { args = append(args, "--selection-model", candidate.Stages.Selection.Model, @@ -631,13 +664,14 @@ func summarizeCases(cases []benchmark.Case) []benchmarkCase { out := make([]benchmarkCase, 0, len(cases)) for _, benchCase := range cases { out = append(out, benchmarkCase{ - ID: benchCase.ID, - PR: benchCase.PR, - ReviewBaseSHA: benchCase.ReviewBaseSHA, - ReviewHeadSHA: benchCase.ReviewHeadSHA, - ExpectedBaseSHA: benchCase.ExpectedBaseSHA, - ExpectedHeadSHA: benchCase.ExpectedHeadSHA, - Anchors: append([]benchmark.Anchor(nil), benchCase.Anchors...), + ID: benchCase.ID, + PR: benchCase.PR, + ReviewBaseSHA: benchCase.ReviewBaseSHA, + ReviewHeadSHA: benchCase.ReviewHeadSHA, + WithoutDiscussion: benchCase.EffectiveWithoutDiscussion(), + ExpectedBaseSHA: benchCase.ExpectedBaseSHA, + ExpectedHeadSHA: benchCase.ExpectedHeadSHA, + Anchors: append([]benchmark.Anchor(nil), benchCase.Anchors...), }) } return out @@ -773,10 +807,12 @@ func writeSuiteArtifacts(summary benchmarkSuiteSummary) error { } for _, run := range summary.Runs { manifest.Runs = append(manifest.Runs, benchmarkManifestRun{ - RunID: run.RunID, - CandidateID: run.CandidateID, - CaseID: run.CaseID, - Artifacts: run.Artifacts, + RunID: run.RunID, + CandidateID: run.CandidateID, + CaseID: run.CaseID, + RequestedWithoutDiscussion: run.RequestedWithoutDiscussion, + WithoutDiscussionVerified: run.WithoutDiscussionVerified, + Artifacts: run.Artifacts, }) } if err := writeJSONFile(summary.Artifacts.Manifest, manifest); err != nil { diff --git a/internal/cmd/catalogcmd/catalogcmd.go b/internal/cmd/catalogcmd/catalogcmd.go index fcfbd51a..a25dd1cb 100644 --- a/internal/cmd/catalogcmd/catalogcmd.go +++ b/internal/cmd/catalogcmd/catalogcmd.go @@ -26,6 +26,7 @@ func Register(rootCmd *cobra.Command, opts *root.Options) { } type catalogView struct { + Digest string `json:"digest"` Source modelcatalog.Source `json:"source"` Manifest modelcatalog.Manifest `json:"manifest"` Runtimes []runtimeView `json:"runtimes"` @@ -164,5 +165,5 @@ func buildView(catalog *modelcatalog.Catalog) catalogView { } return modelViews[i].RuntimeID < modelViews[j].RuntimeID }) - return catalogView{Source: catalog.Source(), Manifest: catalog.Manifest(), Runtimes: runtimeViews, Models: modelViews, Defaults: catalog.Defaults(), Pricing: catalog.Pricing()} + return catalogView{Digest: catalog.Digest(), Source: catalog.Source(), Manifest: catalog.Manifest(), Runtimes: runtimeViews, Models: modelViews, Defaults: catalog.Defaults(), Pricing: catalog.Pricing()} } diff --git a/internal/cmd/catalogcmd/catalogcmd_test.go b/internal/cmd/catalogcmd/catalogcmd_test.go new file mode 100644 index 00000000..c5bd5481 --- /dev/null +++ b/internal/cmd/catalogcmd/catalogcmd_test.go @@ -0,0 +1,47 @@ +package catalogcmd + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/open-cli-collective/codereview-cli/internal/modelcatalog" +) + +func TestCatalogShowJSONIncludesExactRequestBinding(t *testing.T) { + catalog, err := modelcatalog.LoadBundled() + if err != nil { + t.Fatal(err) + } + var output bytes.Buffer + if err := render(&output, true, catalog); err != nil { + t.Fatal(err) + } + var value catalogView + if err := json.Unmarshal(output.Bytes(), &value); err != nil { + t.Fatal(err) + } + if value.Digest != catalog.Digest() || value.Manifest.SchemaVersion != 2 || value.Manifest.Revision != catalog.Revision() { + t.Fatal("catalog JSON lost exact snapshot identity") + } + bound := 0 + for _, price := range value.Pricing { + binding := price.RequestPricing + if binding == nil { + continue + } + bound++ + if binding.RuntimeID != "openai-api-key" || binding.RateApplication != "whole_request" || len(binding.ObservedServiceTiers) == 0 || binding.InputTokensMinInclusive == nil { + t.Fatalf("incomplete JSON binding: %#v", binding) + } + if price.ContextBand == "short" && (*binding.InputTokensMinInclusive != 0 || binding.InputTokensMaxExclusive == nil || *binding.InputTokensMaxExclusive != 272001) { + t.Fatal("JSON lost short interval") + } + if price.ContextBand == "long" && (*binding.InputTokensMinInclusive != 272001 || binding.InputTokensMaxExclusive != nil) { + t.Fatal("JSON lost long interval") + } + } + if bound != 24 { + t.Fatalf("JSON request binding count = %d, want 24", bound) + } +} diff --git a/internal/cmd/reviewcmd/reviewcmd.go b/internal/cmd/reviewcmd/reviewcmd.go index 2d810598..a0b2ed24 100644 --- a/internal/cmd/reviewcmd/reviewcmd.go +++ b/internal/cmd/reviewcmd/reviewcmd.go @@ -82,6 +82,7 @@ type commandFlags struct { reviewerEffort string reviewBaseSHA string reviewHeadSHA string + withoutDiscussion bool maxAgents int maxConcurrency int allowSelfReview bool @@ -126,6 +127,7 @@ func RegisterWithFactory(rootCmd *cobra.Command, opts *root.Options, factory Run cmd.Flags().StringVar(&flags.reviewerEffort, "reviewer-effort", "", "Override reviewer effort for this review") cmd.Flags().StringVar(&flags.reviewBaseSHA, "review-base-sha", "", "Review this base commit SHA instead of the PR's current base SHA; requires --dry-run and --review-head-sha") cmd.Flags().StringVar(&flags.reviewHeadSHA, "review-head-sha", "", "Review this head commit SHA instead of the PR's current head SHA; requires --dry-run and --review-base-sha") + cmd.Flags().BoolVar(&flags.withoutDiscussion, "without-discussion", false, "Replay the pinned review as a first pass, without the PR's existing discussion or review sessions, and deny reviewers common network tools; requires --dry-run, --review-base-sha, and --review-head-sha") cmd.Flags().IntVar(&flags.maxAgents, "max-agents", 0, "Maximum selected reviewer agents") cmd.Flags().IntVar(&flags.maxConcurrency, "max-concurrency", 0, "Maximum concurrent reviewer agents") cmd.Flags().BoolVar(&flags.allowSelfReview, "allow-self-review", false, "Allow reviewer credentials matching the PR author") @@ -204,6 +206,14 @@ func runReview(ctx context.Context, cmd *cobra.Command, opts *root.Options, fact return exitcode.Usage(fmt.Errorf("--review-base-sha and --review-head-sha require --dry-run or --no-post")) } } + if flags.withoutDiscussion { + if !flags.dryRun { + return exitcode.Usage(fmt.Errorf("--without-discussion requires --dry-run or --no-post")) + } + if !reviewBaseChanged { + return exitcode.Usage(fmt.Errorf("--without-discussion requires --review-base-sha and --review-head-sha")) + } + } if flags.rerun && flags.retryPosts { return exitcode.Usage(fmt.Errorf("--rerun and --retry-posts are mutually exclusive")) } @@ -214,6 +224,9 @@ func runReview(ctx context.Context, cmd *cobra.Command, opts *root.Options, fact if sessionName != "" && flags.retryPosts { return exitcode.Usage(fmt.Errorf("--session cannot be used with --retry-posts")) } + if flags.freshSession && flags.withoutDiscussion { + return exitcode.Usage(fmt.Errorf("--fresh-session cannot be used with --without-discussion, which never reuses or resets PR sessions")) + } if flags.freshSession && flags.retryPosts { return exitcode.Usage(fmt.Errorf("--fresh-session cannot be used with --retry-posts")) } @@ -365,6 +378,7 @@ func runReview(ctx context.Context, cmd *cobra.Command, opts *root.Options, fact ReviewerFast: reviewerFast, ReviewBaseSHA: reviewBaseSHA, ReviewHeadSHA: reviewHeadSHA, + WithoutDiscussion: flags.withoutDiscussion, Rerun: flags.rerun, FreshSession: flags.freshSession, ToolVersion: version.Version, diff --git a/internal/cmd/reviewcmd/reviewcmd_test.go b/internal/cmd/reviewcmd/reviewcmd_test.go index bbf52611..b92080e1 100644 --- a/internal/cmd/reviewcmd/reviewcmd_test.go +++ b/internal/cmd/reviewcmd/reviewcmd_test.go @@ -671,6 +671,71 @@ func TestReviewRejectsInvalidReviewSHAOverrides(t *testing.T) { } } +func TestReviewDryRunPassesWithoutDiscussion(t *testing.T) { + for _, mode := range []string{"--dry-run", "--no-post"} { + t.Run(mode, func(t *testing.T) { + runner := &fakeRunner{result: testPipelineResult(false)} + cmd, _ := newTestCommand(t, testConfig(), fakeFactory(runner)) + + err := root.Execute(cmd, []string{ + "review", "https://github.com/open-cli-collective/codereview-cli/pull/29", + mode, + "--review-base-sha", "1111111", + "--review-head-sha", "2222222", + "--without-discussion", + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if len(runner.requests) != 1 { + t.Fatalf("runner calls = %d, want 1", len(runner.requests)) + } + if !runner.requests[0].WithoutDiscussion { + t.Fatal("WithoutDiscussion = false, want true") + } + }) + } +} + +func TestReviewRejectsWithoutDiscussionOutsidePinnedDryRun(t *testing.T) { + tests := []struct { + name string + args []string + wantErr string + }{ + {name: "live with SHAs", args: []string{"--review-base-sha", "1111111", "--review-head-sha", "2222222", "--without-discussion"}, wantErr: "require --dry-run or --no-post"}, + {name: "live without SHAs", args: []string{"--without-discussion"}, wantErr: "--without-discussion requires --dry-run or --no-post"}, + {name: "dry run without SHAs", args: []string{"--dry-run", "--without-discussion"}, wantErr: "--without-discussion requires --review-base-sha and --review-head-sha"}, + {name: "no post without SHAs", args: []string{"--no-post", "--without-discussion"}, wantErr: "--without-discussion requires --review-base-sha and --review-head-sha"}, + {name: "dry run with base only", args: []string{"--dry-run", "--review-base-sha", "1111111", "--without-discussion"}, wantErr: "must be set together"}, + {name: "fresh session", args: []string{"--dry-run", "--review-base-sha", "1111111", "--review-head-sha", "2222222", "--without-discussion", "--fresh-session"}, wantErr: "--fresh-session cannot be used with --without-discussion"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var factoryCalled bool + cmd, _ := newTestCommand(t, testConfig(), func(context.Context, app.OpenRequest) (app.Runtime, error) { + factoryCalled = true + return app.Runtime{Runner: &fakeRunner{result: testPipelineResult(false), liveResult: testLiveResult(false)}}, nil + }) + + args := append([]string{"review", "https://github.com/open-cli-collective/codereview-cli/pull/29"}, tt.args...) + err := root.Execute(cmd, args) + if err == nil { + t.Fatal("Execute error = nil, want usage error") + } + if got := exitcode.FromError(err); got != exitcode.UsageError { + t.Fatalf("exit code = %d, want usage", got) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("error = %q, want %q", err.Error(), tt.wantErr) + } + if factoryCalled { + t.Fatal("runtime factory was called for invalid --without-discussion use") + } + }) + } +} + func TestReviewLiveRejectsStageOverridesBeforeRuntimeFactory(t *testing.T) { tests := []struct { name string @@ -1746,7 +1811,7 @@ func reviewTestCatalog(t *testing.T) (*modelcatalog.Catalog, string) { } files["models.csv"] = append(files["models.csv"], []byte("openai-api-key,custom-review-model,low|medium,Y,,,https://example.invalid/custom-review-model,2026-10-03\n")...) files["defaults.csv"] = []byte(strings.Replace(string(files["defaults.csv"]), "openai-api-key,medium,gpt-6.1-sol,low,", "openai-api-key,medium,custom-review-model,medium,", 1)) - files["pricing.csv"] = append(files["pricing.csv"], []byte("custom-review-model,standard,all,3,7,1,0.5,,,https://example.invalid/custom-review-model,2026-10-03\n")...) + files["pricing.csv"] = append(files["pricing.csv"], []byte("custom-review-model,standard,all,3,7,1,0.5,,,https://example.invalid/custom-review-model,2026-10-03,,,,,\n")...) for name, body := range files { if err := os.WriteFile(filepath.Join(source, name), body, 0o600); err != nil { // #nosec G703 -- source is under t.TempDir. t.Fatalf("write catalog %s: %v", name, err) diff --git a/internal/cmd/root/root_test.go b/internal/cmd/root/root_test.go index ed0a3cfa..437decca 100644 --- a/internal/cmd/root/root_test.go +++ b/internal/cmd/root/root_test.go @@ -2,9 +2,13 @@ package root import ( "bytes" + "crypto/sha256" + "encoding/json" "errors" + "fmt" "os" "path/filepath" + "reflect" "runtime" "strings" "testing" @@ -87,13 +91,18 @@ func TestCatalogSnapshotIsCachedForOneCommand(t *testing.T) { t.Fatal("runtime.Caller failed") } source := t.TempDir() + sourceRoot, err := os.OpenRoot(source) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sourceRoot.Close() }) dataDir := filepath.Join(filepath.Dir(testFile), "..", "..", "modelcatalog", "data") for _, name := range []string{"manifest.json", "runtimes.csv", "models.csv", "defaults.csv", "pricing.csv"} { body, err := os.ReadFile(filepath.Join(dataDir, name)) // #nosec G304 -- dataDir is the repository's bundled fixture. if err != nil { t.Fatalf("read catalog %s: %v", name, err) } - if err := os.WriteFile(filepath.Join(source, name), body, 0o600); err != nil { // #nosec G703 -- source is under t.TempDir. + if err := sourceRoot.WriteFile(name, body, 0o600); err != nil { t.Fatalf("write catalog %s: %v", name, err) } } @@ -106,6 +115,38 @@ func TestCatalogSnapshotIsCachedForOneCommand(t *testing.T) { if first.Source().Kind != "local" { t.Fatalf("first source kind = %q, want local", first.Source().Kind) } + beforeSelection, reason := first.PriceForRequest("openai-api-key", "gpt-6.1-sol", "default", 200000) + if reason != "" || beforeSelection.Price.ContextBand != "short" { + t.Fatalf("original selection = %#v, %q", beforeSelection, reason) + } + body, err := sourceRoot.ReadFile("pricing.csv") + if err != nil { + t.Fatal(err) + } + body = bytes.ReplaceAll(body, []byte("272001"), []byte("100001")) + if err := sourceRoot.WriteFile("pricing.csv", body, 0o600); err != nil { + t.Fatal(err) + } + manifest := first.Manifest() + manifest.Files["pricing.csv"] = fmt.Sprintf("%x", sha256.Sum256(body)) + manifestBody, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + if err := sourceRoot.WriteFile("manifest.json", manifestBody, 0o600); err != nil { + t.Fatal(err) + } + if err := sourceRoot.Close(); err != nil { + t.Fatal(err) + } + fresh, err := modelcatalog.LoadPath(source) + if err != nil { + t.Fatal(err) + } + freshSelection, reason := fresh.PriceForRequest("openai-api-key", "gpt-6.1-sol", "default", 200000) + if reason != "" || freshSelection.Price.ContextBand != "long" || fresh.Digest() == first.Digest() || fresh.Revision() != first.Revision() { + t.Fatal("changed on-disk metadata did not produce a different exact snapshot") + } if err := os.RemoveAll(source); err != nil { t.Fatalf("remove source after first load: %v", err) } @@ -116,6 +157,10 @@ func TestCatalogSnapshotIsCachedForOneCommand(t *testing.T) { if second != first { t.Fatalf("second snapshot pointer = %p, want cached %p", second, first) } + afterSelection, reason := second.PriceForRequest("openai-api-key", "gpt-6.1-sol", "default", 200000) + if reason != "" || !reflect.DeepEqual(afterSelection, beforeSelection) { + t.Fatal("command cached selection changed after on-disk metadata mutation") + } if second.Revision() != first.Revision() { t.Fatalf("second revision = %q, want %q", second.Revision(), first.Revision()) } diff --git a/internal/dossier/dossier.go b/internal/dossier/dossier.go index 153dbb97..b3c21f21 100644 --- a/internal/dossier/dossier.go +++ b/internal/dossier/dossier.go @@ -42,9 +42,12 @@ type ChangedFile struct { // Inputs contains the source material written to raw dossier artifacts. type Inputs struct { - CurrentPR gitprovider.PR - ReviewPR gitprovider.PR - PinnedReview bool + CurrentPR gitprovider.PR + ReviewPR gitprovider.PR + PinnedReview bool + // WithoutDiscussion drops every discussion input, even if a caller + // supplied some, and records that the review ran without discussion. + WithoutDiscussion bool ChangedFiles []ChangedFile Threads []gitprovider.InlineThread ThreadContext []threadcontext.Thread @@ -128,6 +131,7 @@ type dossierPRContextArtifact struct { type dossierDiscussionArtifact struct { PinnedReview bool `json:"pinned_review"` + WithoutDiscussion bool `json:"without_discussion,omitempty"` DiscussionOmittedNote string `json:"discussion_omitted_note,omitempty"` TopLevelComments []dossierTopLevelCommentArtifact `json:"top_level_comments,omitempty"` InlineThreads []dossierInlineThreadArtifact `json:"inline_threads,omitempty"` @@ -138,6 +142,7 @@ type DiscussionSummary struct { SchemaVersion int `json:"schema_version"` SourceFingerprint string `json:"source_fingerprint,omitempty"` PinnedReview bool `json:"pinned_review"` + WithoutDiscussion bool `json:"without_discussion,omitempty"` DiscussionOmittedNote string `json:"discussion_omitted_note,omitempty"` TopLevelOmitted int `json:"top_level_comments_omitted,omitempty"` InlineThreadsOmitted int `json:"inline_threads_omitted,omitempty"` @@ -286,6 +291,12 @@ func WriteRaw(paths runartifact.Paths, in Inputs) error { return fmt.Errorf("pipeline: create dossier dir: %w", err) } } + if in.WithoutDiscussion { + in.Threads = nil + in.ThreadContext = nil + in.Reviews = nil + in.IssueComments = nil + } prContext := dossierPRContextArtifact{ Title: in.CurrentPR.Title, @@ -314,9 +325,10 @@ func WriteRaw(paths runartifact.Paths, in Inputs) error { } discussion := dossierDiscussionArtifact{ PinnedReview: in.PinnedReview, + WithoutDiscussion: in.WithoutDiscussion, DiscussionOmittedNote: strings.TrimSpace(in.DiscussionOmittedNote), } - if !in.PinnedReview { + if !in.PinnedReview && !in.WithoutDiscussion { discussion.TopLevelComments = topLevelComments discussion.InlineThreads = inlineThreads } @@ -395,10 +407,11 @@ func ReadDiscussionSummary(paths runartifact.Paths) (DiscussionSummary, error) { } func summarizeDiscussionArtifacts(ctx context.Context, env Env, req PreparationRequest, discussion dossierDiscussionArtifact) (DiscussionSummary, error) { - if discussion.PinnedReview { + if discussion.PinnedReview || discussion.WithoutDiscussion { return DiscussionSummary{ SchemaVersion: dossierSummarySchemaVersion, - PinnedReview: true, + PinnedReview: discussion.PinnedReview, + WithoutDiscussion: discussion.WithoutDiscussion, DiscussionOmittedNote: strings.TrimSpace(discussion.DiscussionOmittedNote), }, nil } @@ -797,7 +810,7 @@ func renderDossierDiscussionSummaryMarkdown(summary DiscussionSummary, title str var out strings.Builder out.WriteString(title) out.WriteString("\n\n") - if summary.PinnedReview { + if summary.PinnedReview || summary.WithoutDiscussion { note := strings.TrimSpace(summary.DiscussionOmittedNote) if note == "" { note = "Current PR discussion omitted for pinned review." diff --git a/internal/llm/adapter.go b/internal/llm/adapter.go index 485a87b3..c1de797f 100644 --- a/internal/llm/adapter.go +++ b/internal/llm/adapter.go @@ -68,6 +68,10 @@ type ReviewerWorkspaceRequest struct { Env []string AllowedFiles []string MaxToolOutputBytes int + // NoNetwork asks the adapter to deny reviewer tools that reach the network + // (git hosts, web fetch, web search), so a reviewer cannot look up live PR + // state such as discussion. It narrows the tool surface; it is not a sandbox. + NoNetwork bool } // ErrReviewerWorkspaceUnsupported reports that an adapter cannot use a prepared @@ -135,6 +139,7 @@ type Response struct { Usage Usage DurationMS int64 ReviewerToolEvidence *ReviewerToolEvidence + RequestCostEvidence *RequestCostEvidence } // DiffToolStatus records the terminal state of the required reviewer diff tool. @@ -154,8 +159,9 @@ const ( // ReviewerToolEvidence records bounded, machine-significant reviewer tool state. // It is absent when the adapter does not provide reviewer tool evidence. type ReviewerToolEvidence struct { - DiffStatus DiffToolStatus `json:"diff_status"` - DiffDiagnostic string `json:"diff_diagnostic,omitempty"` + DiffStatus DiffToolStatus `json:"diff_status"` + DiffDiagnostic string `json:"diff_diagnostic,omitempty"` + Trace *ReviewerToolTrace `json:"trace,omitempty"` } // Usage records nullable usage metrics. @@ -223,9 +229,25 @@ func (e *StructuredValidationError) Is(target error) bool { // conversation no longer exists is retried once as a fresh conversation // instead of failing; callers that store the session for later reuse must // persist the returned SessionID, which replaces a dangling stored one. -func RunStructuredWithSessionResume[T any](ctx context.Context, adapter Adapter, resumeSessionID string, req Request, decode Decoder[T]) (StructuredResult[T], error) { +func RunStructuredWithSessionResume[T any](ctx context.Context, adapter Adapter, resumeSessionID string, req Request, decode Decoder[T]) (result StructuredResult[T], resultErr error) { + collector := newRequestCostCollector(adapter) + defer func() { + result.Response = cloneResponse(result.Response) + result.Response.RequestCostEvidence = collector.snapshot() + // Validation error payload and returned snapshots must not share mutable + // response evidence with each other or with the adapter. + var copied []StructuredValidationAttempt + if result.ValidationAttempts != nil { + copied = make([]StructuredValidationAttempt, len(result.ValidationAttempts)) + } + for i, attempt := range result.ValidationAttempts { + attempt.Response = cloneResponse(attempt.Response) + copied[i] = attempt + } + result.ValidationAttempts = copied + }() var zero T - sessionID, response, err := runOnceWithSession(ctx, adapter, resumeSessionID, req) + sessionID, response, err := runOnceWithSession(ctx, adapter, resumeSessionID, req, collector, "initial") if err != nil { return StructuredResult[T]{Response: response, SessionID: sessionID}, err } @@ -254,7 +276,7 @@ func RunStructuredWithSessionResume[T any](ctx context.Context, adapter Adapter, if strings.TrimSpace(retryResumeSessionID) == "" && !retryReq.FreshValidationRetrySession { retryResumeSessionID = resumeSessionID } - retrySessionID, retryResponse, err := runOnceWithSession(ctx, adapter, retryResumeSessionID, retryReq) + retrySessionID, retryResponse, err := runOnceWithSession(ctx, adapter, retryResumeSessionID, retryReq, collector, "correction") if err != nil { return StructuredResult[T]{Response: retryResponse, SessionID: retrySessionID, ValidationAttempts: attempts}, err } @@ -305,10 +327,15 @@ func decodeStructuredAccepted[T any](decode Decoder[T], data []byte) (T, []byte, // prior context, neither of which the review depends on. Without the fallback a // dangling session ID fails the task outright, and a whole round of reviewers // can fail before doing any work while still reporting zero findings. -func runOnceWithSession(ctx context.Context, adapter Adapter, resumeSessionID string, req Request) (string, Response, error) { +func runOnceWithSession(ctx context.Context, adapter Adapter, resumeSessionID string, req Request, collector *requestCostCollector, phase string) (string, Response, error) { + var adapterAttempt uint32 sessionDropped := false for attempt := 0; ; attempt++ { + if adapterAttempt < ^uint32(0) { + adapterAttempt++ + } sid, resp, err := runOnceAttempt(ctx, adapter, resumeSessionID, req) + collector.observe(resp, phase, adapterAttempt) if err == nil { return sid, resp, nil } @@ -338,15 +365,19 @@ func runOnceAttempt(ctx context.Context, adapter Adapter, resumeSessionID string ) if req.ReviewerWorkspace != nil { if err := RequireReviewerWorkspace(adapter); err != nil { - return "", Response{}, err + return "", nonDispatchedCostResponse(adapter, req), err } } - if strings.TrimSpace(resumeSessionID) != "" && adapter.SupportsResume() { + resuming := strings.TrimSpace(resumeSessionID) != "" && adapter.SupportsResume() + if resuming { stream, err = adapter.Resume(ctx, resumeSessionID, req) } else { stream, err = adapter.Start(ctx, req) } if err != nil { + if stream == nil && !resuming { + return "", nonDispatchedCostResponse(adapter, req), err + } return "", Response{}, err } response, err := stream.Wait(ctx) @@ -355,6 +386,15 @@ func runOnceAttempt(ctx context.Context, adapter Adapter, resumeSessionID string func cloneResponse(response Response) Response { response.StructuredOutput = append([]byte(nil), response.StructuredOutput...) + response.Usage.TokensIn = costCopy(response.Usage.TokensIn) + response.Usage.TokensOut = costCopy(response.Usage.TokensOut) + response.Usage.CacheRead = costCopy(response.Usage.CacheRead) + response.Usage.CacheCreate = costCopy(response.Usage.CacheCreate) + response.Usage.CacheCreate5m = costCopy(response.Usage.CacheCreate5m) + response.Usage.CacheCreate1h = costCopy(response.Usage.CacheCreate1h) + response.Usage.CostUSD = costCopy(response.Usage.CostUSD) + response.ReviewerToolEvidence = CloneReviewerToolEvidence(response.ReviewerToolEvidence) + response.RequestCostEvidence = CloneRequestCostEvidence(response.RequestCostEvidence) return response } diff --git a/internal/llm/request_cost_evidence.go b/internal/llm/request_cost_evidence.go new file mode 100644 index 00000000..2d0ebc9f --- /dev/null +++ b/internal/llm/request_cost_evidence.go @@ -0,0 +1,570 @@ +package llm + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "math" + "slices" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +// Cost evidence is bounded client telemetry, never provider-reported cost. +const ( + MaxRequestCostAttempts = 64 + MaxRequestCostEvidenceBytes = 256 * 1024 + RequestCostReservedBytes = 4 * 1024 + MaxRequestCostGaps = 16 +) + +// CostEvidenceGap is a finite allow-listed reason for incomplete evidence. +type CostEvidenceGap string + +// Cost evidence gaps describe why observations are incomplete or untrusted. +const ( + CostGapMissing CostEvidenceGap = "missing_evidence" + CostGapInvalid CostEvidenceGap = "invalid_evidence" + CostGapUsage CostEvidenceGap = "missing_or_invalid_usage" + CostGapIdentity CostEvidenceGap = "missing_or_invalid_identity" + CostGapResponse CostEvidenceGap = "unsupported_response" + CostGapTransport CostEvidenceGap = "opaque_transport" + CostGapEndpoint CostEvidenceGap = "custom_or_unverified_endpoint" + CostGapDispatch CostEvidenceGap = "unresolved_dispatch" + CostGapOverflow CostEvidenceGap = "overflow" + CostGapLegacy CostEvidenceGap = "legacy_history_unknown" + CostGapInterrupted CostEvidenceGap = "interrupted_generation" + CostGapCheckpoint CostEvidenceGap = "checkpoint_unavailable" + CostGapConflict CostEvidenceGap = "conflicting_evidence" + CostGapDuplicate CostEvidenceGap = "duplicate_identity" + CostGapScope CostEvidenceGap = "scope_mismatch" + CostGapPending CostEvidenceGap = "pending_generation" +) + +// RequestCostProducer opts into collection. Start errors without a stream MUST +// mean no dispatch. Returning both a stream and an error is unsupported and +// remains missing/possibly-dispatched evidence; the direct API never does so. +// Only the concrete direct OpenAI adapter opts in in v1. +type RequestCostProducer interface{ RequestCostEvidenceSource() string } + +// RequestCostSource returns only an explicitly supported producer contract. +func RequestCostSource(adapter Adapter) string { + p, ok := adapter.(RequestCostProducer) + if ok && p.RequestCostEvidenceSource() == "openai_responses" { + return "openai_responses" + } + return "" +} + +// RequestCostEvidence preserves ordered observations, independently of output. +type RequestCostEvidence struct { + Version int `json:"version"` + Source string `json:"source"` + Scope string `json:"scope"` + ArtifactScope string `json:"artifact_scope"` + RunID string `json:"run_id"` + TaskID string `json:"task_id"` + AttemptsObserved uint64 `json:"attempts_observed"` + AttemptCountExact bool `json:"attempt_count_exact"` + Truncated bool `json:"truncated"` + DroppedAttempts uint64 `json:"dropped_attempts"` + Gaps []CostEvidenceGap `json:"gaps"` + Attempts []RequestCostAttempt `json:"attempts"` +} + +// RequestCostAttempt excludes bodies, URLs, headers and raw errors. +type RequestCostAttempt struct { + GenerationID string `json:"generation_id"` + Ordinal uint32 `json:"ordinal"` + ValidationPhase string `json:"validation_phase"` + AdapterAttempt uint32 `json:"adapter_attempt"` + RuntimeID string `json:"runtime_id"` + EndpointKind string `json:"endpoint_kind"` + TransportMode string `json:"transport_mode"` + Dispatch string `json:"dispatch"` + HTTPStatus *int `json:"http_status"` + BodyState string `json:"body_state"` + ProviderResponseID *string `json:"provider_response_id"` + ObservedModel *string `json:"observed_model"` + ObservedServiceTier *string `json:"observed_service_tier"` + ResponseStatus *string `json:"response_status"` + RequestedModel string `json:"requested_model"` + RequestedServiceTier *string `json:"requested_service_tier"` + Usage *OpenAIRequestUsage `json:"usage"` + Issues []CostEvidenceGap `json:"issues"` +} + +// OpenAIRequestUsage preserves absence and negative observations without pricing. +type OpenAIRequestUsage struct { + InputTokens *int64 `json:"input_tokens"` + OutputTokens *int64 `json:"output_tokens"` + CacheReadTokens *int64 `json:"cache_read_tokens"` + CacheWriteTokens *int64 `json:"cache_write_tokens"` + TotalTokens *int64 `json:"total_tokens"` +} + +func costCopy[T any](p *T) *T { + if p == nil { + return nil + } + v := *p + return &v +} + +// CloneRequestCostEvidence makes a detached copy, including all nested pointers. +func CloneRequestCostEvidence(e *RequestCostEvidence) *RequestCostEvidence { + if e == nil { + return nil + } + c := *e + c.Gaps = append([]CostEvidenceGap{}, e.Gaps...) + c.Attempts = make([]RequestCostAttempt, len(e.Attempts)) + for i, a := range e.Attempts { + a.HTTPStatus = costCopy(a.HTTPStatus) + a.ProviderResponseID = costCopy(a.ProviderResponseID) + a.ObservedModel = costCopy(a.ObservedModel) + a.ObservedServiceTier = costCopy(a.ObservedServiceTier) + a.ResponseStatus = costCopy(a.ResponseStatus) + a.RequestedServiceTier = costCopy(a.RequestedServiceTier) + a.Issues = append([]CostEvidenceGap{}, a.Issues...) + if a.Usage != nil { + u := *a.Usage + u.InputTokens = costCopy(u.InputTokens) + u.OutputTokens = costCopy(u.OutputTokens) + u.CacheReadTokens = costCopy(u.CacheReadTokens) + u.CacheWriteTokens = costCopy(u.CacheWriteTokens) + u.TotalTokens = costCopy(u.TotalTokens) + a.Usage = &u + } + c.Attempts[i] = a + } + return &c +} + +// SaturatingCostAdd prevents observation/omission counters wrapping to zero. +func SaturatingCostAdd(a, b uint64) uint64 { + if math.MaxUint64-a < b { + return math.MaxUint64 + } + return a + b +} + +// SafeCostString accepts bounded UTF-8 without controls, preserving exact bytes. +func SafeCostString(s string, limit int) bool { + if s == "" || len(s) > limit || !utf8.ValidString(s) { + return false + } + for _, r := range s { + if unicode.IsControl(r) { + return false + } + } + return true +} + +func validCostGap(g CostEvidenceGap) bool { + switch g { + case CostGapMissing, CostGapInvalid, CostGapUsage, CostGapIdentity, CostGapResponse, CostGapTransport, + CostGapEndpoint, CostGapDispatch, CostGapOverflow, CostGapLegacy, CostGapInterrupted, CostGapCheckpoint, + CostGapConflict, CostGapDuplicate, CostGapScope, CostGapPending: + return true + } + return false +} + +// AddCostGap deduplicates/sorts the finite enum set; arbitrary errors are excluded. +func AddCostGap(gaps []CostEvidenceGap, gap CostEvidenceGap) []CostEvidenceGap { + if !validCostGap(gap) { + gap = CostGapInvalid + } + if !slices.Contains(gaps, gap) { + gaps = append(gaps, gap) + } + slices.Sort(gaps) + return gaps +} + +// UnknownRequestCostEvidence constructs an explicit bounded unknown value. +func UnknownRequestCostEvidence(gap CostEvidenceGap) *RequestCostEvidence { + return &RequestCostEvidence{Version: 1, Source: "openai_responses", Scope: "invocation", Gaps: []CostEvidenceGap{gap}, Attempts: []RequestCostAttempt{}} +} + +// NormalizeRequestCostEvidence bounds and sanitizes a detached snapshot. It +// never repairs a gap or manufactures usage/identity. The retained prefix wins. +func NormalizeRequestCostEvidence(e *RequestCostEvidence) *RequestCostEvidence { + if e == nil { + return nil + } + c := *e + c.Gaps = []CostEvidenceGap{} + c.Attempts = []RequestCostAttempt{} + for _, g := range e.Gaps { + c.Gaps = AddCostGap(c.Gaps, g) + } + if c.Version != 1 || c.Source != "openai_responses" || (c.Scope != "invocation" && c.Scope != "task_artifact_history") { + return UnknownRequestCostEvidence(CostGapInvalid) + } + for _, p := range []*string{&c.ArtifactScope, &c.RunID, &c.TaskID} { + if *p != "" && !SafeCostString(*p, 256) { + *p = "" + c.Gaps = AddCostGap(c.Gaps, CostGapScope) + } + } + for i, raw := range e.Attempts { + if i >= MaxRequestCostAttempts { + c.DroppedAttempts = SaturatingCostAdd(c.DroppedAttempts, uint64(len(e.Attempts[i:]))) + c.Truncated = true + break + } + a := raw + a.Issues = []CostEvidenceGap{} + for _, g := range raw.Issues { + a.Issues = AddCostGap(a.Issues, g) + } + for _, p := range []*string{&a.GenerationID, &a.RequestedModel} { + if *p != "" && !SafeCostString(*p, 256) { + *p = "" + a.Issues = AddCostGap(a.Issues, CostGapIdentity) + } + } + for _, field := range []struct { + p **string + limit int + }{{&a.ProviderResponseID, 256}, {&a.ObservedModel, 256}, {&a.ObservedServiceTier, 64}, {&a.ResponseStatus, 64}, {&a.RequestedServiceTier, 64}} { + if *field.p != nil && !SafeCostString(**field.p, field.limit) { + *field.p = nil + a.Issues = AddCostGap(a.Issues, CostGapIdentity) + } + } + if a.RuntimeID != "openai-api-key" { + a.RuntimeID = "" + a.Issues = AddCostGap(a.Issues, CostGapInvalid) + } + if a.ValidationPhase != "" && a.ValidationPhase != "initial" && a.ValidationPhase != "correction" { + a.ValidationPhase = "" + a.Issues = AddCostGap(a.Issues, CostGapInvalid) + } + if a.EndpointKind != "official_global" { + a.EndpointKind = "custom_or_unverified" + a.Issues = AddCostGap(a.Issues, CostGapEndpoint) + } + if a.TransportMode != "single_send_v1" { + a.TransportMode = "opaque" + a.Issues = AddCostGap(a.Issues, CostGapTransport) + if a.Dispatch != "not_dispatched" { + c.AttemptCountExact = false + } + } + if a.Dispatch != "not_dispatched" { + a.Dispatch = "possibly_dispatched" + } + switch a.BodyState { + case "complete", "missing", "read_error", "over_limit", "malformed": + default: + a.BodyState = "missing" + a.Issues = AddCostGap(a.Issues, CostGapInvalid) + } + if a.HTTPStatus != nil && (*a.HTTPStatus < 100 || *a.HTTPStatus > 599) { + a.HTTPStatus = nil + a.Issues = AddCostGap(a.Issues, CostGapInvalid) + } + if a.Dispatch != "not_dispatched" { + validateObservedCostAttempt(&a) + } + c.Attempts = append(c.Attempts, a) + } + if c.Truncated || c.DroppedAttempts != 0 { + c.Truncated = true + c.Gaps = AddCostGap(c.Gaps, CostGapOverflow) + c.AttemptCountExact = false + } + if c.AttemptsObserved < uint64(len(c.Attempts)) { + c.AttemptsObserved = uint64(len(c.Attempts)) + c.Gaps = AddCostGap(c.Gaps, CostGapInvalid) + } + if c.AttemptsObserved != SaturatingCostAdd(uint64(len(c.Attempts)), c.DroppedAttempts) { + c.Gaps = AddCostGap(c.Gaps, CostGapMissing) + } + if len(c.Gaps) != 0 { + c.AttemptCountExact = false + } + for len(c.Attempts) > 0 { + b, _ := json.Marshal(c) + if len(b) <= MaxRequestCostEvidenceBytes-RequestCostReservedBytes { + break + } + c.Attempts = c.Attempts[:len(c.Attempts)-1] + c.Truncated = true + c.DroppedAttempts = SaturatingCostAdd(c.DroppedAttempts, 1) + c.Gaps = AddCostGap(c.Gaps, CostGapOverflow) + c.AttemptCountExact = false + } + return CloneRequestCostEvidence(&c) +} + +func validateObservedCostAttempt(a *RequestCostAttempt) { + if a.BodyState != "complete" || a.HTTPStatus == nil || *a.HTTPStatus < 200 || *a.HTTPStatus > 299 { + a.Issues = AddCostGap(a.Issues, CostGapDispatch) + } + if a.ProviderResponseID == nil || a.ObservedModel == nil || a.ObservedServiceTier == nil { + a.Issues = AddCostGap(a.Issues, CostGapIdentity) + } + if a.ResponseStatus == nil || (*a.ResponseStatus != "completed" && *a.ResponseStatus != "incomplete") { + a.Issues = AddCostGap(a.Issues, CostGapResponse) + } + if a.ObservedServiceTier != nil { + switch *a.ObservedServiceTier { + case "default", "standard", "priority", "fast", "flex", "scale": + default: + a.Issues = AddCostGap(a.Issues, CostGapIdentity) + } + } + u := a.Usage + if u == nil || u.InputTokens == nil || u.OutputTokens == nil || u.CacheReadTokens == nil || u.CacheWriteTokens == nil { + a.Issues = AddCostGap(a.Issues, CostGapUsage) + return + } + in, out, read, write := *u.InputTokens, *u.OutputTokens, *u.CacheReadTokens, *u.CacheWriteTokens + if in < 0 || out < 0 || read < 0 || write < 0 || read > in || write > in-read { + a.Issues = AddCostGap(a.Issues, CostGapUsage) + } + if u.TotalTokens != nil && (in < 0 || out < 0 || in > math.MaxInt64-out || *u.TotalTokens != in+out) { + a.Issues = AddCostGap(a.Issues, CostGapUsage) + } +} + +// RequestCostEvidenceBytes supplies fixed-order, map-free v1 digest bytes. +func RequestCostEvidenceBytes(e *RequestCostEvidence) ([]byte, error) { + return json.Marshal(NormalizeRequestCostEvidence(e)) +} + +// RequestCostEvidenceDigest binds the full scoped raw evidence, excluding prices. +func RequestCostEvidenceDigest(e *RequestCostEvidence) (string, error) { + b, err := RequestCostEvidenceBytes(e) + if err != nil { + return "", err + } + h := sha256.New() + _, _ = h.Write([]byte("cr-request-cost-evidence/v1\n")) + _, _ = h.Write(b) + return hex.EncodeToString(h.Sum(nil)), nil +} + +// DecodeRequestCostEvidence isolates malformed/unsupported optional telemetry. +func DecodeRequestCostEvidence(data []byte) *RequestCostEvidence { + if len(data) == 0 || string(data) == "null" { + return nil + } + if len(data) > MaxRequestCostEvidenceBytes { + return UnknownRequestCostEvidence(CostGapOverflow) + } + var e RequestCostEvidence + if !RequestCostJSONValid(data) || json.Unmarshal(data, &e) != nil { + return UnknownRequestCostEvidence(CostGapInvalid) + } + if len(e.Attempts) > MaxRequestCostAttempts || len(e.Gaps) > MaxRequestCostGaps { + return UnknownRequestCostEvidence(CostGapOverflow) + } + for _, a := range e.Attempts { + if len(a.Issues) > MaxRequestCostGaps { + return UnknownRequestCostEvidence(CostGapOverflow) + } + } + return NormalizeRequestCostEvidence(&e) +} + +type requestCostCollector struct { + evidence *RequestCostEvidence + ordinal uint64 +} + +func newRequestCostCollector(adapter Adapter) *requestCostCollector { + if RequestCostSource(adapter) == "" { + return nil + } + return &requestCostCollector{evidence: &RequestCostEvidence{Version: 1, Source: "openai_responses", Scope: "invocation", AttemptCountExact: true, Gaps: []CostEvidenceGap{}, Attempts: []RequestCostAttempt{}}} +} +func (c *requestCostCollector) snapshot() *RequestCostEvidence { + if c == nil { + return nil + } + return NormalizeRequestCostEvidence(c.evidence) +} +func (c *requestCostCollector) observe(resp Response, phase string, attempt uint32) { + if c == nil { + return + } + c.ordinal = SaturatingCostAdd(c.ordinal, 1) + c.evidence.AttemptsObserved = SaturatingCostAdd(c.evidence.AttemptsObserved, 1) + e := NormalizeRequestCostEvidence(resp.RequestCostEvidence) + a := RequestCostAttempt{RuntimeID: "openai-api-key", TransportMode: "opaque", EndpointKind: "custom_or_unverified", Dispatch: "possibly_dispatched", BodyState: "missing", Issues: []CostEvidenceGap{CostGapMissing}} + if e != nil && len(e.Attempts) == 1 && e.AttemptsObserved == 1 { + a = e.Attempts[0] + } else { + c.evidence.Gaps = AddCostGap(c.evidence.Gaps, CostGapMissing) + c.evidence.AttemptCountExact = false + } + if e != nil { + for _, g := range e.Gaps { + c.evidence.Gaps = AddCostGap(c.evidence.Gaps, g) + } + c.evidence.AttemptCountExact = c.evidence.AttemptCountExact && e.AttemptCountExact + } + a.ValidationPhase = phase + a.AdapterAttempt = attempt + if c.ordinal <= math.MaxUint32 { + a.Ordinal = uint32(c.ordinal) + } else { + c.evidence.Gaps = AddCostGap(c.evidence.Gaps, CostGapOverflow) + } + if len(c.evidence.Attempts) < MaxRequestCostAttempts && !c.evidence.Truncated { + c.evidence.Attempts = append(c.evidence.Attempts, a) + } else { + c.evidence.Truncated = true + c.evidence.DroppedAttempts = SaturatingCostAdd(c.evidence.DroppedAttempts, 1) + } + c.evidence = NormalizeRequestCostEvidence(c.evidence) +} + +func nonDispatchedCostResponse(adapter Adapter, req Request) Response { + if RequestCostSource(adapter) == "" { + return Response{} + } + attempt := RequestCostAttempt{RuntimeID: "openai-api-key", EndpointKind: "custom_or_unverified", TransportMode: "opaque", Dispatch: "not_dispatched", BodyState: "missing", RequestedModel: req.Model} + if req.Fast { + tier := "fast" + attempt.RequestedServiceTier = &tier + } + return Response{RequestCostEvidence: NormalizeRequestCostEvidence(&RequestCostEvidence{Version: 1, Source: "openai_responses", Scope: "invocation", AttemptsObserved: 1, AttemptCountExact: true, Attempts: []RequestCostAttempt{attempt}})} +} + +// RequestCostJSONValid rejects duplicate keys and excessive nesting before a +// bounded telemetry envelope is decoded. Unknown unique fields remain allowed. +func RequestCostJSONValid(data []byte) bool { + if !CostJSONUnicodeValid(data) { + return false + } + d := json.NewDecoder(bytes.NewReader(data)) + d.UseNumber() + if !costJSONValue(d, 0) { + return false + } + _, err := d.Token() + return err == io.EOF +} +func costJSONValue(d *json.Decoder, depth int) bool { + if depth > 64 { + return false + } + token, err := d.Token() + if err != nil { + return false + } + delim, ok := token.(json.Delim) + if !ok { + return true + } + switch delim { + case '{': + seen := map[string]bool{} + for d.More() { + t, err := d.Token() + if err != nil { + return false + } + key, ok := t.(string) + if !ok { + return false + } + folded := costKeyFold(key) + if seen[folded] { + return false + } + seen[folded] = true + if !costJSONValue(d, depth+1) { + return false + } + } + end, err := d.Token() + return err == nil && end == json.Delim('}') + case '[': + for d.More() { + if !costJSONValue(d, depth+1) { + return false + } + } + end, err := d.Token() + return err == nil && end == json.Delim(']') + default: + return false + } +} + +// CostJSONUnicodeValid prevents encoding/json from turning invalid UTF-8 or +// unpaired surrogate escapes into a different, apparently valid identity. It +// does not replace JSON syntax validation and does not unescape provider data. +func CostJSONUnicodeValid(data []byte) bool { + if !utf8.Valid(data) { + return false + } + inString := false + for i := 0; i < len(data); i++ { + if data[i] == '"' { + inString = !inString + continue + } + if !inString || data[i] != '\\' { + continue + } + i++ + if i >= len(data) { + return false + } + if data[i] != 'u' { + continue + } + if i+4 >= len(data) { + return false + } + value, err := strconv.ParseUint(string(data[i+1:i+5]), 16, 16) + if err != nil { + return false + } + i += 4 + if value >= 0xdc00 && value <= 0xdfff { + return false + } + if value >= 0xd800 && value <= 0xdbff { + if i+6 >= len(data) || data[i+1] != '\\' || data[i+2] != 'u' { + return false + } + low, err := strconv.ParseUint(string(data[i+3:i+7]), 16, 16) + if err != nil || low < 0xdc00 || low > 0xdfff { + return false + } + i += 6 + } + } + return true +} + +// encoding/json matches tagged fields with Unicode simple-fold semantics. +// Use the same equivalence classes so alternate casing cannot become a second +// spelling of one field and silently overwrite it during typed decoding. +func costKeyFold(key string) string { + var out strings.Builder + for _, r := range key { + least := r + for next := unicode.SimpleFold(r); next != r; next = unicode.SimpleFold(next) { + if next < least { + least = next + } + } + out.WriteRune(least) + } + return out.String() +} diff --git a/internal/llm/request_cost_evidence_test.go b/internal/llm/request_cost_evidence_test.go new file mode 100644 index 00000000..20746392 --- /dev/null +++ b/internal/llm/request_cost_evidence_test.go @@ -0,0 +1,361 @@ +package llm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "os" + "reflect" + "slices" + "strings" + "testing" +) + +type costFakeAdapter struct { + *FakeAdapter + waits []error +} + +func (a *costFakeAdapter) Queue(r FakeResult) { + a.waits = append(a.waits, r.WaitErr) + r.WaitErr = nil + a.FakeAdapter.Queue(r) +} +func (a *costFakeAdapter) Start(ctx context.Context, req Request) (Stream, error) { + stream, err := a.FakeAdapter.Start(ctx, req) + var waitErr error + if len(a.waits) > 0 { + waitErr = a.waits[0] + a.waits = a.waits[1:] + } + if err != nil { + return nil, err + } + return costEvidenceStream{Stream: stream, waitErr: waitErr}, nil +} + +type costEvidenceStream struct { + Stream + waitErr error +} + +func (s costEvidenceStream) Wait(ctx context.Context) (Response, error) { + response, err := s.Stream.Wait(ctx) + if err != nil { + return response, err + } + return response, s.waitErr +} +func (*costFakeAdapter) RequestCostEvidenceSource() string { return "openai_responses" } +func costPtr[T any](v T) *T { return &v } +func costResponse(id, text string) Response { + return Response{StructuredOutput: []byte(text), RequestCostEvidence: &RequestCostEvidence{ + Version: 1, Source: "openai_responses", Scope: "invocation", AttemptsObserved: 1, AttemptCountExact: true, + Attempts: []RequestCostAttempt{{RuntimeID: "openai-api-key", EndpointKind: "official_global", TransportMode: "single_send_v1", Dispatch: "possibly_dispatched", HTTPStatus: costPtr(200), BodyState: "complete", ProviderResponseID: costPtr(id), ObservedModel: costPtr("observed"), ObservedServiceTier: costPtr("default"), ResponseStatus: costPtr("completed"), RequestedModel: "requested", Usage: &OpenAIRequestUsage{InputTokens: costPtr(int64(10)), OutputTokens: costPtr(int64(3)), CacheReadTokens: costPtr(int64(2)), CacheWriteTokens: costPtr(int64(0)), TotalTokens: costPtr(int64(13))}}}, + }} +} +func costDecode(b []byte) (bool, error) { + if string(b) == `{"ok":true}` { + return true, nil + } + return false, errors.New("invalid synthetic output") +} + +func TestRequestCostCollectorValidationAndTerminalPaths(t *testing.T) { + for _, tc := range []struct { + name string + second FakeResult + callback bool + wantAttempts uint64 + wantErr bool + }{ + {"correction success", FakeResult{Response: costResponse("B", `{"ok":true}`)}, false, 2, false}, + {"two invalid", FakeResult{Response: costResponse("B", `invalid`)}, false, 2, true}, + {"correction start", FakeResult{StartErr: errors.New("start")}, false, 2, true}, + {"correction wait", FakeResult{Response: costResponse("B", ``), WaitErr: errors.New("wait")}, false, 2, true}, + {"correction callback", FakeResult{}, true, 1, true}, + } { + t.Run(tc.name, func(t *testing.T) { + a := &costFakeAdapter{FakeAdapter: &FakeAdapter{}} + first := costResponse("A", `invalid`) + a.Queue(FakeResult{Response: first}) + a.Queue(tc.second) + req := Request{Model: "requested", FreshValidationRetrySession: true} + if tc.callback { + req.OnValidationRetry = func(*Request) error { return errors.New("callback") } + } + got, err := RunStructuredWithSessionResume(context.Background(), a, "", req, costDecode) + if (err != nil) != tc.wantErr { + t.Fatalf("error = %v", err) + } + e := got.Response.RequestCostEvidence + if e == nil || uint64(len(e.Attempts)) != tc.wantAttempts || e.AttemptsObserved != tc.wantAttempts { + t.Fatalf("history = %#v", e) + } + if !tc.wantErr && string(got.AcceptedOutput) != `{"ok":true}` { + t.Fatal("accepted output replaced by initial attempt") + } + if *e.Attempts[0].ProviderResponseID != "A" || e.Attempts[0].Ordinal != 1 || e.Attempts[0].ValidationPhase != "initial" { + t.Fatal("initial receipt lost") + } + if tc.wantAttempts == 2 && (e.Attempts[1].Ordinal != 2 || e.Attempts[1].ValidationPhase != "correction" || e.Attempts[1].AdapterAttempt != 1) { + t.Fatal("phase ordering lost") + } + if tc.name == "correction start" && e.Attempts[1].Dispatch != "not_dispatched" { + t.Fatal("Start failure is not explicit non-dispatch") + } + *first.RequestCostEvidence.Attempts[0].Usage.InputTokens = 999 + *got.ValidationAttempts[0].Response.RequestCostEvidence.Attempts[0].Usage.InputTokens = 777 + if *e.Attempts[0].Usage.InputTokens != 10 { + t.Fatal("history aliases adapter or validation snapshot") + } + }) + } +} + +func TestRequestCostCollectorTransientPhasesAndExhaustion(t *testing.T) { + old := activeRetryPolicy + activeRetryPolicy = retryPolicy{MaxRetries: 2} + t.Cleanup(func() { activeRetryPolicy = old }) + for _, success := range []bool{false, true} { + a := &costFakeAdapter{FakeAdapter: &FakeAdapter{}} + for i := 0; i < 3; i++ { + waitErr := error(ErrTransient) + if success && i == 2 { + waitErr = nil + } + a.Queue(FakeResult{Response: costResponse(fmt.Sprint(i), `{"ok":true}`), WaitErr: waitErr}) + } + got, err := RunStructuredWithSessionResume(context.Background(), a, "", Request{Model: "requested"}, costDecode) + if (err == nil) != success || len(got.Response.RequestCostEvidence.Attempts) != 3 { + t.Fatalf("err=%v history=%#v", err, got.Response.RequestCostEvidence) + } + for i, a := range got.Response.RequestCostEvidence.Attempts { + if a.Ordinal != uint32(i+1) || a.AdapterAttempt != uint32(i+1) { + t.Fatal("attempt budget inferred or reset") + } + } + } +} + +func TestRequestCostCollectorMissingUnsupportedAndCancellation(t *testing.T) { + a := &costFakeAdapter{FakeAdapter: &FakeAdapter{}} + a.Queue(FakeResult{Response: Response{StructuredOutput: []byte(`{"ok":true}`)}}) + got, err := RunStructuredWithSessionResume(context.Background(), a, "", Request{Model: "requested"}, costDecode) + if err != nil || !slices.Contains(got.Response.RequestCostEvidence.Gaps, CostGapMissing) { + t.Fatal("missing expected receipt was omitted") + } + plain := &FakeAdapter{} + plain.Queue(FakeResult{Response: costResponse("ignored", `{"ok":true}`)}) + unsupported, _ := RunStructuredWithSessionResume(context.Background(), plain, "", Request{}, costDecode) + if unsupported.Response.RequestCostEvidence != nil { + t.Fatal("unsupported producer gained collection") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + canceled, _ := RunStructuredWithSessionResume(ctx, &costFakeAdapter{FakeAdapter: &FakeAdapter{}}, "", Request{Model: "requested"}, costDecode) + if len(canceled.Response.RequestCostEvidence.Attempts) != 1 || canceled.Response.RequestCostEvidence.Attempts[0].Dispatch != "not_dispatched" { + t.Fatal("canceled Start omitted") + } +} + +func TestRequestCostBoundsCounterSaturationAndDetachedToolTrace(t *testing.T) { + collector := newRequestCostCollector(&costFakeAdapter{FakeAdapter: &FakeAdapter{}}) + for i := 0; i < 65; i++ { + collector.observe(costResponse(fmt.Sprint(i), ""), "initial", uint32(i+1)) + } + e := collector.snapshot() + if len(e.Attempts) != 64 || e.AttemptsObserved != 65 || e.DroppedAttempts != 1 || !e.Truncated || e.AttemptCountExact { + t.Fatalf("bounds = %#v", e) + } + if *e.Attempts[0].ProviderResponseID != "0" || *e.Attempts[63].ProviderResponseID != "63" { + t.Fatal("retained last success instead of prefix") + } + if SaturatingCostAdd(math.MaxUint64, 1) != math.MaxUint64 { + t.Fatal("counter wrapped") + } + original := costResponse("clone", "") + original.ReviewerToolEvidence = &ReviewerToolEvidence{DiffStatus: DiffToolStatusSucceeded} + cloned := cloneResponse(original) + cloned.RequestCostEvidence.Attempts[0].Issues = append(cloned.RequestCostEvidence.Attempts[0].Issues, CostGapInvalid) + *cloned.RequestCostEvidence.Attempts[0].Usage.CacheWriteTokens = 9 + cloned.ReviewerToolEvidence.DiffStatus = DiffToolStatusFailed + if *original.RequestCostEvidence.Attempts[0].Usage.CacheWriteTokens != 0 || original.ReviewerToolEvidence.DiffStatus != DiffToolStatusSucceeded { + t.Fatal("response ownership regression") + } + data, _ := RequestCostEvidenceBytes(e) + if len(data) > MaxRequestCostEvidenceBytes-RequestCostReservedBytes { + t.Fatal("evidence exceeded budget") + } +} + +func TestRequestCostDecoderMalformedOptionalAndIdentityBounds(t *testing.T) { + for _, raw := range []string{`[]`, `{"version":2}`, `{"version":1,"version":1}`, `{"version":"bad"}`, strings.Repeat(" ", MaxRequestCostEvidenceBytes) + `{}`} { + e := DecodeRequestCostEvidence([]byte(raw)) + if e == nil || e.AttemptCountExact || len(e.Gaps) == 0 { + t.Fatalf("malformed envelope became complete: %s", raw[:min(len(raw), 40)]) + } + } + e := costResponse("safe", "").RequestCostEvidence + e.Attempts[0].ObservedModel = costPtr("malicious\nmodel") + e.Attempts[0].ProviderResponseID = costPtr(strings.Repeat("x", 257)) + n := NormalizeRequestCostEvidence(e) + if n.Attempts[0].ObservedModel != nil || n.Attempts[0].ProviderResponseID != nil { + t.Fatal("invalid identity retained") + } + if e.Attempts[0].ObservedModel == nil { + t.Fatal("normalization mutated input") + } +} + +func TestRequestCostDigestNullZeroAndOrder(t *testing.T) { + e := costResponse("A", "").RequestCostEvidence + e.Attempts[0].Usage.CacheWriteTokens = nil + first, _ := RequestCostEvidenceDigest(e) + e.Attempts[0].Usage.CacheWriteTokens = costPtr(int64(0)) + second, _ := RequestCostEvidenceDigest(e) + if first == second { + t.Fatal("null equals numeric zero") + } + e.Attempts = append(e.Attempts, costResponse("B", "").RequestCostEvidence.Attempts[0]) + e.AttemptsObserved = 2 + third, _ := RequestCostEvidenceDigest(e) + slices.Reverse(e.Attempts) + fourth, _ := RequestCostEvidenceDigest(e) + if third == fourth { + t.Fatal("chronological order absent from digest") + } + clone := CloneRequestCostEvidence(e) + if !reflect.DeepEqual(NormalizeRequestCostEvidence(clone), NormalizeRequestCostEvidence(e)) { + t.Fatal("clone changed values") + } + data, _ := RequestCostEvidenceBytes(e) + var decoded RequestCostEvidence + if json.Unmarshal(data, &decoded) != nil { + t.Fatal("canonical bytes are not JSON") + } +} + +type costBackoffContext struct { + context.Context + canceled bool + done chan struct{} +} + +func (c *costBackoffContext) Done() <-chan struct{} { + if !c.canceled { + c.canceled = true + close(c.done) + } + return c.done +} +func (c *costBackoffContext) Err() error { + if c.canceled { + return context.Canceled + } + return nil +} +func TestRequestCostCancellationInsideBackoffRetainsReceipt(t *testing.T) { + old := activeRetryPolicy + activeRetryPolicy = retryPolicy{MaxRetries: 3, Base: 1_000_000_000, Multiplier: 2} + t.Cleanup(func() { activeRetryPolicy = old }) + ctx := &costBackoffContext{Context: context.Background(), done: make(chan struct{})} + a := &costFakeAdapter{FakeAdapter: &FakeAdapter{}} + a.Queue(FakeResult{Response: costResponse("observed-before-backoff", ``), WaitErr: ErrTransient}) + got, err := RunStructuredWithSessionResume(ctx, a, "", Request{Model: "requested"}, costDecode) + if err == nil || !ctx.canceled || len(a.Requests()) != 1 || len(got.Response.RequestCostEvidence.Attempts) < 1 || *got.Response.RequestCostEvidence.Attempts[0].ProviderResponseID != "observed-before-backoff" { + t.Fatal("canceled backoff erased or retried receipt") + } +} +func TestRequestCostTransientBudgetsAcrossValidationPhases(t *testing.T) { + old := activeRetryPolicy + activeRetryPolicy = retryPolicy{MaxRetries: 1} + t.Cleanup(func() { activeRetryPolicy = old }) + a := &costFakeAdapter{FakeAdapter: &FakeAdapter{}} + for _, r := range []FakeResult{ + {Response: costResponse("1", ""), WaitErr: ErrTransient}, {Response: costResponse("2", "invalid")}, + {Response: costResponse("3", ""), WaitErr: ErrTransient}, {Response: costResponse("4", `{"ok":true}`)}, + } { + a.Queue(r) + } + got, err := RunStructuredWithSessionResume(context.Background(), a, "", Request{Model: "requested", FreshValidationRetrySession: true}, costDecode) + if err != nil || len(got.Response.RequestCostEvidence.Attempts) != 4 || len(got.ValidationAttempts) != 1 { + t.Fatalf("err=%v result=%#v", err, got) + } + for i, r := range got.Response.RequestCostEvidence.Attempts { + if r.Ordinal != uint32(i+1) || r.AdapterAttempt != uint32(i%2+1) { + t.Fatal("phase budget reset global ordinal") + } + } +} + +func TestRequestCostEvidenceCanonicalGolden(t *testing.T) { + want, err := os.ReadFile("testdata/request-cost-evidence-v1.golden.json") + if err != nil { + t.Fatal(err) + } + var e RequestCostEvidence + if json.Unmarshal(want, &e) != nil { + t.Fatal("golden fixture") + } + got, err := RequestCostEvidenceBytes(&e) + if err != nil || string(got) != string(want) { + t.Fatalf("canonical bytes changed: %s %v", got, err) + } + digest, err := RequestCostEvidenceDigest(&e) + if err != nil || digest != "40a0d89467879b363d192692bb0df2599c92fb476b4e1f7db8ad88368ad842d0" { + t.Fatalf("digest=%s err=%v", digest, err) + } +} + +func TestRequestCostRejectsUnicodeReplacementIdentities(t *testing.T) { + for _, raw := range [][]byte{[]byte(`"\ud800"`), []byte(`"\udc00"`), []byte(`"\ud800\u0041"`), {'"', 0xff, '"'}} { + if CostJSONUnicodeValid(raw) { + t.Fatalf("invalid Unicode accepted: %q", raw) + } + } + for _, raw := range []string{`"\ud83d\ude00"`, `"\\ud800"`, `"valid\u0041"`, `"literal �"`} { + if !CostJSONUnicodeValid([]byte(raw)) { + t.Fatalf("valid Unicode rejected: %s", raw) + } + } + if RequestCostJSONValid([]byte(`{"identity":"\ud800"}`)) { + t.Fatal("optional cost decoder could replace identity") + } +} + +func TestRequestCostBaseStreamReturnsDetachedSnapshots(t *testing.T) { + stream := NewBaseStream(func() {}) + source := costResponse("source", `{"ok":true}`) + source.Usage.TokensIn = costPtr(10) + source.ReviewerToolEvidence = &ReviewerToolEvidence{DiffStatus: DiffToolStatusSucceeded, Trace: &ReviewerToolTrace{Version: 1, Source: "pi_rpc", Calls: []ReviewerToolCallEvidence{{CallID: "one"}}}} + stream.Finish(source, nil) + *source.RequestCostEvidence.Attempts[0].Usage.InputTokens = 999 + *source.Usage.TokensIn = 999 + source.ReviewerToolEvidence.Trace.Calls[0].CallID = "mutated" + first, err := stream.Wait(context.Background()) + if err != nil { + t.Fatal(err) + } + *first.RequestCostEvidence.Attempts[0].Usage.InputTokens = 777 + *first.Usage.TokensIn = 777 + first.ReviewerToolEvidence.Trace.Calls[0].CallID = "again" + second, err := stream.Wait(context.Background()) + if err != nil { + t.Fatal(err) + } + if *second.RequestCostEvidence.Attempts[0].Usage.InputTokens != 10 || *second.Usage.TokensIn != 10 || second.ReviewerToolEvidence.Trace.Calls[0].CallID != "one" { + t.Fatal("Finish or Wait retained shared mutable response") + } + results := make(chan Response, 2) + for i := 0; i < 2; i++ { + go func() { r, _ := stream.Wait(context.Background()); results <- r }() + } + a, b := <-results, <-results + *a.RequestCostEvidence.Attempts[0].Usage.InputTokens = 666 + if *b.RequestCostEvidence.Attempts[0].Usage.InputTokens != 10 { + t.Fatal("concurrent Wait copies alias") + } +} diff --git a/internal/llm/subprocess.go b/internal/llm/subprocess.go index e4cb0b2e..07800154 100644 --- a/internal/llm/subprocess.go +++ b/internal/llm/subprocess.go @@ -206,6 +206,7 @@ func (s *BaseStream) Wait(ctx context.Context) (Response, error) { s.mu.Lock() result := s.result + result.response = cloneResponse(result.response) if result.err == nil && ctx.Err() != nil { result.err = ctx.Err() } @@ -281,7 +282,7 @@ func (s *BaseStream) Cleanup() { // Finish publishes a stream result and wakes waiters. func (s *BaseStream) Finish(response Response, err error) { s.mu.Lock() - s.result = streamResult{response: response, err: err} + s.result = streamResult{response: cloneResponse(response), err: err} s.mu.Unlock() close(s.done) } diff --git a/internal/llm/testdata/request-cost-evidence-v1.golden.json b/internal/llm/testdata/request-cost-evidence-v1.golden.json new file mode 100644 index 00000000..953a6aed --- /dev/null +++ b/internal/llm/testdata/request-cost-evidence-v1.golden.json @@ -0,0 +1 @@ +{"version":1,"source":"openai_responses","scope":"invocation","artifact_scope":"","run_id":"","task_id":"","attempts_observed":1,"attempt_count_exact":true,"truncated":false,"dropped_attempts":0,"gaps":[],"attempts":[{"generation_id":"","ordinal":1,"validation_phase":"initial","adapter_attempt":1,"runtime_id":"openai-api-key","endpoint_kind":"official_global","transport_mode":"single_send_v1","dispatch":"not_dispatched","http_status":null,"body_state":"missing","provider_response_id":null,"observed_model":null,"observed_service_tier":null,"response_status":null,"requested_model":"requested\u003cmodel\u003e","requested_service_tier":null,"usage":null,"issues":[]}]} \ No newline at end of file diff --git a/internal/llm/tool_evidence.go b/internal/llm/tool_evidence.go new file mode 100644 index 00000000..e70687fa --- /dev/null +++ b/internal/llm/tool_evidence.go @@ -0,0 +1,95 @@ +package llm + +// ReviewerToolTrace is a bounded record of execution events from one adapter +// invocation. It is telemetry, not proof of meaningful inspection or coverage. +// A nil trace means this adapter did not provide the per-call contract. +type ReviewerToolTrace struct { + Version int `json:"version"` + Source string `json:"source"` + StreamComplete bool `json:"stream_complete"` + Truncated bool `json:"truncated"` + DroppedEvents uint64 `json:"dropped_events"` + Calls []ReviewerToolCallEvidence `json:"calls"` +} + +// ReviewerToolCallStatus describes an observed, correlated execution outcome. +type ReviewerToolCallStatus string + +const ( + // ReviewerToolCallIncomplete means the trace cannot establish an outcome. + ReviewerToolCallIncomplete ReviewerToolCallStatus = "incomplete" + // ReviewerToolCallSucceeded means a matched execution end explicitly succeeded. + ReviewerToolCallSucceeded ReviewerToolCallStatus = "succeeded" + // ReviewerToolCallFailed means a matched execution end explicitly failed. + ReviewerToolCallFailed ReviewerToolCallStatus = "failed" +) + +// ReviewerToolCallEvidence retains only identity, request range and outcome. +// Path is the unmodified, canonical repository-relative cr_read argument, not +// an assignment or a claim that the entire file was read. cr_diff has no path. +// Missing/ambiguous observations are explicit; raw arguments and output are +// deliberately not copied into metadata. +type ReviewerToolCallEvidence struct { + CallID string `json:"call_id,omitempty"` + Tool string `json:"tool"` + Path string `json:"path,omitempty"` + ReadView ReviewerToolReadView `json:"read_view,omitempty"` + Status ReviewerToolCallStatus `json:"status"` + StartObserved bool `json:"start_observed"` + EndObserved bool `json:"end_observed"` + ProvenanceIssue string `json:"provenance_issue,omitempty"` + Range *ReviewerToolRange `json:"requested_range,omitempty"` + Output *ReviewerToolOutput `json:"output,omitempty"` +} + +// ReviewerToolReadView distinguishes ordinary file-body requests from pinned +// symlink metadata. Neither view establishes complete or meaningful inspection. +type ReviewerToolReadView string + +const ( + // ReviewerToolReadFile is an ordinary cr_read request without a special view. + ReviewerToolReadFile ReviewerToolReadView = "file" + // ReviewerToolReadSymlink is a cr_read view=symlink request. Its ranges cover + // pinned metadata, not the symlink destination or a head regular-file body. + ReviewerToolReadSymlink ReviewerToolReadView = "symlink" +) + +// ReviewerToolRange is the requested byte range. Zero values select the tool's +// default bounded read; they do not establish that all bytes were returned. +type ReviewerToolRange struct { + Offset int64 `json:"offset"` + Limit int64 `json:"limit"` +} + +// ReviewerToolOutput describes transport bounds reported by the CR extension. +// A nil output means unknown, not untruncated. Even Truncated=false says nothing +// about whether the helper returned only a range of the underlying file/diff. +type ReviewerToolOutput struct { + Bytes int64 `json:"bytes"` + Truncated bool `json:"truncated"` +} + +// CloneReviewerToolEvidence prevents mutable traces from crossing attempt or +// lifecycle snapshot boundaries. +func CloneReviewerToolEvidence(evidence *ReviewerToolEvidence) *ReviewerToolEvidence { + if evidence == nil { + return nil + } + cloned := *evidence + if evidence.Trace != nil { + trace := *evidence.Trace + trace.Calls = append([]ReviewerToolCallEvidence{}, evidence.Trace.Calls...) + for i := range trace.Calls { + if value := trace.Calls[i].Range; value != nil { + copiedRange := *value + trace.Calls[i].Range = &copiedRange + } + if value := trace.Calls[i].Output; value != nil { + copiedOutput := *value + trace.Calls[i].Output = &copiedOutput + } + } + cloned.Trace = &trace + } + return &cloned +} diff --git a/internal/llm/tool_evidence_test.go b/internal/llm/tool_evidence_test.go new file mode 100644 index 00000000..d18b7421 --- /dev/null +++ b/internal/llm/tool_evidence_test.go @@ -0,0 +1,30 @@ +package llm + +import ( + "reflect" + "testing" +) + +func TestCloneResponseSnapshotsReviewerToolTrace(t *testing.T) { + evidence := &ReviewerToolEvidence{DiffStatus: DiffToolStatusSucceeded, Trace: &ReviewerToolTrace{ + Version: 1, Source: "pi_rpc", Calls: []ReviewerToolCallEvidence{{ + CallID: "read", Tool: "cr_read", Path: "src/file.go", Status: ReviewerToolCallSucceeded, + Range: &ReviewerToolRange{Offset: 20, Limit: 50}, Output: &ReviewerToolOutput{Bytes: 50}, + }}, + }} + cloned := cloneResponse(Response{ReviewerToolEvidence: evidence}).ReviewerToolEvidence + if !reflect.DeepEqual(cloned, evidence) { + t.Fatalf("clone = %#v, want %#v", cloned, evidence) + } + cloned.DiffStatus = DiffToolStatusFailed + cloned.Trace.Truncated = true + cloned.Trace.Calls[0].Path = "different.go" + cloned.Trace.Calls[0].Range.Offset = 100 + cloned.Trace.Calls[0].Output.Truncated = true + if evidence.DiffStatus != DiffToolStatusSucceeded || evidence.Trace.Truncated || evidence.Trace.Calls[0].Path != "src/file.go" || evidence.Trace.Calls[0].Range.Offset != 20 || evidence.Trace.Calls[0].Output.Truncated { + t.Fatalf("clone mutated original: %#v", evidence) + } + if CloneReviewerToolEvidence(nil) != nil { + t.Fatal("absent evidence must stay absent") + } +} diff --git a/internal/llmadapters/api.go b/internal/llmadapters/api.go index bee29171..41c564b5 100644 --- a/internal/llmadapters/api.go +++ b/internal/llmadapters/api.go @@ -22,6 +22,8 @@ import ( // ErrAPIAdapterConfig reports invalid direct API adapter configuration. var ErrAPIAdapterConfig = errors.New("llm api: invalid configuration") +var errAPIResponseOverLimit = errors.New("llm api: response body exceeds limit") + const ( defaultAnthropicBaseURL = "https://api.anthropic.com/" defaultOpenAIBaseURL = "https://api.openai.com/" @@ -50,13 +52,14 @@ type APIOptions struct { // APIAdapter calls a direct provider HTTP API as an LLM adapter. type APIAdapter struct { - kind apiKind - apiKey string - httpClient *http.Client - baseURL *url.URL - maxTokens int - anthropicVersion string - fastModeModels []string + kind apiKind + apiKey string + httpClient *http.Client + ownedOpenAIClient *http.Client + baseURL *url.URL + maxTokens int + anthropicVersion string + fastModeModels []string } var _ llm.Adapter = (*APIAdapter)(nil) @@ -132,21 +135,28 @@ func newAPIAdapter(kind apiKind, opts APIOptions) (*APIAdapter, error) { return nil, err } httpClient := opts.HTTPClient + var ownedOpenAIClient *http.Client if httpClient == nil { - httpClient = &http.Client{Timeout: defaultAPIClientTimeout} + if kind == apiOpenAI { + ownedOpenAIClient = newSingleSendOpenAIClient() + httpClient = ownedOpenAIClient + } else { + httpClient = &http.Client{Timeout: defaultAPIClientTimeout} + } } version := strings.TrimSpace(opts.AnthropicVersion) if version == "" { version = defaultAnthropicVersion } return &APIAdapter{ - kind: kind, - apiKey: opts.APIKey, - httpClient: httpClient, - baseURL: baseURL, - maxTokens: maxTokens, - anthropicVersion: version, - fastModeModels: append([]string(nil), opts.FastModeModels...), + kind: kind, + apiKey: opts.APIKey, + httpClient: httpClient, + ownedOpenAIClient: ownedOpenAIClient, + baseURL: baseURL, + maxTokens: maxTokens, + anthropicVersion: version, + fastModeModels: append([]string(nil), opts.FastModeModels...), }, nil } @@ -229,18 +239,35 @@ func (s *apiStream) run(ctx context.Context, adapter *APIAdapter, req Request) { s.Finish(response, err) } -func (a *APIAdapter) execute(ctx context.Context, req Request) (string, Response, error) { +func (a *APIAdapter) execute(ctx context.Context, req Request) (sessionID string, response Response, resultErr error) { + cost := a.newCostAttempt(req) + defer func() { response.RequestCostEvidence = singleCostEvidence(cost) }() endpoint, requestBody, err := a.buildProviderRequest(req) if err != nil { return "", Response{}, err } + if cost != nil && officialResponsesEndpoint(endpoint) { + cost.EndpointKind = "official_global" + } httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(requestBody)) if err != nil { return "", Response{}, err } a.applyHeaders(httpReq, req) - + if cost != nil { + // Clearing GetBody blocks consumed-body transport/redirect replay for + // the owned client. Injected clients remain explicitly opaque. + httpReq.GetBody = nil + if err := ctx.Err(); err != nil { + return "", Response{}, err + } + cost.Dispatch = "possibly_dispatched" + } httpResp, err := a.httpClient.Do(httpReq) + if cost != nil && httpResp != nil { + status := httpResp.StatusCode + cost.HTTPStatus = &status + } if err != nil { if isTransientTransportError(err) { return "", Response{}, fmt.Errorf("%w: %w", ErrTransient, err) @@ -248,10 +275,24 @@ func (a *APIAdapter) execute(ctx context.Context, req Request) (string, Response return "", Response{}, err } defer httpResp.Body.Close() + if cost != nil { + status := httpResp.StatusCode + cost.HTTPStatus = &status + } responseBody, err := readAPIResponseBody(httpResp.Body) if err != nil { + if cost != nil { + cost.BodyState = "read_error" + if errors.Is(err, errAPIResponseOverLimit) { + cost.BodyState = "over_limit" + } + } return "", Response{}, err } + if cost != nil { + cost.BodyState = "complete" + observeCostEnvelope(cost, responseBody) + } if httpResp.StatusCode < 200 || httpResp.StatusCode > 299 { statusErr := fmt.Errorf("llm api %s: provider returned %s", a.kind, httpResp.Status) if classifyHTTPStatusTransient(httpResp.StatusCode) { @@ -449,7 +490,8 @@ type openAIUsage struct { } type openAIInputTokensDetails struct { - CachedTokens *int `json:"cached_tokens"` + CachedTokens *int `json:"cached_tokens"` + CacheWriteTokens *int `json:"cache_write_tokens"` } func parseOpenAIResponse(body []byte) (string, Response, error) { @@ -468,9 +510,6 @@ func parseOpenAIResponse(body []byte) (string, Response, error) { if text.Len() == 0 && payload.OutputText != "" { text.WriteString(payload.OutputText) } - if text.Len() == 0 { - return payload.ID, Response{}, errors.New("llm api openai_api: no text output") - } speed := "" switch strings.ToLower(strings.TrimSpace(payload.ServiceTier)) { case "fast", "priority": @@ -478,15 +517,22 @@ func parseOpenAIResponse(body []byte) (string, Response, error) { case "standard", "default": speed = "standard" } - return payload.ID, Response{ + // OpenAI's input total includes cache reads and writes. Keep that total + // intact rather than treating cache writes as an additional input bucket. + response := Response{ StructuredOutput: []byte(text.String()), Usage: Usage{ - TokensIn: payload.Usage.InputTokens, - TokensOut: payload.Usage.OutputTokens, - CacheRead: payload.Usage.InputTokensDetails.CachedTokens, - Speed: speed, + TokensIn: payload.Usage.InputTokens, + TokensOut: payload.Usage.OutputTokens, + CacheRead: payload.Usage.InputTokensDetails.CachedTokens, + CacheCreate: payload.Usage.InputTokensDetails.CacheWriteTokens, + Speed: speed, }, - }, nil + } + if text.Len() == 0 { + return payload.ID, response, errors.New("llm api openai_api: no text output") + } + return payload.ID, response, nil } func readAPIResponseBody(reader io.Reader) ([]byte, error) { @@ -495,7 +541,7 @@ func readAPIResponseBody(reader io.Reader) ([]byte, error) { return nil, err } if len(body) > apiResponseLogLimit { - return nil, errors.New("llm api: response body exceeds limit") + return nil, errAPIResponseOverLimit } return body, nil } diff --git a/internal/llmadapters/api_test.go b/internal/llmadapters/api_test.go index 27b0a137..968561a1 100644 --- a/internal/llmadapters/api_test.go +++ b/internal/llmadapters/api_test.go @@ -16,6 +16,8 @@ import ( "github.com/open-cli-collective/codereview-cli/internal/config" "github.com/open-cli-collective/codereview-cli/internal/credentials" + "github.com/open-cli-collective/codereview-cli/internal/modelcatalog" + "github.com/open-cli-collective/codereview-cli/internal/pricing" ) func TestAnthropicAPIAdapterRequestAndResponse(t *testing.T) { @@ -234,6 +236,111 @@ func TestOpenAIAPIAdapterFastRequest(t *testing.T) { } } +func TestParseOpenAIResponseCacheWrites(t *testing.T) { + zero, five, seven, twelve, negative := 0, 5, 7, 12, -3 + for _, tt := range []struct { + name string + details string + want *int + wantRead *int + }{ + {name: "missing details"}, + {name: "null details", details: `,"input_tokens_details":null`}, + {name: "missing count", details: `,"input_tokens_details":{"cached_tokens":5}`, wantRead: &five}, + {name: "null count", details: `,"input_tokens_details":{"cached_tokens":5,"cache_write_tokens":null}`, wantRead: &five}, + {name: "zero count", details: `,"input_tokens_details":{"cached_tokens":5,"cache_write_tokens":0}`, want: &zero, wantRead: &five}, + {name: "positive count", details: `,"input_tokens_details":{"cached_tokens":5,"cache_write_tokens":7}`, want: &seven, wantRead: &five}, + {name: "negative count retained", details: `,"input_tokens_details":{"cached_tokens":5,"cache_write_tokens":-3}`, want: &negative, wantRead: &five}, + {name: "overlapping counts retained", details: `,"input_tokens_details":{"cached_tokens":5,"cache_write_tokens":12}`, want: &twelve, wantRead: &five}, + {name: "unknown field ignored", details: `,"input_tokens_details":{"cached_tokens":5,"cache_write_tokens":7,"future_detail":{"count":9}}`, want: &seven, wantRead: &five}, + } { + t.Run(tt.name, func(t *testing.T) { + body := fmt.Sprintf(`{"id":"resp-cache","service_tier":"default","output_text":"{}","usage":{"input_tokens":16,"output_tokens":3%s}}`, tt.details) + _, response, err := parseOpenAIResponse([]byte(body)) + if err != nil { + t.Fatalf("parseOpenAIResponse: %v", err) + } + got := response.Usage.CacheCreate + if (got == nil) != (tt.want == nil) || got != nil && *got != *tt.want { + t.Fatalf("CacheCreate = %v, want %v", got, tt.want) + } + read := response.Usage.CacheRead + if (read == nil) != (tt.wantRead == nil) || read != nil && *read != *tt.wantRead { + t.Fatalf("CacheRead = %v, want %v", read, tt.wantRead) + } + // OpenAI input is inclusive of cache reads and writes. Preserve the + // provider's total. Raw negative or overlapping counts are not + // proof of billable usage; a future estimator must validate them. + if response.Usage.TokensIn == nil || *response.Usage.TokensIn != 16 || + response.Usage.TokensOut == nil || *response.Usage.TokensOut != 3 { + t.Fatalf("Usage = %#v, want unchanged input/output totals", response.Usage) + } + if response.Usage.CacheCreate5m != nil || response.Usage.CacheCreate1h != nil || response.Usage.CostUSD != nil { + t.Fatalf("Usage = %#v, want unknown TTL buckets and cost", response.Usage) + } + if response.Usage.Speed != "standard" { + t.Fatalf("Speed = %q, want observed standard tier", response.Usage.Speed) + } + }) + } +} + +func TestParseOpenAIResponseRejectsMalformedCacheWriteCount(t *testing.T) { + for _, count := range []string{`"7"`, `1.5`, `true`, `{}`, `[]`, `9223372036854775808`} { + t.Run(count, func(t *testing.T) { + body := fmt.Sprintf(`{"output_text":"{}","usage":{"input_tokens_details":{"cache_write_tokens":%s}}}`, count) + if _, _, err := parseOpenAIResponse([]byte(body)); err == nil { + t.Fatal("expected malformed cache-write count to fail response decoding") + } + }) + } +} + +func TestParsedOpenAIUsageLeavesBundledEstimatesUnavailable(t *testing.T) { + catalog, err := modelcatalog.LoadBundled() + if err != nil { + t.Fatalf("LoadBundled: %v", err) + } + for _, count := range []string{`null`, `0`, `7`} { + t.Run(count, func(t *testing.T) { + body := fmt.Sprintf(`{"service_tier":"default","output_text":"{}","usage":{"input_tokens":16,"output_tokens":3,"input_tokens_details":{"cached_tokens":5,"cache_write_tokens":%s}}}`, count) + _, response, err := parseOpenAIResponse([]byte(body)) + if err != nil { + t.Fatalf("parseOpenAIResponse: %v", err) + } + usage := response.Usage + _, ok := pricing.EstimateUsageUSDFor(catalog, "gpt-6.1-sol", pricing.Usage{ + TokensIn: usage.TokensIn, + TokensOut: usage.TokensOut, + CacheRead: usage.CacheRead, + CacheCreateTotal: usage.CacheCreate, + CacheCreate5m: usage.CacheCreate5m, + CacheCreate1h: usage.CacheCreate1h, + Speed: usage.Speed, + }) + if ok { + t.Fatal("parsed cache-write telemetry must not activate bundled context-banded estimates") + } + }) + } +} + +func TestParseAnthropicResponseKeepsSeparateCacheBuckets(t *testing.T) { + _, response, err := parseAnthropicResponse([]byte(`{"content":[{"type":"text","text":"{}"}],"usage":{"input_tokens":16,"output_tokens":3,"cache_read_input_tokens":5,"cache_creation_input_tokens":7,"cache_creation":{"ephemeral_5m_input_tokens":3,"ephemeral_1h_input_tokens":4},"input_tokens_details":{"cached_tokens":100,"cache_write_tokens":200}}}`)) + if err != nil { + t.Fatalf("parseAnthropicResponse: %v", err) + } + assertIntPtr(t, "TokensIn", response.Usage.TokensIn, 16) + assertIntPtr(t, "TokensOut", response.Usage.TokensOut, 3) + assertIntPtr(t, "CacheRead", response.Usage.CacheRead, 5) + assertIntPtr(t, "CacheCreate", response.Usage.CacheCreate, 7) + assertIntPtr(t, "CacheCreate5m", response.Usage.CacheCreate5m, 3) + assertIntPtr(t, "CacheCreate1h", response.Usage.CacheCreate1h, 4) + if response.Usage.CostUSD != nil { + t.Fatal("cache usage must not become a reported cost") + } +} + func TestParseOpenAIResponseSpeed(t *testing.T) { for _, tt := range []struct { name string diff --git a/internal/llmadapters/pi_rpc.go b/internal/llmadapters/pi_rpc.go index a61d5cd7..43ff337b 100644 --- a/internal/llmadapters/pi_rpc.go +++ b/internal/llmadapters/pi_rpc.go @@ -14,6 +14,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/open-cli-collective/codereview-cli/internal/llm" @@ -27,9 +28,14 @@ const ( piRPCReviewerToolNames = "cr_read,cr_search,cr_list,cr_diff" piRPCToolEvidenceReserve = 256 piRPCToolDiagnosticMaxRunes = 128 + piRPCErrorMaxRunes = 512 piRPCPreflightTimeout = 5 * time.Second piRPCPreflightOutputBytes = 64 * 1024 piRPCPreflightRegistration = "codereview-pi-reviewer-tools-registered cr_read,cr_search,cr_list,cr_diff" + // piRPCSettlementGrace bounds the quiet period after a run boundary that + // does not announce more work. Pi 1.0 follows it with agent_settled; older + // runtimes never do and would otherwise idle until the task deadline. + piRPCSettlementGrace = 30 * time.Second ) // ErrPiRPCIncompatible reports that the installed Pi runtime cannot enforce @@ -54,6 +60,7 @@ type PiRPCAdapter struct { timeout time.Duration scratchDirFactory ScratchDirFactory fastModeModels []string + settlementGrace time.Duration preflightMu sync.Mutex preflightReady bool } @@ -84,6 +91,7 @@ func NewPiRPCAdapter(opts PiRPCOptions) *PiRPCAdapter { timeout: timeout, scratchDirFactory: factory, fastModeModels: append([]string(nil), opts.FastModeModels...), + settlementGrace: piRPCSettlementGrace, } } @@ -165,6 +173,7 @@ func (a *PiRPCAdapter) Start(ctx context.Context, req Request) (Stream, error) { stdin: process.Stdin(), allowReviewerTools: req.ReviewerWorkspace != nil, logBytesLeft: -1, + settlementGrace: a.settlementGrace, } if req.ReviewerWorkspace != nil { stream.toolEvidenceBytesLeft = min(piRPCToolEvidenceReserve, req.ReviewerWorkspace.MaxToolOutputBytes) @@ -232,6 +241,7 @@ func (a *PiRPCAdapter) preflightReviewerRuntime(parent context.Context) error { args := append(append([]string(nil), a.commandArgsPrefix...), "--mode", "rpc", "--system-prompt", piRPCReviewerSystemPrompt, + "--append-system-prompt", "", "--no-builtin-tools", "--tools", piRPCReviewerToolNames, "--extension", extensionPath, @@ -392,6 +402,13 @@ func (a *PiRPCAdapter) buildArgs(req Request, extensionPath string) ([]string, e args := []string{ "--mode", "rpc", "--system-prompt", systemPrompt, + // An explicit empty CLI append source suppresses APPEND_SYSTEM.md + // discovery in Pi 1.0. --no-context-files separately suppresses + // AGENTS.md/CLAUDE.md; --system-prompt overrides SYSTEM.md discovery. + // Keep the selected agent directory intact so authentication refreshes + // continue to persist in the user's original credential store. + "--append-system-prompt", "", + "--no-context-files", "--no-extensions", "--no-skills", "--no-prompt-templates", @@ -402,7 +419,7 @@ func (a *PiRPCAdapter) buildArgs(req Request, extensionPath string) ([]string, e args = append(args, "--no-tools") } else { args[3] = piRPCReviewerSystemPrompt - args = append(args, "--no-builtin-tools", "--no-context-files", "--no-approve", "--tools", piRPCReviewerToolNames, "--extension", extensionPath) + args = append(args, "--no-builtin-tools", "--no-approve", "--tools", piRPCReviewerToolNames, "--extension", extensionPath) } if req.Model != "" { args = append(args, "--model", req.Model) @@ -415,21 +432,22 @@ func (a *PiRPCAdapter) buildArgs(req Request, extensionPath string) ([]string, e func (a *PiRPCAdapter) validateArgs(args []string, req Request, extensionPath string) error { allowedFlags := map[string]bool{ - "--mode": true, - "--system-prompt": true, - "--no-tools": false, - "--no-builtin-tools": false, - "--no-context-files": false, - "--no-approve": false, - "--tools": true, - "--no-extensions": false, - "--extension": true, - "--no-skills": false, - "--no-prompt-templates": false, - "--no-themes": false, - "--no-session": false, - "--model": true, - "--thinking": true, + "--mode": true, + "--system-prompt": true, + "--append-system-prompt": true, + "--no-tools": false, + "--no-builtin-tools": false, + "--no-context-files": false, + "--no-approve": false, + "--tools": true, + "--no-extensions": false, + "--extension": true, + "--no-skills": false, + "--no-prompt-templates": false, + "--no-themes": false, + "--no-session": false, + "--model": true, + "--thinking": true, } if err := validateAllowedFlags("pi_rpc", args, allowedFlags); err != nil { return err @@ -442,11 +460,14 @@ func (a *PiRPCAdapter) validateArgs(args []string, req Request, extensionPath st if flagValue(args, "--mode") != "rpc" { return fmt.Errorf("%w: pi_rpc must use rpc mode", ErrUnsafeSubprocessConfig) } - for _, flag := range []string{"--system-prompt", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-themes", "--no-session"} { + for _, flag := range []string{"--system-prompt", "--append-system-prompt", "--no-context-files", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-themes", "--no-session"} { if !containsFlag(args, flag) { return fmt.Errorf("%w: missing %s", ErrUnsafeSubprocessConfig, flag) } } + if appendPrompt, ok := flagValueOK(args, "--append-system-prompt"); !ok || appendPrompt != "" { + return fmt.Errorf("%w: pi_rpc must use an explicit empty append system prompt", ErrUnsafeSubprocessConfig) + } wantSystemPrompt := piRPCSystemPrompt if req.ReviewerWorkspace == nil { if !containsFlag(args, "--no-tools") || containsFlag(args, "--no-builtin-tools") || containsFlag(args, "--tools") || containsFlag(args, "-t") || containsFlag(args, "--extension") { @@ -495,6 +516,7 @@ type piRPCStream struct { baseStream stdin io.Closer allowReviewerTools bool + settlementGrace time.Duration logLimitMu sync.Mutex logBytesLeft int logCapped bool @@ -503,6 +525,7 @@ type piRPCStream struct { diffToolCompleted int diffToolFailed int diffToolError string + toolTrace piRPCReviewerToolTrace } func (s *piRPCStream) run(ctx context.Context, cmd *exec.Cmd, stdout io.Reader, stderr io.Reader) { @@ -518,34 +541,45 @@ func (s *piRPCStream) run(ctx context.Context, cmd *exec.Cmd, stdout io.Reader, }() scanResult := s.scanStdout(stdout) + // Closing stdin asks Pi to shut down, so it happens only after settlement + // or a terminal failure. Keep reading stdout so shutdown never blocks on a + // full pipe. if s.stdin != nil { _ = s.stdin.Close() } + stdoutDone := make(chan struct{}) + go func() { + defer close(stdoutDone) + _, _ = io.Copy(piRPCLogWriter{stream: s}, stdout) + }() waitErr := cmd.Wait() + <-stdoutDone <-stderrDone - evidence := s.reviewerToolEvidence() - s.writeReviewerToolEvidence(evidence) - scanResult.response.ReviewerToolEvidence = evidence - result := subprocessResult{response: scanResult.response} switch { case scanResult.err != nil: result.err = scanResult.err case errors.Is(ctx.Err(), context.DeadlineExceeded): result.err = ctx.Err() - case scanResult.agentEnd && waitErr != nil: + case scanResult.settled && waitErr != nil: result.err = waitErr - case scanResult.agentEnd && len(scanResult.response.StructuredOutput) == 0: + case scanResult.settled && len(scanResult.response.StructuredOutput) == 0: result.err = errors.New("llm pi rpc: no structured output") - case scanResult.agentEnd: + case scanResult.settled: result.err = nil case ctx.Err() != nil: result.err = ctx.Err() case waitErr != nil: result.err = waitErr - case !scanResult.agentEnd: - result.err = errors.New("llm pi rpc: missing agent_end") + default: + result.err = errors.New("llm pi rpc: stream ended before agent_settled") } + // Settlement is the protocol boundary, but a terminal error or canceled + // context must not promote partial execution evidence to a complete stream. + s.toolTrace.streamComplete = scanResult.settled && result.err == nil && ctx.Err() == nil + evidence := s.reviewerToolEvidence() + s.writeReviewerToolEvidence(evidence) + result.response.ReviewerToolEvidence = evidence recordRequestDuration(&result.response, start, result.err) s.Cancel() s.CloseProcessGroup() @@ -557,14 +591,26 @@ func (s *piRPCStream) run(ctx context.Context, cmd *exec.Cmd, stdout io.Reader, type piRPCScanResult struct { response Response err error - agentEnd bool + settled bool } +// scanStdout consumes RPC records until agent_settled, the only boundary after +// which Pi does no further automatic work. agent_end can be followed by +// retries, compaction recovery, or queued work, and a successful prompt +// response only acknowledges acceptance. func (s *piRPCStream) scanStdout(stdout io.Reader) piRPCScanResult { scanner := bufio.NewScanner(stdout) scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) var result piRPCScanResult + var final *piRPCAssistantMessage + var retryFailure string + settlement := piRPCSettlementWatch{grace: s.settlementGrace, expire: s.Cancel} + defer settlement.stop() for scanner.Scan() { + if settlement.expired.Load() { + break + } + settlement.stop() line := append([]byte(nil), scanner.Bytes()...) s.writeLog(normalizePiRPCLogLine(line)) event, err := parsePiRPCEvent(line) @@ -587,14 +633,30 @@ func (s *piRPCStream) scanStdout(stdout io.Reader) piRPCScanResult { result.err = fmt.Errorf("llm pi rpc: prompt failed: %s", event.responseFailure) return result } - if len(event.structuredOutput) > 0 { - result.response.StructuredOutput = event.structuredOutput + if event.promptHandled { + s.Cancel() + result.err = errors.New("llm pi rpc: prompt was handled without starting a run") + return result + } + result.response.Usage = addPiRPCUsage(result.response.Usage, event.usage) + if event.assistant != nil { + final = event.assistant } - result.response.Usage = mergeUsage(result.response.Usage, event.usage) - if event.agentEnd { - result.agentEnd = true + if event.retryFailure != "" { + retryFailure = event.retryFailure + } + if event.settled { + result.settled = true + result.response.StructuredOutput, result.err = piRPCFinalAnswer(final, retryFailure) return result } + if event.awaitsSettlement { + settlement.arm() + } + } + if settlement.expired.Load() { + result.err = fmt.Errorf("%w: no agent_settled within %s after the run ended; CR requires the Pi 1.0 RPC completion contract", ErrPiRPCIncompatible, s.settlementGrace) + return result } if err := scanner.Err(); err != nil && result.err == nil { result.err = err @@ -602,6 +664,80 @@ func (s *piRPCStream) scanStdout(stdout io.Reader) piRPCScanResult { return result } +// piRPCFinalAnswer accepts only the last completed assistant message, so text +// from an earlier turn or failed attempt never stands in for the answer. +func piRPCFinalAnswer(final *piRPCAssistantMessage, retryFailure string) ([]byte, error) { + switch { + case final == nil: + return nil, nil + case final.stopReason == "error" || final.stopReason == "aborted": + diagnostic := final.errorMessage + if diagnostic == "" { + diagnostic = retryFailure + } + if diagnostic == "" { + diagnostic = "no error message reported" + } + return nil, fmt.Errorf("llm pi rpc: final assistant message %s: %s", final.stopReason, boundPiRPCDiagnostic(diagnostic, piRPCErrorMaxRunes)) + case final.stopReason == "toolUse": + return nil, errors.New("llm pi rpc: run settled without a final answer after tool use") + default: + return final.output, nil + } +} + +// piRPCSettlementWatch fails a run that goes quiet after a boundary which +// announced no further work but never reports agent_settled. +type piRPCSettlementWatch struct { + grace time.Duration + expire func() + timer *time.Timer + expired atomic.Bool +} + +func (w *piRPCSettlementWatch) arm() { + if w.grace <= 0 { + return + } + w.timer = time.AfterFunc(w.grace, func() { + w.expired.Store(true) + w.expire() + }) +} + +func (w *piRPCSettlementWatch) stop() { + if w.timer != nil { + w.timer.Stop() + w.timer = nil + } +} + +// addPiRPCUsage adds one completed usage report to the run total. Fields that +// no report has provided stay nil (unknown) rather than becoming zero. +func addPiRPCUsage(total Usage, next Usage) Usage { + total.TokensIn = addPiRPCUsageField(total.TokensIn, next.TokensIn) + total.TokensOut = addPiRPCUsageField(total.TokensOut, next.TokensOut) + total.CacheRead = addPiRPCUsageField(total.CacheRead, next.CacheRead) + total.CacheCreate = addPiRPCUsageField(total.CacheCreate, next.CacheCreate) + total.CacheCreate5m = addPiRPCUsageField(total.CacheCreate5m, next.CacheCreate5m) + total.CacheCreate1h = addPiRPCUsageField(total.CacheCreate1h, next.CacheCreate1h) + total.CostUSD = addPiRPCUsageField(total.CostUSD, next.CostUSD) + return total +} + +func addPiRPCUsageField[T int | float64](total *T, next *T) *T { + switch { + case next == nil: + return total + case total == nil: + value := *next + return &value + default: + value := *total + *next + return &value + } +} + type piRPCLogWriter struct{ stream *piRPCStream } func (w piRPCLogWriter) Write(p []byte) (int, error) { @@ -644,7 +780,11 @@ func (s *piRPCStream) writeLog(p []byte) { } func (s *piRPCStream) observeReviewerToolEvent(event piRPCEvent) { - if !s.allowReviewerTools || event.toolName != "cr_diff" { + if !s.allowReviewerTools { + return + } + s.toolTrace.observe(event) + if event.toolName != "cr_diff" { return } if event.toolStarted { @@ -674,9 +814,9 @@ func (s *piRPCStream) reviewerToolEvidence() *llm.ReviewerToolEvidence { case s.diffToolStarted > 0: status = llm.DiffToolStatusIncomplete } - evidence := &llm.ReviewerToolEvidence{DiffStatus: status} + evidence := &llm.ReviewerToolEvidence{DiffStatus: status, Trace: s.toolTrace.snapshot()} if status == llm.DiffToolStatusFailed { - evidence.DiffDiagnostic = boundPiRPCToolError(s.diffToolError) + evidence.DiffDiagnostic = boundPiRPCDiagnostic(s.diffToolError, piRPCToolDiagnosticMaxRunes) } return evidence } @@ -699,13 +839,13 @@ func (s *piRPCStream) writeReviewerToolEvidence(evidence *llm.ReviewerToolEviden } } -func boundPiRPCToolError(value string) string { +func boundPiRPCDiagnostic(value string, maxRunes int) string { value = strings.Join(strings.Fields(strings.TrimSpace(value)), " ") runes := []rune(value) - if len(runes) <= piRPCToolDiagnosticMaxRunes { + if len(runes) <= maxRunes { return value } - return string(runes[:piRPCToolDiagnosticMaxRunes-3]) + "..." + return string(runes[:maxRunes-3]) + "..." } func piRPCToolError(raw json.RawMessage) string { @@ -878,7 +1018,7 @@ func compactPiRPCPartialForLog(partialRaw json.RawMessage) json.RawMessage { type piRPCEvent struct { sessionID string - structuredOutput []byte + assistant *piRPCAssistantMessage usage Usage toolUse bool toolName string @@ -886,8 +1026,23 @@ type piRPCEvent struct { toolCompleted bool toolFailed bool toolError string + toolCallID string + toolArgs json.RawMessage + toolOutcomeKnown bool + toolOutput *llm.ReviewerToolOutput responseFailure string - agentEnd bool + promptHandled bool + retryFailure string + awaitsSettlement bool + settled bool +} + +// piRPCAssistantMessage is the authoritative message_end form of one +// completed assistant message. +type piRPCAssistantMessage struct { + output []byte + stopReason string + errorMessage string } func parsePiRPCEvent(line []byte) (piRPCEvent, error) { @@ -902,20 +1057,23 @@ func parsePiRPCEvent(line []byte) (piRPCEvent, error) { eventType := rawString(raw, "type") event := piRPCEvent{ toolUse: piRPCEventIndicatesToolUse(eventType) || valueIndicatesToolUse(decoded), - usage: parsePiRPCUsage(raw), } if event.toolUse { event.toolName = firstRawString(raw, "toolName", "tool_name", "name") + event.toolCallID = piRPCToolCallID(raw) switch eventType { case "tool_execution_start": event.toolStarted = true + event.toolArgs = raw["args"] case "tool_execution_end": event.toolCompleted = true event.toolFailed = rawBool(raw, "isError") var resultRaw map[string]json.RawMessage - if err := json.Unmarshal(raw["result"], &resultRaw); err == nil && rawBool(resultRaw, "isError") { - event.toolFailed = true + if err := json.Unmarshal(raw["result"], &resultRaw); err == nil { + event.toolFailed = event.toolFailed || rawBool(resultRaw, "isError") + event.toolOutput = parsePiRPCToolOutput(resultRaw["details"]) } + event.toolOutcomeKnown = piRPCToolOutcomeKnown(raw, resultRaw, event.toolFailed) if event.toolFailed { event.toolError = piRPCToolError(raw["result"]) if event.toolError == "" { @@ -927,26 +1085,41 @@ func parsePiRPCEvent(line []byte) (piRPCEvent, error) { if id := firstRawString(raw, "sessionId", "session_id"); id != "" { event.sessionID = id } - if eventType == "response" && rawString(raw, "command") == "prompt" && !rawBool(raw, "success") { - event.responseFailure = rawString(raw, "error") - if event.responseFailure == "" { - event.responseFailure = "unknown error" + if eventType == "response" && rawString(raw, "command") == "prompt" { + if !rawBool(raw, "success") { + event.responseFailure = rawString(raw, "error") + if event.responseFailure == "" { + event.responseFailure = "unknown error" + } + return event, nil + } + var data map[string]json.RawMessage + if err := json.Unmarshal(raw["data"], &data); err == nil && rawString(data, "disposition") == "handled" { + event.promptHandled = true } return event, nil } - if eventType == "message_end" { - if output, usage := parsePiRPCMessageEnd(raw["message"]); len(output) > 0 { - event.structuredOutput = output - event.usage = mergeUsage(event.usage, usage) - } else { - event.usage = mergeUsage(event.usage, usage) + // Usage is counted only from completed reports: message_end for each + // assistant message and compaction_end for summary generation. + // message_update usage is cumulative, and agent_end.messages repeats + // messages already reported by message_end. + switch eventType { + case "message_end": + event.assistant, event.usage = parsePiRPCMessageEnd(raw["message"]) + case "compaction_end": + var result map[string]json.RawMessage + if err := json.Unmarshal(raw["result"], &result); err == nil { + event.usage = parsePiRPCUsage(result) } - } - if eventType == "agent_end" { - event.agentEnd = true - if output := parsePiRPCAgentEnd(raw["messages"]); len(output) > 0 { - event.structuredOutput = output + event.awaitsSettlement = !rawBool(raw, "willRetry") + case "auto_retry_end": + if !rawBool(raw, "success") { + event.retryFailure = rawString(raw, "finalError") } + case "agent_end": + event.awaitsSettlement = !rawBool(raw, "willRetry") + case "agent_settled": + event.settled = true } return event, nil } @@ -986,8 +1159,9 @@ function runTool(tool, params, signal) { }); let stdout = Buffer.alloc(0); let stderr = Buffer.alloc(0); + let stdoutTruncated = false; const append = (current, chunk) => Buffer.concat([current, chunk]).subarray(0, maxOutputBytes); - child.stdout.on("data", (chunk) => { stdout = append(stdout, chunk); }); + child.stdout.on("data", (chunk) => { stdoutTruncated ||= stdout.length + chunk.length > maxOutputBytes; stdout = append(stdout, chunk); }); child.stderr.on("data", (chunk) => { stderr = append(stderr, chunk); }); const kill = () => { try { @@ -1004,7 +1178,8 @@ function runTool(tool, params, signal) { clearTimeout(timer); signal?.removeEventListener("abort", kill); const text = code === 0 ? stdout.toString("utf8") : (stderr.toString("utf8") || ("tool exited " + code)); - resolve({ content: [{ type: "text", text }], details: {}, isError: code !== 0 }); + const details = code === 0 ? { codereview_tool_output: { version: 1, bytes: stdout.length, truncated: stdoutTruncated } } : {}; + resolve({ content: [{ type: "text", text }], details, isError: code !== 0 }); }); child.stdin.end(JSON.stringify({ ...params, tool })); }); @@ -1032,7 +1207,7 @@ func piRPCEventIndicatesToolUse(value string) bool { normalized == "bash" } -func parsePiRPCMessageEnd(value json.RawMessage) ([]byte, Usage) { +func parsePiRPCMessageEnd(value json.RawMessage) (*piRPCAssistantMessage, Usage) { var raw map[string]json.RawMessage if len(value) == 0 { return nil, Usage{} @@ -1040,30 +1215,14 @@ func parsePiRPCMessageEnd(value json.RawMessage) ([]byte, Usage) { if err := json.Unmarshal(value, &raw); err != nil { return nil, Usage{} } - usage := parsePiRPCUsage(raw) if rawString(raw, "role") != "assistant" { - return nil, usage - } - return extractPiRPCText(raw["content"]), usage -} - -func parsePiRPCAgentEnd(value json.RawMessage) []byte { - var messages []map[string]json.RawMessage - if len(value) == 0 { - return nil - } - if err := json.Unmarshal(value, &messages); err != nil { - return nil - } - for i := len(messages) - 1; i >= 0; i-- { - if rawString(messages[i], "role") != "assistant" { - continue - } - if output := extractPiRPCText(messages[i]["content"]); len(output) > 0 { - return output - } + return nil, Usage{} } - return nil + return &piRPCAssistantMessage{ + output: extractPiRPCText(raw["content"]), + stopReason: rawString(raw, "stopReason"), + errorMessage: rawString(raw, "errorMessage"), + }, parsePiRPCUsage(raw) } func extractPiRPCText(value json.RawMessage) []byte { @@ -1099,7 +1258,11 @@ func parsePiRPCUsage(raw map[string]json.RawMessage) Usage { TokensOut: firstRawIntPtr(usageRaw, "tokens_out", "tokensOut", "output", "outputTokens", "completionTokens"), CacheRead: firstRawIntPtr(usageRaw, "cache_read", "cacheRead"), CacheCreate: firstRawIntPtr(usageRaw, "cache_create", "cacheCreate", "cache_write", "cacheWrite"), - CostUSD: firstRawFloatPtr(usageRaw, "cost_usd", "costUSD", "totalCost", "totalCostUSD"), + // Pi reports cacheWrite1h as the one-hour subset of cacheWrite. The + // remainder is not labeled with a retention, so no five-minute value + // is derived from it. + CacheCreate1h: rawIntPtr(usageRaw, "cacheWrite1h"), + CostUSD: firstRawFloatPtr(usageRaw, "cost_usd", "costUSD", "totalCost", "totalCostUSD"), } if usage.CostUSD == nil { usage.CostUSD = nestedRawFloatPtr(usageRaw, "cost", "total") diff --git a/internal/llmadapters/pi_rpc_extension_unix_test.go b/internal/llmadapters/pi_rpc_extension_unix_test.go index 8395ba81..0ce34957 100644 --- a/internal/llmadapters/pi_rpc_extension_unix_test.go +++ b/internal/llmadapters/pi_rpc_extension_unix_test.go @@ -3,6 +3,8 @@ package llmadapters import ( + "context" + "encoding/json" "os" "os/exec" "path/filepath" @@ -11,6 +13,8 @@ import ( "syscall" "testing" "time" + + "github.com/open-cli-collective/codereview-cli/internal/llm" ) func TestPiRPCReviewerHelperStaysInParentProcessGroup(t *testing.T) { @@ -111,3 +115,49 @@ func TestPiRPCReviewerExtensionRequiresDiffBeforeHeadTools(t *testing.T) { } } } + +func TestPiRPCReviewerExtensionReportsTransportTruncation(t *testing.T) { + nodePath, err := exec.LookPath("node") + if err != nil { + t.Skip("Node is not installed") + } + for _, size := range []int{10, 128, 129} { + t.Run(strconv.Itoa(size), func(t *testing.T) { + tempDir := t.TempDir() + helperPath := filepath.Join(tempDir, "helper.mjs") + helper := "#!/usr/bin/env node\nprocess.stdin.resume();\nprocess.stdin.on('end', () => process.stdout.write('x'.repeat(" + strconv.Itoa(size) + ")));\n" + if err := os.WriteFile(helperPath, []byte(helper), 0o700); err != nil { // #nosec G306,G703 -- executable helper is rooted in t.TempDir. + t.Fatal(err) + } + extensionPath := filepath.Join(tempDir, "extension.mjs") + extension := piRPCReviewerExtension(helperPath, filepath.Join(tempDir, "config.json"), tempDir, 128, 5*time.Second, "") + if err := os.WriteFile(extensionPath, []byte(extension), 0o600); err != nil { // #nosec G703 -- generated extension is rooted in t.TempDir. + t.Fatal(err) + } + runnerPath := filepath.Join(tempDir, "runner.mjs") + runner := "import extension from " + strconv.Quote(extensionPath) + ";\nconst tools = {}; extension({ registerTool(tool) { tools[tool.name] = tool; } });\nconsole.log(JSON.stringify(await tools.cr_diff.execute('diff', {}, new AbortController().signal)));\n" + if err := os.WriteFile(runnerPath, []byte(runner), 0o600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, nodePath, runnerPath) // #nosec G204 -- discovered Node executable runs a test-owned script. + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("extension: %v\n%s", err, output) + } + var result struct { + Details json.RawMessage `json:"details"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(output, &result); err != nil || result.IsError { + t.Fatalf("result = %s, err %v", output, err) + } + got := parsePiRPCToolOutput(result.Details) + want := llm.ReviewerToolOutput{Bytes: int64(min(size, 128)), Truncated: size > 128} + if got == nil || *got != want { + t.Fatalf("output = %#v, want %#v", got, want) + } + }) + } +} diff --git a/internal/llmadapters/pi_rpc_integration_settlement_test.go b/internal/llmadapters/pi_rpc_integration_settlement_test.go new file mode 100644 index 00000000..1b94a012 --- /dev/null +++ b/internal/llmadapters/pi_rpc_integration_settlement_test.go @@ -0,0 +1,142 @@ +package llmadapters + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/open-cli-collective/codereview-cli/internal/llm" +) + +// The evidence collector and settlement protocol arrived on separate branches. +// Keep execution evidence across retries without treating agent_end as final. +func TestPiRPCReviewerTraceRequiresSettledRun(t *testing.T) { + final := piRPCTestTurn{text: `{"ok":true}`, stopReason: "stop"} + failure := piRPCTestTurn{stopReason: "error", errorMessage: "fixture retry failure"} + firstRead := []string{ + piRPCTestPromptStarted, + `{"type":"tool_execution_start","toolCallId":"read-1","toolName":"cr_read","args":{"path":"one.go"}}`, + `{"type":"tool_execution_end","toolCallId":"read-1","toolName":"cr_read","isError":false}`, + } + for _, tt := range []struct { + name string + tail []string + wantComplete bool + wantCalls int + wantErr string + cancelAfterCall bool + }{ + { + name: "retry retains calls until successful settlement", + tail: []string{ + failure.end(), piRPCTestAgentEnd(true, failure), + `{"type":"auto_retry_start","attempt":1}`, + `#pause`, + `{"type":"tool_execution_start","toolCallId":"read-2","toolName":"cr_read","args":{"path":"two.go","view":"symlink"}}`, + `{"type":"tool_execution_end","toolCallId":"read-2","toolName":"cr_read","isError":false}`, + final.end(), piRPCTestAgentEnd(false, final), `{"type":"agent_settled"}`, + }, + wantComplete: true, + wantCalls: 2, + }, + { + name: "agent end without settlement is incomplete", + tail: []string{final.end(), piRPCTestAgentEnd(false, final), `#exit`}, + wantCalls: 1, + wantErr: "agent_settled", + }, + { + name: "caller cancellation preserves partial trace", + tail: strings.Split(strings.Repeat("#pause\n", 100), "\n"), + wantCalls: 1, + wantErr: "context canceled", + cancelAfterCall: true, + }, + { + name: "malformed settlement is incomplete", + tail: []string{final.end(), piRPCTestAgentEnd(false, final), `{"type":"agent_settled"`}, + wantCalls: 1, + wantErr: "malformed JSONL", + }, + { + name: "settlement without final answer is incomplete", + tail: []string{`{"type":"agent_settled"}`}, + wantCalls: 1, + wantErr: "no structured output", + }, + { + name: "settled provider failure is incomplete", + tail: []string{failure.end(), piRPCTestAgentEnd(false, failure), `{"type":"agent_settled"}`}, + wantCalls: 1, + wantErr: "final assistant message error", + }, + } { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + repoDir, scratchDir := filepath.Join(root, "repo"), filepath.Join(root, "scratch") + for _, dir := range []string{repoDir, scratchDir} { + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + } + diffPath := filepath.Join(root, "diff.patch") + if err := os.WriteFile(diffPath, []byte("fixed diff\n"), 0o600); err != nil { + t.Fatal(err) + } + script := append(append([]string(nil), firstRead...), tt.tail...) + adapter := piRPCScriptAdapter(t, 5*time.Second, script...) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + logPath := filepath.Join(root, "events.jsonl") + stream, err := adapter.Start(ctx, Request{ + Prompt: "review", + LogPath: logPath, + ReviewerWorkspace: &ReviewerWorkspaceRequest{ + RepoDir: repoDir, ScratchDir: scratchDir, DiffPath: diffPath, MaxToolOutputBytes: 2048, + }, + }) + if err != nil { + t.Fatalf("Start: %v", err) + } + if tt.cancelAfterCall { + deadline := time.Now().Add(3 * time.Second) + for { + logged, _ := os.ReadFile(logPath) // #nosec G304 -- fixture-owned path. + if strings.Contains(string(logged), `"tool_execution_end"`) { + cancel() + break + } + if time.Now().After(deadline) { + t.Fatal("tool execution checkpoint was not observed before cancellation") + } + time.Sleep(time.Millisecond) + } + } + response, err := stream.Wait(context.Background()) + if tt.wantErr == "" && err != nil || tt.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tt.wantErr)) { + t.Fatalf("Wait error = %v, want %q", err, tt.wantErr) + } + evidence := response.ReviewerToolEvidence + if evidence == nil || evidence.Trace == nil || evidence.Trace.StreamComplete != tt.wantComplete || len(evidence.Trace.Calls) != tt.wantCalls { + t.Fatalf("evidence = %#v, want complete=%t and %d calls", evidence, tt.wantComplete, tt.wantCalls) + } + if evidence.DiffStatus != llm.DiffToolStatusNotInvoked { + t.Fatalf("diff status = %s, file reads must not satisfy the diff gate", evidence.DiffStatus) + } + for _, call := range evidence.Trace.Calls { + if call.Status != llm.ReviewerToolCallSucceeded { + t.Fatalf("call = %#v, want retained successful execution", call) + } + } + if tt.wantComplete && (string(response.StructuredOutput) != `{"ok":true}` || evidence.Trace.Calls[1].ReadView != llm.ReviewerToolReadSymlink) { + t.Fatalf("settled output = %s, calls = %#v", response.StructuredOutput, evidence.Trace.Calls) + } + if !tt.wantComplete && len(response.StructuredOutput) != 0 { + t.Fatalf("failed stream returned structured output: %s", response.StructuredOutput) + } + }) + } +} diff --git a/internal/llmadapters/pi_rpc_runtime_test.go b/internal/llmadapters/pi_rpc_runtime_test.go new file mode 100644 index 00000000..bd6c5106 --- /dev/null +++ b/internal/llmadapters/pi_rpc_runtime_test.go @@ -0,0 +1,890 @@ +package llmadapters + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/open-cli-collective/codereview-cli/internal/llm" + "github.com/open-cli-collective/codereview-cli/internal/pireviewtool" +) + +// piRuntimeVersionEnv opts this package into tests that drive the installed Pi +// runtime against a local mock provider. Its value is the exact version that +// `pi --version` must report: once it is set, a missing or different Pi fails +// instead of skipping. `make test-pi-runtime` sets it from the Makefile pin. +const piRuntimeVersionEnv = "CR_PI_RUNTIME_VERSION" + +const ( + piRuntimeProviderName = "crruntime" + piRuntimeModelID = "local" + piRuntimeModel = piRuntimeProviderName + "/" + piRuntimeModelID + piRuntimeAPIKey = "cr-runtime-fixture-key" // #nosec G101 -- dummy key accepted only by the loopback mock provider. + // piRuntimeHostileMarker is planted in global and project Pi resources the + // fixture must never load. It must not reach the provider. + piRuntimeHostileMarker = "CR_PI_RUNTIME_HOSTILE_RESOURCE" + piRuntimeTestTimeout = 60 * time.Second + piRuntimeTaskTimeout = 45 * time.Second +) + +// TestMain lets the generated reviewer extension dispatch its tool helper to +// this test binary the way it dispatches to the cr binary in production, so +// runtime tests exercise the real pireviewtool.Run. +func TestMain(m *testing.M) { + if len(os.Args) > 1 && os.Args[1] == "__pi-review-tool" { + os.Exit(pireviewtool.Run(context.Background(), os.Args[2:], os.Stdin, os.Stdout, os.Stderr)) + } + os.Exit(m.Run()) +} + +func TestPiRPCRuntimeStructuredJSONWithoutTools(t *testing.T) { + fixture := newPiRuntimeFixture(t, + piRuntimeAnswer(`{"ok":true}`, piRuntimeUsage{prompt: 120, completion: 9, cached: 40, cacheWrite: 10}), + ) + + response, err := fixture.run(fixture.adapter(piRuntimeTaskTimeout), Request{Model: piRuntimeModel, Prompt: `Return {"ok":true}.`}) + if err != nil { + t.Fatalf("run: %v", err) + } + if string(response.StructuredOutput) != `{"ok":true}` { + t.Fatalf("StructuredOutput = %q, want the provider answer", response.StructuredOutput) + } + // input = 120 - 40 cached - 10 written; cost = 70*1 + 9*2 + 40*0.5 + 10*2.5. + assertPiRuntimeUsage(t, response.Usage, 70, 9, 40, 10, 133) + if response.ReviewerToolEvidence != nil { + t.Fatalf("ReviewerToolEvidence = %#v, want none for a no-tools call", response.ReviewerToolEvidence) + } + requests := fixture.provider.requests() + if len(requests) != 1 { + t.Fatalf("provider requests = %d, want 1", len(requests)) + } + if names := requests[0].toolNames(); len(names) != 0 { + t.Fatalf("provider request tools = %v, want none", names) + } + if system := requests[0].systemText(); !strings.Contains(system, piRPCSystemPrompt) { + t.Fatalf("provider system prompt = %q, want the CR structured-output prompt", system) + } +} + +// Plant each instruction independently so Pi's AGENTS/CLAUDE precedence cannot +// hide an uncovered source. The selected directory also contains the fixture's +// real models.json and dummy auth.json, which must continue to be used. +func TestPiRPCRuntimeSelectedAgentInstructionIsolation(t *testing.T) { + for _, reviewer := range []bool{false, true} { + mode := "structured" + if reviewer { + mode = "reviewer" + } + t.Run(mode, func(t *testing.T) { + for _, filename := range []string{"AGENTS.md", "CLAUDE.md", "AGENTS.override.md", "SYSTEM.md", "APPEND_SYSTEM.md"} { + t.Run(filename, func(t *testing.T) { + fixture := newPiRuntimeFixture(t, piRuntimeAnswer(`{"ok":true}`, piRuntimeUsage{prompt: 10, completion: 1})) + for _, candidate := range []string{"AGENTS.md", "CLAUDE.md", "SYSTEM.md", "APPEND_SYSTEM.md"} { + if err := os.Remove(filepath.Join(fixture.agentDir, candidate)); err != nil { // #nosec G703 -- synthetic instruction fixtures only. + t.Fatalf("Remove(selected instruction): %v", err) + } + } + fixture.writeFile(filepath.Join(fixture.agentDir, filename), piRuntimeHostileMarker+" selected "+filename+"\n") + req := Request{Model: piRuntimeModel, Prompt: `Return {"ok":true}.`} + wantPrompt := piRPCSystemPrompt + if reviewer { + req.ReviewerWorkspace = fixture.reviewerWorkspace() + wantPrompt = piRPCReviewerSystemPrompt + } + response, err := fixture.run(fixture.adapter(piRuntimeTaskTimeout), req) + if err != nil || string(response.StructuredOutput) != `{"ok":true}` { + t.Fatalf("run output = %q (err %v), want CR response", response.StructuredOutput, err) + } + requests := fixture.provider.requests() + if len(requests) != 1 || !strings.Contains(requests[0].systemText(), wantPrompt) { + t.Fatalf("provider requests = %d, want one with the CR prompt", len(requests)) + } + // assertIsolated checks every request for the marker and the + // dummy selected-store authorization, including tool turns. + }) + } + }) + } +} + +func TestPiRPCRuntimeRecoversTransientProviderFailure(t *testing.T) { + fixture := newPiRuntimeFixture(t, + piRuntimeFailure(http.StatusServiceUnavailable, "fixture transient overload"), + piRuntimeAnswer(`{"ok":true}`, piRuntimeUsage{prompt: 100, completion: 10}), + ) + + response, err := fixture.run(fixture.adapter(piRuntimeTaskTimeout), Request{Model: piRuntimeModel, Prompt: `Return {"ok":true}.`}) + if err != nil { + t.Fatalf("run: %v", err) + } + if string(response.StructuredOutput) != `{"ok":true}` { + t.Fatalf("StructuredOutput = %q, want the answer from the retried attempt", response.StructuredOutput) + } + // The failed attempt reports zero usage, so the total is the retry alone. + assertPiRuntimeUsage(t, response.Usage, 100, 10, 0, 0, 120) + if requests := fixture.provider.requests(); len(requests) != 2 { + t.Fatalf("provider requests = %d, want the failed attempt and one retry", len(requests)) + } +} + +func TestPiRPCRuntimePreservesPermanentProviderFailure(t *testing.T) { + for _, tc := range []struct { + name string + steps []piRuntimeStep + requests int + want string + }{ + { + name: "non-retryable rejection", + steps: []piRuntimeStep{piRuntimeFailure(http.StatusBadRequest, "fixture permanent rejection")}, + requests: 1, + want: "fixture permanent rejection", + }, + { + name: "retries exhausted", + steps: []piRuntimeStep{ + piRuntimeFailure(http.StatusServiceUnavailable, "fixture overload one"), + piRuntimeFailure(http.StatusServiceUnavailable, "fixture overload two"), + }, + requests: 2, + want: "fixture overload two", + }, + } { + t.Run(tc.name, func(t *testing.T) { + fixture := newPiRuntimeFixture(t, tc.steps...) + + response, err := fixture.run(fixture.adapter(piRuntimeTaskTimeout), Request{Model: piRuntimeModel, Prompt: `Return {"ok":true}.`}) + if err == nil { + t.Fatalf("run succeeded with %q, want the provider failure", response.StructuredOutput) + } + if !strings.Contains(err.Error(), "final assistant message error") || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("run error = %v, want final provider diagnostic %q", err, tc.want) + } + if len(response.StructuredOutput) != 0 { + t.Fatalf("StructuredOutput = %q, want none on failure", response.StructuredOutput) + } + if requests := fixture.provider.requests(); len(requests) != tc.requests { + t.Fatalf("provider requests = %d, want %d", len(requests), tc.requests) + } + }) + } +} + +func TestPiRPCRuntimeReviewerToolsAndUsage(t *testing.T) { + fixture := newPiRuntimeFixture(t, + piRuntimeToolCalls(piRuntimeUsage{prompt: 100, completion: 5}, + piRuntimeToolCall{id: "read-early", name: "cr_read", args: `{"path":"main.go"}`}, + ), + piRuntimeToolCalls(piRuntimeUsage{prompt: 150, completion: 7, cached: 60, cacheWrite: 20}, + piRuntimeToolCall{id: "diff", name: "cr_diff", args: `{}`}, + ), + piRuntimeToolCalls(piRuntimeUsage{prompt: 200, completion: 11, cached: 120}, + piRuntimeToolCall{id: "read", name: "cr_read", args: `{"path":"main.go"}`}, + piRuntimeToolCall{id: "search", name: "cr_search", args: `{"query":"func Hello"}`}, + piRuntimeToolCall{id: "list", name: "cr_list", args: `{}`}, + piRuntimeToolCall{id: "escape", name: "cr_read", args: `{"path":"../outside.txt"}`}, + ), + piRuntimeAnswer(`{"findings":[]}`, piRuntimeUsage{prompt: 260, completion: 13, cached: 180, cacheWrite: 30}), + ) + workspace := fixture.reviewerWorkspace() + + response, err := fixture.run(fixture.adapter(piRuntimeTaskTimeout), Request{ + Model: piRuntimeModel, + Prompt: "Review the pinned change.", + ReviewerWorkspace: workspace, + }) + if err != nil { + t.Fatalf("run: %v", err) + } + if string(response.StructuredOutput) != `{"findings":[]}` { + t.Fatalf("StructuredOutput = %q, want the final answer after every tool turn", response.StructuredOutput) + } + if response.ReviewerToolEvidence == nil || response.ReviewerToolEvidence.DiffStatus != llm.DiffToolStatusSucceeded { + t.Fatalf("ReviewerToolEvidence = %#v, want a succeeded cr_diff", response.ReviewerToolEvidence) + } + // Every assistant turn counts once: input 100+70+80+50, output 5+7+11+13, + // cache read 0+60+120+180, cache write 0+20+0+30, cost 300 + 36*2 + + // 360*0.5 + 50*2.5. + assertPiRuntimeUsage(t, response.Usage, 300, 36, 360, 50, 677) + + requests := fixture.provider.requests() + if len(requests) != 4 { + t.Fatalf("provider requests = %d, want one per assistant turn", len(requests)) + } + wantTools := []string{"cr_diff", "cr_list", "cr_read", "cr_search"} + for i, request := range requests { + if got := request.toolNames(); !reflect.DeepEqual(got, wantTools) { + t.Fatalf("request %d tools = %v, want only %v", i, got, wantTools) + } + if system := request.systemText(); !strings.Contains(system, piRPCReviewerSystemPrompt) { + t.Fatalf("request %d system prompt = %q, want the CR reviewer prompt", i, system) + } + } + assertPiRuntimeToolResult(t, requests[1], "read-early", "cr_diff must be invoked before inspecting repository files") + assertPiRuntimeToolResult(t, requests[2], "diff", "+func Hello() string") + assertPiRuntimeToolResult(t, requests[3], "read", `return "hello from the head revision"`) + assertPiRuntimeToolResult(t, requests[3], "search", "main.go") + assertPiRuntimeToolResult(t, requests[3], "list", "main.go\n") + assertPiRuntimeToolResult(t, requests[3], "escape", "path traversal") + if strings.Contains(string(requests[3].raw), "outside the repository") { + t.Fatal("cr_read returned content from outside the repository root") + } + assertPiRuntimeDirEmpty(t, workspace.ScratchDir) +} + +func TestPiRPCRuntimeReviewerRejectsUnlistedTool(t *testing.T) { + shellMarker := filepath.Join(t.TempDir(), "shell-ran") + fixture := newPiRuntimeFixture(t, + piRuntimeToolCalls(piRuntimeUsage{prompt: 100, completion: 5}, + piRuntimeToolCall{id: "diff", name: "cr_diff", args: `{}`}, + ), + piRuntimeToolCalls(piRuntimeUsage{prompt: 150, completion: 5}, + piRuntimeToolCall{id: "shell", name: "bash", args: mustPiRuntimeJSON(t, map[string]string{"command": "touch " + shellMarker})}, + ), + piRuntimeAnswer(`{"findings":[]}`, piRuntimeUsage{prompt: 200, completion: 5}), + ) + workspace := fixture.reviewerWorkspace() + + response, err := fixture.run(fixture.adapter(piRuntimeTaskTimeout), Request{ + Model: piRuntimeModel, + Prompt: "Review the pinned change.", + ReviewerWorkspace: workspace, + }) + if !errors.Is(err, ErrToolUse) { + t.Fatalf("run error = %v (output %q), want ErrToolUse for an unlisted tool", err, response.StructuredOutput) + } + if len(response.StructuredOutput) != 0 { + t.Fatalf("StructuredOutput = %q, want none after a denied tool", response.StructuredOutput) + } + if _, statErr := os.Stat(shellMarker); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("shell marker stat = %v, want the bash request never executed", statErr) + } + if requests := fixture.provider.requests(); len(requests) != 2 { + t.Fatalf("provider requests = %d, want the run stopped at the denied tool call", len(requests)) + } + assertPiRuntimeDirEmpty(t, workspace.ScratchDir) +} + +// requirePiRuntime returns the installed Pi path when runtime tests are +// enabled. It skips only when the opt-in is unset; a set but blank version +// fails so a broken pin cannot pass by skipping. +func requirePiRuntime(t *testing.T) string { + t.Helper() + want, set := os.LookupEnv(piRuntimeVersionEnv) + if !set { + t.Skipf("set %s to the required Pi version (make test-pi-runtime) to run installed Pi runtime tests", piRuntimeVersionEnv) + } + if strings.TrimSpace(want) == "" { + t.Fatalf("%s is set but blank; it must name the exact required Pi version", piRuntimeVersionEnv) + } + install := installedPiRuntime() + if install.err != nil { + t.Fatalf("%s=%s requires the installed Pi runtime: %v", piRuntimeVersionEnv, want, install.err) + } + if install.version != want { + t.Fatalf("installed Pi %s reports version %q, want exactly %q", install.path, install.version, want) + } + return install.path +} + +type piRuntimeInstall struct { + path string + version string + err error +} + +var installedPiRuntime = sync.OnceValue(func() piRuntimeInstall { + path, err := exec.LookPath("pi") + if err != nil { + return piRuntimeInstall{err: err} + } + agentDir, err := os.MkdirTemp("", "cr-pi-runtime-version-*") + if err != nil { + return piRuntimeInstall{path: path, err: err} + } + defer func() { _ = os.RemoveAll(agentDir) }() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, path, "--version") // #nosec G204 -- test runs the discovered Pi executable with a fixed argument. + cmd.Env = append(os.Environ(), piRuntimeIsolationEnv(agentDir, agentDir)...) + output, err := cmd.Output() + if err != nil { + return piRuntimeInstall{path: path, err: fmt.Errorf("pi --version: %w", err)} + } + return piRuntimeInstall{path: path, version: strings.TrimSpace(string(output))} +}) + +// piRuntimeGoroutines counts live goroutines, leaving out LaunchProcess's +// pipe-close grace, which outlives a finished run by a bounded delay by design. +func piRuntimeGoroutines() (int, string) { + buf := make([]byte, 1<<20) + for { + n := runtime.Stack(buf, true) + if n < len(buf) { + buf = buf[:n] + break + } + buf = make([]byte, 2*len(buf)) + } + var kept []string + for _, stack := range strings.Split(string(buf), "\n\n") { + if strings.Contains(stack, "internal/llm.LaunchProcess.func") && strings.Contains(stack, "time.Sleep") { + continue + } + kept = append(kept, stack) + } + return len(kept), strings.Join(kept, "\n\n") +} + +// piRuntimeIsolationEnv points Pi at a fixture-owned agent directory and home +// and turns off automatic network activity other than the model request. +func piRuntimeIsolationEnv(agentDir, home string) []string { + return []string{ + "PI_CODING_AGENT_DIR=" + agentDir, + "PI_CODING_AGENT_SESSION_DIR=" + filepath.Join(agentDir, "sessions"), + "PI_OFFLINE=1", + "PI_SKIP_VERSION_CHECK=1", + "PI_TELEMETRY=0", + "HOME=" + home, + "USERPROFILE=" + home, + } +} + +type piRuntimeFixture struct { + t *testing.T + piPath string + // command launches Pi; tests may point it at a wrapper of piPath. + command string + root string + agentDir string + env []string + provider *piRuntimeProvider + // sideEffects are files that hostile global or project resources would + // create if Pi loaded them. + sideEffects []string +} + +func newPiRuntimeFixture(t *testing.T, steps ...piRuntimeStep) *piRuntimeFixture { + t.Helper() + piPath := requirePiRuntime(t) + goroutines, _ := piRuntimeGoroutines() + t.Cleanup(func() { + deadline := time.Now().Add(5 * time.Second) + for { + got, stacks := piRuntimeGoroutines() + if got <= goroutines { + return + } + if time.Now().After(deadline) { + t.Errorf("goroutines = %d after the runtime test, want at most %d:\n%s", got, goroutines, stacks) + return + } + time.Sleep(20 * time.Millisecond) + } + }) + + root := t.TempDir() + fixture := &piRuntimeFixture{ + t: t, + piPath: piPath, + command: piPath, + root: root, + agentDir: filepath.Join(root, "agent"), + provider: newPiRuntimeProvider(t, steps), + } + home := filepath.Join(root, "home") + fixture.env = piRuntimeIsolationEnv(fixture.agentDir, home) + fixture.writeJSON(filepath.Join(fixture.agentDir, "models.json"), map[string]any{ + "providers": map[string]any{ + piRuntimeProviderName: map[string]any{ + "baseUrl": fixture.provider.server.URL + "/v1", + "api": "openai-completions", + "models": []any{map[string]any{ + "id": piRuntimeModelID, + "name": "CR runtime fixture", + "reasoning": false, + "input": []string{"text"}, + "contextWindow": 100000, + "maxTokens": 1000, + // Rates per million tokens that make each token cost an + // exactly representable amount: 1, 2, 0.5, and 2.5. + "cost": map[string]any{"input": 1000000, "output": 2000000, "cacheRead": 500000, "cacheWrite": 2500000}, + }}, + }, + }, + }) + fixture.writeJSON(filepath.Join(fixture.agentDir, "auth.json"), map[string]any{ + piRuntimeProviderName: map[string]any{"type": "api_key", "key": piRuntimeAPIKey}, + }) + fixture.writeJSON(filepath.Join(fixture.agentDir, "settings.json"), map[string]any{ + "retry": map[string]any{"enabled": true, "maxRetries": 1, "baseDelayMs": 10}, + "enableInstallTelemetry": false, + }) + // The actual selected directory must reject instruction resources without + // hiding its model configuration, credentials, or refresh destination. + fixture.plantHostileResources(fixture.agentDir, "selected-agent") + // Resources at Pi's default global location. The agent directory override + // must keep every one of them out of the run. + fixture.plantHostileResources(filepath.Join(home, ".pi", "agent"), "global") + fixture.writeFile(filepath.Join(home, ".pi", "agent", "settings.json"), `{"retry":{"enabled":false},"defaultProjectTrust":"always"}`) + fixture.writeFile(filepath.Join(home, ".agents", "skills", "hostile", "SKILL.md"), "---\nname: hostile\ndescription: "+piRuntimeHostileMarker+"\n---\n"+piRuntimeHostileMarker+"\n") + + t.Cleanup(fixture.assertIsolated) + return fixture +} + +// plantHostileResources writes instructions, an extension, and an MCP server +// under dir. Loading any of them leaves the marker in a provider request or +// creates a side-effect file. +func (f *piRuntimeFixture) plantHostileResources(dir, label string) { + f.t.Helper() + extensionMarker := filepath.Join(f.root, label+"-extension-loaded") + mcpMarker := filepath.Join(f.root, label+"-mcp-started") + f.sideEffects = append(f.sideEffects, extensionMarker, mcpMarker) + for _, name := range []string{"AGENTS.md", "CLAUDE.md", "SYSTEM.md", "APPEND_SYSTEM.md"} { + f.writeFile(filepath.Join(dir, name), piRuntimeHostileMarker+" "+label+" "+name+"\n") + } + f.writeFile(filepath.Join(dir, "extensions", "hostile.js"), + "import fs from \"node:fs\";\nexport default function () { fs.writeFileSync("+mustPiRuntimeJSON(f.t, extensionMarker)+", \"loaded\"); }\n") + f.writeJSON(filepath.Join(dir, "mcp.json"), map[string]any{ + "mcpServers": map[string]any{ + "hostile": map[string]any{ + "command": "node", + "args": []string{"-e", "require('node:fs').writeFileSync(" + mustPiRuntimeJSON(f.t, mcpMarker) + ", 'started')"}, + "description": piRuntimeHostileMarker, + }, + }, + }) +} + +// reviewerWorkspace builds a disposable checkout that also carries hostile +// project-level Pi resources, as an untrusted pull request could. +func (f *piRuntimeFixture) reviewerWorkspace() *ReviewerWorkspaceRequest { + f.t.Helper() + workspaceRoot := filepath.Join(f.root, "workspace") + repoDir := filepath.Join(workspaceRoot, "repo") + scratchDir := filepath.Join(workspaceRoot, "scratch") + f.writeFile(filepath.Join(repoDir, "main.go"), "package main\n\nfunc Hello() string {\n\treturn \"hello from the head revision\"\n}\n") + f.writeFile(filepath.Join(workspaceRoot, "outside.txt"), "outside the repository\n") + f.plantHostileResources(filepath.Join(repoDir, ".pi"), "project") + for _, name := range []string{"AGENTS.md", "CLAUDE.md"} { + f.writeFile(filepath.Join(repoDir, name), piRuntimeHostileMarker+" project root "+name+"\n") + } + f.writeFile(filepath.Join(repoDir, ".agents", "skills", "hostile", "SKILL.md"), "---\nname: hostile\ndescription: "+piRuntimeHostileMarker+"\n---\n"+piRuntimeHostileMarker+"\n") + if err := os.MkdirAll(scratchDir, 0o700); err != nil { + f.t.Fatalf("MkdirAll(scratch): %v", err) + } + diffPath := filepath.Join(workspaceRoot, "diff.patch") + f.writeFile(diffPath, "diff --git a/main.go b/main.go\n--- a/main.go\n+++ b/main.go\n@@ -1,2 +1,5 @@\n package main\n+\n+func Hello() string {\n+\treturn \"hello from the head revision\"\n+}\n") + return &ReviewerWorkspaceRequest{ + RepoDir: repoDir, + ScratchDir: scratchDir, + DiffPath: diffPath, + MaxToolOutputBytes: 64 * 1024, + } +} + +func (f *piRuntimeFixture) adapter(timeout time.Duration) *PiRPCAdapter { + return NewPiRPCAdapter(PiRPCOptions{ + Command: f.command, + Env: f.env, + Timeout: timeout, + ScratchDirFactory: func() (string, func() error, error) { + dir, err := os.MkdirTemp(f.root, "scratch-*") + if err != nil { + return "", nil, err + } + return dir, func() error { return os.RemoveAll(dir) }, nil + }, + }) +} + +func (f *piRuntimeFixture) run(adapter *PiRPCAdapter, req Request) (Response, error) { + f.t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), piRuntimeTestTimeout) + defer cancel() + stream, err := adapter.Start(ctx, req) + if err != nil { + return Response{}, err + } + return stream.Wait(ctx) +} + +func (f *piRuntimeFixture) assertIsolated() { + t := f.t + t.Helper() + for i, request := range f.provider.requests() { + if request.authorization != "Bearer "+piRuntimeAPIKey { + t.Errorf("request %d authorization is not the fixture key", i) + } + if bytes.Contains(request.raw, []byte(piRuntimeHostileMarker)) { + t.Errorf("request %d carries a global or project resource the fixture must not load", i) + } + } + for _, unexpected := range f.provider.unexpectedRequests() { + t.Errorf("unexpected provider request: %s", unexpected) + } + for _, path := range f.sideEffects { + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Errorf("hostile resource side effect %s exists (stat err %v)", filepath.Base(path), err) + } + } + entries, err := os.ReadDir(f.root) + if err != nil { + t.Errorf("ReadDir(fixture root): %v", err) + return + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), "scratch-") { + t.Errorf("adapter scratch %s was not removed", entry.Name()) + } + } +} + +func (f *piRuntimeFixture) writeJSON(path string, value any) { + f.t.Helper() + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + f.t.Fatalf("Marshal(%s): %v", filepath.Base(path), err) + } + f.writeFile(path, string(data)+"\n") +} + +func (f *piRuntimeFixture) writeFile(path, content string) { + f.t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + f.t.Fatalf("MkdirAll(%s): %v", filepath.Dir(path), err) + } + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { // #nosec G703 -- path is rooted in the fixture's t.TempDir. + f.t.Fatalf("WriteFile(%s): %v", path, err) + } +} + +// piRuntimeStep answers one provider request. +type piRuntimeStep func(w http.ResponseWriter, r *http.Request, stop <-chan struct{}) + +type piRuntimeUsage struct { + prompt int + completion int + cached int + cacheWrite int +} + +func (u piRuntimeUsage) payload() map[string]any { + return map[string]any{ + "prompt_tokens": u.prompt, + "completion_tokens": u.completion, + "total_tokens": u.prompt + u.completion, + "prompt_tokens_details": map[string]any{ + "cached_tokens": u.cached, + "cache_write_tokens": u.cacheWrite, + }, + } +} + +type piRuntimeToolCall struct { + id string + name string + args string +} + +func piRuntimeAnswer(text string, usage piRuntimeUsage) piRuntimeStep { + return func(w http.ResponseWriter, _ *http.Request, _ <-chan struct{}) { + writePiRuntimeSSE(w, + piRuntimeChunk(map[string]any{"role": "assistant", "content": text}, nil, nil), + piRuntimeChunk(map[string]any{}, "stop", &usage), + ) + } +} + +func piRuntimeToolCalls(usage piRuntimeUsage, calls ...piRuntimeToolCall) piRuntimeStep { + toolCalls := make([]any, 0, len(calls)) + for i, call := range calls { + toolCalls = append(toolCalls, map[string]any{ + "index": i, + "id": call.id, + "type": "function", + "function": map[string]any{"name": call.name, "arguments": call.args}, + }) + } + return func(w http.ResponseWriter, _ *http.Request, _ <-chan struct{}) { + writePiRuntimeSSE(w, + piRuntimeChunk(map[string]any{"role": "assistant", "tool_calls": toolCalls}, nil, nil), + piRuntimeChunk(map[string]any{}, "tool_calls", &usage), + ) + } +} + +// piRuntimeFailure returns an OpenAI-style error. Pi retries by matching the +// error text, so only 5xx responses carry the retryable server_error type. +func piRuntimeFailure(status int, message string) piRuntimeStep { + errorType := "invalid_request_error" + if status >= http.StatusInternalServerError { + errorType = "server_error" + } + return func(w http.ResponseWriter, _ *http.Request, _ <-chan struct{}) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{"message": message, "type": errorType}}) + } +} + +// piRuntimeStall starts a stream and then holds the request open until the +// client disconnects or the fixture shuts down. started is closed once the +// stream is open; disconnected is closed if the client goes away first. +func piRuntimeStall(started, disconnected chan<- struct{}) piRuntimeStep { + return func(w http.ResponseWriter, r *http.Request, stop <-chan struct{}) { + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + data, _ := json.Marshal(piRuntimeChunk(map[string]any{"role": "assistant", "content": ""}, nil, nil)) + _, _ = fmt.Fprintf(w, "data: %s\n\n", data) + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + } + close(started) + select { + case <-r.Context().Done(): + close(disconnected) + case <-stop: + } + } +} + +func piRuntimeChunk(delta map[string]any, finishReason any, usage *piRuntimeUsage) map[string]any { + chunk := map[string]any{ + "id": "cr-runtime-completion", + "object": "chat.completion.chunk", + "created": 0, + "model": piRuntimeModelID, + "choices": []any{map[string]any{"index": 0, "delta": delta, "finish_reason": finishReason}}, + } + if usage != nil { + chunk["usage"] = usage.payload() + } + return chunk +} + +func writePiRuntimeSSE(w http.ResponseWriter, chunks ...map[string]any) { + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + for _, chunk := range chunks { + data, _ := json.Marshal(chunk) + _, _ = fmt.Fprintf(w, "data: %s\n\n", data) + } + _, _ = io.WriteString(w, "data: [DONE]\n\n") +} + +// piRuntimeProvider is a scripted OpenAI-compatible chat completions endpoint +// bound to loopback. Request n gets step n; any extra request is rejected with +// a non-retryable error and reported by the fixture. +type piRuntimeProvider struct { + server *httptest.Server + stop chan struct{} + mu sync.Mutex + steps []piRuntimeStep + recorded []piRuntimeRequest + unexpected []string +} + +func newPiRuntimeProvider(t *testing.T, steps []piRuntimeStep) *piRuntimeProvider { + t.Helper() + provider := &piRuntimeProvider{stop: make(chan struct{}), steps: steps} + provider.server = httptest.NewServer(provider) + t.Cleanup(provider.server.Close) + // Release stalled handlers before Close waits for them. + t.Cleanup(func() { close(provider.stop) }) + host, _, err := net.SplitHostPort(provider.server.Listener.Addr().String()) + if err != nil || !net.ParseIP(host).IsLoopback() { + t.Fatalf("mock provider address %s is not loopback (%v)", provider.server.Listener.Addr(), err) + } + return provider +} + +func (p *piRuntimeProvider) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/v1/chat/completions" { + p.recordUnexpected(r.Method + " " + r.URL.Path) + http.Error(w, "unexpected endpoint", http.StatusNotFound) + return + } + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 8<<20)) + if err != nil { + p.recordUnexpected("unreadable request body: " + err.Error()) + http.Error(w, "unreadable body", http.StatusBadRequest) + return + } + request := piRuntimeRequest{authorization: r.Header.Get("Authorization"), raw: body} + if err := json.Unmarshal(body, &request.body); err != nil { + p.recordUnexpected("malformed request body: " + err.Error()) + } + p.mu.Lock() + index := len(p.recorded) + p.recorded = append(p.recorded, request) + var step piRuntimeStep + if index < len(p.steps) { + step = p.steps[index] + } + p.mu.Unlock() + if step == nil { + p.recordUnexpected(fmt.Sprintf("request %d beyond the %d scripted steps", index, len(p.steps))) + piRuntimeFailure(http.StatusBadRequest, "unexpected extra provider request")(w, r, p.stop) + return + } + step(w, r, p.stop) +} + +func (p *piRuntimeProvider) recordUnexpected(message string) { + p.mu.Lock() + defer p.mu.Unlock() + p.unexpected = append(p.unexpected, message) +} + +func (p *piRuntimeProvider) requests() []piRuntimeRequest { + p.mu.Lock() + defer p.mu.Unlock() + return append([]piRuntimeRequest(nil), p.recorded...) +} + +func (p *piRuntimeProvider) unexpectedRequests() []string { + p.mu.Lock() + defer p.mu.Unlock() + return append([]string(nil), p.unexpected...) +} + +type piRuntimeRequest struct { + authorization string + raw []byte + body struct { + Messages []struct { + Role string `json:"role"` + Content json.RawMessage `json:"content"` + ToolCallID string `json:"tool_call_id"` + } `json:"messages"` + Tools []struct { + Function struct { + Name string `json:"name"` + } `json:"function"` + } `json:"tools"` + } +} + +func (r piRuntimeRequest) toolNames() []string { + names := make([]string, 0, len(r.body.Tools)) + for _, tool := range r.body.Tools { + names = append(names, tool.Function.Name) + } + slices.Sort(names) + return names +} + +func (r piRuntimeRequest) systemText() string { + var out strings.Builder + for _, message := range r.body.Messages { + if message.Role == "system" || message.Role == "developer" { + out.WriteString(piRuntimeContentText(message.Content)) + } + } + return out.String() +} + +func (r piRuntimeRequest) toolResult(id string) (string, bool) { + for _, message := range r.body.Messages { + if message.Role == "tool" && message.ToolCallID == id { + return piRuntimeContentText(message.Content), true + } + } + return "", false +} + +func piRuntimeContentText(content json.RawMessage) string { + var text string + if err := json.Unmarshal(content, &text); err == nil { + return text + } + var parts []struct { + Type string `json:"type"` + Text string `json:"text"` + } + if err := json.Unmarshal(content, &parts); err != nil { + return "" + } + var out strings.Builder + for _, part := range parts { + if part.Type == "text" { + out.WriteString(part.Text) + } + } + return out.String() +} + +func assertPiRuntimeToolResult(t *testing.T, request piRuntimeRequest, id, want string) { + t.Helper() + got, ok := request.toolResult(id) + if !ok { + t.Fatalf("provider request has no tool result for %s", id) + } + if !strings.Contains(got, want) { + t.Fatalf("tool result %s = %q, want it to contain %q", id, got, want) + } +} + +func assertPiRuntimeUsage(t *testing.T, usage Usage, tokensIn, tokensOut, cacheRead, cacheCreate int, costUSD float64) { + t.Helper() + got := fmt.Sprintf("in=%s out=%s cacheRead=%s cacheCreate=%s cacheCreate1h=%s cost=%s", + piRuntimeIntField(usage.TokensIn), piRuntimeIntField(usage.TokensOut), piRuntimeIntField(usage.CacheRead), + piRuntimeIntField(usage.CacheCreate), piRuntimeIntField(usage.CacheCreate1h), piRuntimeFloatField(usage.CostUSD)) + want := fmt.Sprintf("in=%d out=%d cacheRead=%d cacheCreate=%d cacheCreate1h=nil cost=%g", + tokensIn, tokensOut, cacheRead, cacheCreate, costUSD) + if got != want { + t.Fatalf("usage = %s, want %s", got, want) + } +} + +func piRuntimeIntField(value *int) string { + if value == nil { + return "nil" + } + return fmt.Sprint(*value) +} + +func piRuntimeFloatField(value *float64) string { + if value == nil { + return "nil" + } + return fmt.Sprintf("%g", *value) +} + +func assertPiRuntimeDirEmpty(t *testing.T, dir string) { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("ReadDir(%s): %v", dir, err) + } + if len(entries) != 0 { + t.Fatalf("%s still holds %d entries after the run, want adapter cleanup", dir, len(entries)) + } +} + +func mustPiRuntimeJSON(t *testing.T, value any) string { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + return string(data) +} diff --git a/internal/llmadapters/pi_rpc_runtime_unix_test.go b/internal/llmadapters/pi_rpc_runtime_unix_test.go new file mode 100644 index 00000000..d1c3cb69 --- /dev/null +++ b/internal/llmadapters/pi_rpc_runtime_unix_test.go @@ -0,0 +1,168 @@ +//go:build unix + +package llmadapters + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +// Process-group assertions are POSIX-specific; the CI runtime gate runs on +// Linux. +func TestPiRPCRuntimeCancellationStopsPi(t *testing.T) { + for _, tc := range []struct { + name string + timeout time.Duration + cancel bool + want error + }{ + {name: "caller cancellation", timeout: piRuntimeTaskTimeout, cancel: true, want: context.Canceled}, + {name: "task deadline", timeout: 3 * time.Second, want: context.DeadlineExceeded}, + } { + t.Run(tc.name, func(t *testing.T) { + started := make(chan struct{}) + disconnected := make(chan struct{}) + fixture := newPiRuntimeFixture(t, piRuntimeStall(started, disconnected)) + // exec keeps the wrapper's PID, which is also the process group the + // adapter starts Pi in. + pidPath := filepath.Join(fixture.root, "pi.pid") + wrapperPath := filepath.Join(fixture.root, "pi-wrapper.sh") + fixture.writeFile(wrapperPath, "#!/bin/sh\necho $$ > "+strconv.Quote(pidPath)+"\nexec "+strconv.Quote(fixture.piPath)+" \"$@\"\n") + if err := os.Chmod(wrapperPath, 0o700); err != nil { // #nosec G302 -- test wrapper must be executable and is rooted in t.TempDir. + t.Fatalf("Chmod(wrapper): %v", err) + } + fixture.command = wrapperPath + + ctx, cancel := context.WithTimeout(context.Background(), piRuntimeTestTimeout) + defer cancel() + stream, err := fixture.adapter(tc.timeout).Start(ctx, Request{Model: piRuntimeModel, Prompt: `Return {"ok":true}.`}) + if err != nil { + t.Fatalf("Start: %v", err) + } + select { + case <-started: + case <-time.After(30 * time.Second): + cancel() + _, _ = stream.Wait(context.Background()) + t.Fatal("Pi never reached the mock provider") + } + pid := readPiRuntimePID(t, pidPath) + + began := time.Now() + if tc.cancel { + cancel() + } + response, err := stream.Wait(ctx) + elapsed := time.Since(began) + if !errors.Is(err, tc.want) { + t.Fatalf("Wait error = %v, want %v", err, tc.want) + } + if len(response.StructuredOutput) != 0 { + t.Fatalf("StructuredOutput = %q, want none after %s", response.StructuredOutput, tc.name) + } + limit := 10 * time.Second + if !tc.cancel { + limit += tc.timeout + } + if elapsed > limit { + t.Fatalf("Wait returned after %s, want termination within %s", elapsed, limit) + } + select { + case <-disconnected: + case <-time.After(5 * time.Second): + t.Fatal("Pi kept the provider request open after the run ended") + } + eventually(t, 5*time.Second, func() bool { + return errors.Is(syscall.Kill(-pid, 0), syscall.ESRCH) + }) + if requests := fixture.provider.requests(); len(requests) != 1 { + t.Fatalf("provider requests = %d, want the stalled request only", len(requests)) + } + }) + } +} + +func readPiRuntimePID(t *testing.T, path string) int { + t.Helper() + var data []byte + eventually(t, 5*time.Second, func() bool { + var err error + data, err = os.ReadFile(path) // #nosec G304 -- path is rooted in the fixture's t.TempDir. + return err == nil && strings.TrimSpace(string(data)) != "" + }) + pid, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil || pid <= 1 { + t.Fatalf("Pi PID file = %q: %v", data, err) + } + return pid +} + +// A blank Pi pin must stop both Make targets: it would make the runtime tests +// skip and report ok, and it would let npm install the latest Pi. A fake go on +// PATH records any dispatch instead of running the suite. +func TestPiRuntimeMakeTargetsRejectBlankPin(t *testing.T) { + makePath, err := exec.LookPath("make") + if err != nil { + t.Skip("make is not installed") + } + repoRoot, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatalf("Abs(repo root): %v", err) + } + binDir := t.TempDir() + recordPath := filepath.Join(t.TempDir(), "go-dispatch") + fakeGo := "#!/bin/sh\nprintf '%s' \"$CR_PI_RUNTIME_VERSION\" > " + strconv.Quote(recordPath) + "\n" + if err := os.WriteFile(filepath.Join(binDir, "go"), []byte(fakeGo), 0o700); err != nil { // #nosec G306 -- fake go must be executable and is rooted in t.TempDir. + t.Fatalf("WriteFile(fake go): %v", err) + } + env := []string{"PATH=" + binDir + string(os.PathListSeparator) + os.Getenv("PATH")} + for _, entry := range os.Environ() { + switch key, _, _ := strings.Cut(entry, "="); key { + case "PATH", "MAKEFLAGS", "MFLAGS", "MAKELEVEL", "PI_RUNTIME_VERSION", piRuntimeVersionEnv: + default: + env = append(env, entry) + } + } + runMake := func(args ...string) (string, bool, error) { + t.Helper() + _ = os.Remove(recordPath) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, makePath, append([]string{"-s", "--no-print-directory", "-C", repoRoot}, args...)...) // #nosec G204 -- test runs make with fixed targets. + cmd.Env = env + output, err := cmd.Output() + recorded, readErr := os.ReadFile(recordPath) // #nosec G304 -- recordPath is rooted in t.TempDir. + if readErr != nil && !errors.Is(readErr, os.ErrNotExist) { + t.Fatalf("ReadFile(go dispatch record): %v", readErr) + } + if readErr == nil { + return string(recorded), true, err + } + return string(output), false, err + } + + pin, dispatched, err := runMake("pi-runtime-version") + pin = strings.TrimSpace(pin) + if err != nil || dispatched || pin == "" || strings.ContainsAny(pin, " \t") { + t.Fatalf("make pi-runtime-version = %q (err %v), want one default version", pin, err) + } + if got, dispatched, err := runMake("test-pi-runtime"); err != nil || !dispatched || got != pin { + t.Fatalf("make test-pi-runtime dispatched=%v with %s=%q (err %v), want go test with %q", dispatched, piRuntimeVersionEnv, got, err, pin) + } + for _, bad := range []string{"", " ", "\t", pin + " " + pin} { + if got, dispatched, err := runMake("pi-runtime-version", "PI_RUNTIME_VERSION="+bad); err == nil || dispatched || strings.TrimSpace(got) != "" { + t.Errorf("make pi-runtime-version PI_RUNTIME_VERSION=%q printed %q (err %v), want failure without output", bad, got, err) + } + if got, dispatched, err := runMake("test-pi-runtime", "PI_RUNTIME_VERSION="+bad); err == nil || dispatched { + t.Errorf("make test-pi-runtime PI_RUNTIME_VERSION=%q dispatched=%v with %q (err %v), want failure before go test", bad, dispatched, got, err) + } + } +} diff --git a/internal/llmadapters/pi_rpc_test.go b/internal/llmadapters/pi_rpc_test.go index c42be9d9..11adbdda 100644 --- a/internal/llmadapters/pi_rpc_test.go +++ b/internal/llmadapters/pi_rpc_test.go @@ -73,7 +73,8 @@ func TestPiRPCLaunchSafetyAndSuccess(t *testing.T) { assertFlagValue(t, record.AdapterArgs, "--model", "opencode-go/kimi-k2.6") assertFlagValue(t, record.AdapterArgs, "--thinking", "max") assertFlagValue(t, record.AdapterArgs, "--system-prompt", piRPCSystemPrompt) - for _, flag := range []string{"--no-tools", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-themes", "--no-session"} { + assertPiRPCEmptyAppend(t, record.AdapterArgs) + for _, flag := range []string{"--no-tools", "--no-context-files", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-themes", "--no-session"} { if !containsFlag(record.AdapterArgs, flag) { t.Fatalf("args = %#v, want %s", record.AdapterArgs, flag) } @@ -120,6 +121,86 @@ func TestPiRPCStreamRecordsDuration(t *testing.T) { assertSlowSuccessDuration(t, response) } +// This subprocess double models an auth.json refresh without using credentials +// or a provider. Prompt isolation must leave the selected store in place, so +// its updated synthetic state survives invocation scratch cleanup. +func TestPiRPCPreservesSelectedAgentAuthenticationState(t *testing.T) { + for _, reviewer := range []bool{false, true} { + name := "structured" + if reviewer { + name = "reviewer" + } + t.Run(name, func(t *testing.T) { + root := t.TempDir() + agentDir := filepath.Join(root, "selected-agent") + home := filepath.Join(root, "home") + repoDir := filepath.Join(root, "repo") + scratchDir := filepath.Join(root, "scratch") + for _, dir := range []string{agentDir, home, repoDir, scratchDir} { + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatalf("Mkdir: %v", err) + } + } + authPath := filepath.Join(agentDir, "auth.json") + if err := os.WriteFile(authPath, []byte(`{"fixture":"initial synthetic auth state"}`), 0o600); err != nil { + t.Fatalf("WriteFile(synthetic auth): %v", err) + } + recordPath := filepath.Join(root, "record.json") + adapter := NewPiRPCAdapter(PiRPCOptions{ + Command: os.Args[0], + commandArgsPrefix: piRPCHelperPrefix(), + Env: append(piRPCHelperEnv("success", recordPath), + "PI_CODING_AGENT_DIR="+agentDir, + "HOME="+home, + "LLM_PI_RPC_SYNTHETIC_AUTH_REFRESH=1", + ), + Timeout: 5 * time.Second, + ScratchDirFactory: func() (string, func() error, error) { + dir, err := os.MkdirTemp(scratchDir, "invocation-*") + return dir, func() error { return os.RemoveAll(dir) }, err + }, + }) + req := Request{Model: "fixture/model", Prompt: "fixture prompt"} + wantPrompt := piRPCSystemPrompt + if reviewer { + wantPrompt = piRPCReviewerSystemPrompt + diffPath := filepath.Join(root, "diff.patch") + if err := os.WriteFile(diffPath, []byte("fixture diff\n"), 0o600); err != nil { + t.Fatalf("WriteFile(diff): %v", err) + } + req.ReviewerWorkspace = &ReviewerWorkspaceRequest{ + RepoDir: repoDir, ScratchDir: scratchDir, DiffPath: diffPath, MaxToolOutputBytes: 2048, + } + } + stream, err := adapter.Start(context.Background(), req) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err := stream.Wait(context.Background()); err != nil { + t.Fatalf("Wait: %v", err) + } + record := readPiRPCRecord(t, recordPath) + if record.Env["PI_CODING_AGENT_DIR"] != agentDir || record.Env["HOME"] != home { + t.Fatalf("selected agent/home = %q/%q, want %q/%q", record.Env["PI_CODING_AGENT_DIR"], record.Env["HOME"], agentDir, home) + } + assertFlagValue(t, record.AdapterArgs, "--system-prompt", wantPrompt) + if appendPrompt, present := flagValueOK(record.AdapterArgs, "--append-system-prompt"); !present || appendPrompt != "" { + t.Fatalf("append system prompt = %q (present %v), want explicit empty input", appendPrompt, present) + } + if !containsFlag(record.AdapterArgs, "--no-context-files") { + t.Fatalf("args = %#v, want context discovery disabled", record.AdapterArgs) + } + data, err := os.ReadFile(authPath) // #nosec G304 -- synthetic test-owned auth state only. + if err != nil || string(data) != `{"fixture":"refreshed synthetic auth state"}` { + t.Fatalf("synthetic auth state = %q (err %v), want refresh in original selected store", data, err) + } + if entries, err := os.ReadDir(scratchDir); err != nil || len(entries) != 0 { + t.Fatalf("scratch entries = %v (err %v), want cleanup without auth state loss", entries, err) + } + }) + } +} + func TestPiRPCFailureLeavesDurationZero(t *testing.T) { recordPath := filepath.Join(t.TempDir(), "record.json") adapter := NewPiRPCAdapter(PiRPCOptions{ @@ -202,6 +283,7 @@ func TestPiRPCReviewerWorkspaceLaunchUsesOnlyCROwnedTools(t *testing.T) { t.Fatalf("args = %#v, reviewer extension tools must remain enabled", record.AdapterArgs) } assertFlagValue(t, record.AdapterArgs, "--tools", piRPCReviewerToolNames) + assertPiRPCEmptyAppend(t, record.AdapterArgs) reviewerPrompt := flagValue(record.AdapterArgs, "--system-prompt") for _, instruction := range []string{"Invoke cr_diff before cr_read, cr_search, or cr_list", "If cr_diff fails", "view=symlink", "assignment.base_only_symlink_paths", "ordinary cr_read and no view", "payload_omitted_reason", "keep that path skipped", "payload_size=0 is an inspected empty payload"} { if !strings.Contains(reviewerPrompt, instruction) { @@ -360,6 +442,10 @@ func TestPiRPCReviewerLogCapDoesNotBreakProtocolCompletion(t *testing.T) { if response.ReviewerToolEvidence == nil || response.ReviewerToolEvidence.DiffStatus != llm.DiffToolStatusNotInvoked { t.Fatalf("reviewer tool evidence = %#v, want not-invoked cr_diff", response.ReviewerToolEvidence) } + trace := response.ReviewerToolEvidence.Trace + if trace == nil || !trace.StreamComplete || len(trace.Calls) != 20 || trace.Calls[19].ProvenanceIssue != "missing_start" { + t.Fatalf("trace = %#v, want all unmatched executions despite capped logs", trace) + } logged, err := os.ReadFile(logPath) // #nosec G304 -- logPath is rooted in t.TempDir. if err != nil { t.Fatalf("ReadFile(log): %v", err) @@ -603,6 +689,7 @@ func TestPiRPCReviewerPreflightUsesEmptyDiscoveryDisabledDirectory(t *testing.T) } assertFlagValue(t, record.AdapterArgs, "--mode", "rpc") assertFlagValue(t, record.AdapterArgs, "--tools", piRPCReviewerToolNames) + assertPiRPCEmptyAppend(t, record.AdapterArgs) for _, flag := range []string{"--no-builtin-tools", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-themes", "--no-context-files", "--no-approve", "--no-session"} { if !containsFlag(record.AdapterArgs, flag) { t.Fatalf("preflight args = %#v, want %s", record.AdapterArgs, flag) @@ -1192,7 +1279,7 @@ func TestPiRPCProtocolFailures(t *testing.T) { } }) - t.Run("caller cancellation propagates before agent_end", func(t *testing.T) { + t.Run("caller cancellation propagates before any event", func(t *testing.T) { recordPath := filepath.Join(t.TempDir(), "record.json") ctx, cancel := context.WithCancel(context.Background()) adapter := NewPiRPCAdapter(PiRPCOptions{ @@ -1233,7 +1320,7 @@ func TestPiRPCProtocolFailures(t *testing.T) { } }) - t.Run("missing agent_end fails stream", func(t *testing.T) { + t.Run("stream ending before agent_settled fails", func(t *testing.T) { recordPath := filepath.Join(t.TempDir(), "record.json") adapter := NewPiRPCAdapter(PiRPCOptions{ Command: os.Args[0], @@ -1246,17 +1333,17 @@ func TestPiRPCProtocolFailures(t *testing.T) { t.Fatalf("Start: %v", err) } _, err = stream.Wait(context.Background()) - if err == nil || !strings.Contains(err.Error(), "agent_end") { - t.Fatalf("Wait error = %v, want missing agent_end", err) + if err == nil || !strings.Contains(err.Error(), "agent_settled") { + t.Fatalf("Wait error = %v, want missing agent_settled", err) } }) - t.Run("non-zero exit after agent_end fails stream", func(t *testing.T) { + t.Run("non-zero exit after agent_settled fails stream", func(t *testing.T) { recordPath := filepath.Join(t.TempDir(), "record.json") adapter := NewPiRPCAdapter(PiRPCOptions{ Command: os.Args[0], commandArgsPrefix: piRPCHelperPrefix(), - Env: piRPCHelperEnv("agent-end-exit-failure", recordPath), + Env: piRPCHelperEnv("settled-exit-failure", recordPath), Timeout: 5 * time.Second, }) stream, err := adapter.Start(context.Background(), Request{Model: "opencode-go/kimi-k2.6", Prompt: "prompt"}) @@ -1269,7 +1356,7 @@ func TestPiRPCProtocolFailures(t *testing.T) { } }) - t.Run("non-zero exit before agent_end fails stream", func(t *testing.T) { + t.Run("non-zero exit before agent_settled fails stream", func(t *testing.T) { recordPath := filepath.Join(t.TempDir(), "record.json") adapter := NewPiRPCAdapter(PiRPCOptions{ Command: os.Args[0], @@ -1288,6 +1375,445 @@ func TestPiRPCProtocolFailures(t *testing.T) { }) } +const piRPCTestPromptStarted = `{"id":"prompt-1","type":"response","command":"prompt","success":true,"data":{"disposition":"started"}}` + +func TestPiRPCSettlementContract(t *testing.T) { + answer := `{"ok":true}` + overloaded := piRPCTestTurn{stopReason: "error", errorMessage: "503 overloaded", usage: piRPCTestUsage(7, 0, nil, 0.25)} + final := piRPCTestTurn{text: answer, stopReason: "stop", usage: piRPCTestUsage(100, 10, map[string]any{"cacheRead": 3, "cacheWrite": 4}, 0.5)} + finalUsage := Usage{TokensIn: piRPCTestPtr(100), TokensOut: piRPCTestPtr(10), CacheRead: piRPCTestPtr(3), CacheCreate: piRPCTestPtr(4), CostUSD: piRPCTestPtr(0.5)} + for _, tt := range []struct { + name string + script []string + wantOutput string + wantErr string + wantUsage Usage + }{ + { + name: "recoverable agent_end continues through retry to settlement", + script: []string{ + piRPCTestPromptStarted, + `{"type":"agent_start"}`, + overloaded.end(), + piRPCTestAgentEnd(true, overloaded), + `#pause`, + `{"type":"auto_retry_start","attempt":1,"maxAttempts":3,"delayMs":100,"errorMessage":"503 overloaded"}`, + `{"type":"agent_start"}`, + `{"type":"message_update","usage":{"input":90,"output":5,"cost":{"total":9}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":"{"}}`, + final.end(), + `{"type":"auto_retry_end","success":true,"attempt":1}`, + piRPCTestAgentEnd(false, final), + `{"type":"agent_settled"}`, + }, + wantOutput: answer, + wantUsage: Usage{TokensIn: piRPCTestPtr(107), TokensOut: piRPCTestPtr(10), CacheRead: piRPCTestPtr(3), CacheCreate: piRPCTestPtr(4), CostUSD: piRPCTestPtr(0.75)}, + }, + { + name: "overflow compaction recovery counts summary usage once", + script: []string{ + piRPCTestPromptStarted, + piRPCTestTurn{stopReason: "error", errorMessage: "context overflow", usage: piRPCTestUsage(0, 0, nil, 0)}.end(), + `{"type":"agent_end","messages":[],"willRetry":false}`, + `{"type":"compaction_start","reason":"overflow"}`, + `{"type":"compaction_end","reason":"overflow","result":{"summary":"s","tokensBefore":9000,"usage":{"input":50,"output":5,"cost":{"total":0.25}}},"aborted":false,"willRetry":true}`, + `{"type":"agent_start"}`, + final.end(), + piRPCTestAgentEnd(false, final), + `{"type":"agent_settled"}`, + }, + wantOutput: answer, + wantUsage: Usage{TokensIn: piRPCTestPtr(150), TokensOut: piRPCTestPtr(15), CacheRead: piRPCTestPtr(3), CacheCreate: piRPCTestPtr(4), CostUSD: piRPCTestPtr(0.75)}, + }, + { + name: "completed assistant turns sum usage once and keep only the final answer", + script: []string{ + piRPCTestPromptStarted, + piRPCTestTurn{text: "Checking the diff.", toolCall: true, stopReason: "toolUse", usage: piRPCTestUsage(100, 10, map[string]any{"cacheRead": 1}, 0.25)}.end(), + `{"type":"message_end","message":{"role":"toolResult","toolCallId":"call-1","toolName":"cr_diff","content":[{"type":"text","text":"diff"}],"isError":false}}`, + `{"type":"message_update","usage":{"input":200,"output":20,"cost":{"total":0.5}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":"{"}}`, + `{"type":"message_update","usage":{"input":200,"output":20,"cost":{"total":0.5}},"assistantMessageEvent":{"type":"text_end","contentIndex":0,"content":"{\"ok\":true}"}}`, + piRPCTestTurn{text: answer, stopReason: "stop", usage: piRPCTestUsage(200, 20, map[string]any{"cacheRead": 2}, 0.5)}.end(), + `{"type":"agent_end","messages":[{"role":"assistant","content":[],"usage":{"input":100,"output":10},"stopReason":"toolUse"},{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}],"usage":{"input":200,"output":20},"stopReason":"stop"}],"willRetry":false}`, + `{"type":"agent_settled"}`, + }, + wantOutput: answer, + wantUsage: Usage{TokensIn: piRPCTestPtr(300), TokensOut: piRPCTestPtr(30), CacheRead: piRPCTestPtr(3), CostUSD: piRPCTestPtr(0.75)}, + }, + { + name: "one-hour cache writes sum once per completed assistant message", + script: []string{ + piRPCTestPromptStarted, + piRPCTestTurn{text: "Checking the diff.", toolCall: true, stopReason: "toolUse", usage: piRPCTestUsage(100, 10, map[string]any{"cacheWrite": 5, "cacheWrite1h": 1}, 0.25)}.end(), + `{"type":"message_update","usage":{"input":200,"output":20,"cacheWrite":7,"cacheWrite1h":2,"cost":{"total":0.5}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":"{"}}`, + piRPCTestTurn{text: answer, stopReason: "stop", usage: piRPCTestUsage(200, 20, map[string]any{"cacheWrite": 7, "cacheWrite1h": 2}, 0.5)}.end(), + `{"type":"agent_end","messages":[{"role":"assistant","content":[],"usage":{"input":100,"output":10,"cacheWrite":5,"cacheWrite1h":1},"stopReason":"toolUse"},{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}],"usage":{"input":200,"output":20,"cacheWrite":7,"cacheWrite1h":2},"stopReason":"stop"}],"willRetry":false}`, + `{"type":"agent_settled"}`, + }, + wantOutput: answer, + wantUsage: Usage{TokensIn: piRPCTestPtr(300), TokensOut: piRPCTestPtr(30), CacheCreate: piRPCTestPtr(12), CacheCreate1h: piRPCTestPtr(3), CostUSD: piRPCTestPtr(0.75)}, + }, + { + name: "one-hour cache split reported by only some messages sums the reported values", + script: []string{ + piRPCTestPromptStarted, + piRPCTestTurn{text: "Checking the diff.", toolCall: true, stopReason: "toolUse", usage: piRPCTestUsage(100, 10, map[string]any{"cacheWrite": 5}, 0.25)}.end(), + piRPCTestTurn{text: answer, stopReason: "stop", usage: piRPCTestUsage(200, 20, map[string]any{"cacheWrite": 7, "cacheWrite1h": 2}, 0.5)}.end(), + piRPCTestAgentEnd(false), + `{"type":"agent_settled"}`, + }, + wantOutput: answer, + wantUsage: Usage{TokensIn: piRPCTestPtr(300), TokensOut: piRPCTestPtr(30), CacheCreate: piRPCTestPtr(12), CacheCreate1h: piRPCTestPtr(2), CostUSD: piRPCTestPtr(0.75)}, + }, + { + name: "provider error after exhausted retries fails with diagnostic", + script: []string{ + piRPCTestPromptStarted, + piRPCTestTurn{text: answer, stopReason: "error", errorMessage: "503 overloaded", usage: piRPCTestUsage(7, 0, nil, 0.25)}.end(), + `{"type":"agent_end","messages":[],"willRetry":true}`, + `{"type":"auto_retry_start","attempt":1,"maxAttempts":1,"delayMs":1,"errorMessage":"503 overloaded"}`, + overloaded.end(), + `{"type":"auto_retry_end","success":false,"attempt":1,"finalError":"503 overloaded"}`, + `{"type":"agent_end","messages":[],"willRetry":false}`, + `{"type":"agent_settled"}`, + }, + wantErr: "503 overloaded", + wantUsage: Usage{TokensIn: piRPCTestPtr(14), TokensOut: piRPCTestPtr(0), CostUSD: piRPCTestPtr(0.5)}, + }, + { + name: "aborted final assistant fails", + script: []string{ + piRPCTestPromptStarted, + piRPCTestTurn{text: `{"ok":`, stopReason: "aborted", errorMessage: "Request was aborted", usage: piRPCTestUsage(5, 1, nil, 0)}.end(), + `{"type":"agent_end","messages":[],"willRetry":false}`, + `{"type":"agent_settled"}`, + }, + wantErr: "Request was aborted", + wantUsage: Usage{TokensIn: piRPCTestPtr(5), TokensOut: piRPCTestPtr(1), CostUSD: piRPCTestPtr(0.0)}, + }, + { + name: "earlier valid answer then final provider error fails", + script: []string{ + piRPCTestPromptStarted, + piRPCTestTurn{text: answer, toolCall: true, stopReason: "toolUse", usage: piRPCTestUsage(10, 2, nil, 0)}.end(), + piRPCTestTurn{stopReason: "error", errorMessage: "terminated", usage: piRPCTestUsage(0, 0, nil, 0)}.end(), + `{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}],"stopReason":"toolUse"}],"willRetry":false}`, + `{"type":"agent_settled"}`, + }, + wantErr: "terminated", + wantUsage: Usage{TokensIn: piRPCTestPtr(10), TokensOut: piRPCTestPtr(2), CostUSD: piRPCTestPtr(0.0)}, + }, + { + name: "settled run without a final answer fails", + script: []string{ + piRPCTestPromptStarted, + piRPCTestTurn{text: "Checking the diff.", toolCall: true, stopReason: "toolUse", usage: piRPCTestUsage(10, 2, nil, 0)}.end(), + `{"type":"agent_end","messages":[],"willRetry":false}`, + `{"type":"agent_settled"}`, + }, + wantErr: "final answer", + wantUsage: Usage{TokensIn: piRPCTestPtr(10), TokensOut: piRPCTestPtr(2), CostUSD: piRPCTestPtr(0.0)}, + }, + { + name: "stream truncated after agent_end fails", + script: []string{ + piRPCTestPromptStarted, + final.end(), + piRPCTestAgentEnd(false, final), + `#exit`, + }, + wantErr: "agent_settled", + wantUsage: finalUsage, + }, + { + name: "malformed record after valid answer fails without output", + script: []string{ + piRPCTestPromptStarted, + final.end(), + `{"type":`, + }, + wantErr: "malformed JSONL", + wantUsage: finalUsage, + }, + { + name: "prompt handled without a run fails", + script: []string{ + `{"id":"prompt-1","type":"response","command":"prompt","success":true,"data":{"disposition":"handled"}}`, + }, + wantErr: "handled", + }, + { + name: "prompt rejection fails", + script: []string{ + `{"id":"prompt-1","type":"response","command":"prompt","success":false,"error":"No API key found for opencode-go"}`, + }, + wantErr: "No API key found", + }, + } { + t.Run(tt.name, func(t *testing.T) { + adapter := piRPCScriptAdapter(t, 5*time.Second, tt.script...) + stream, err := adapter.Start(context.Background(), Request{Model: "opencode-go/kimi-k2.6", Prompt: "prompt"}) + if err != nil { + t.Fatalf("Start: %v", err) + } + start := time.Now() + response, err := stream.Wait(context.Background()) + if elapsed := time.Since(start); elapsed > 3*time.Second { + t.Fatalf("Wait took %s, want completion without waiting for the task deadline", elapsed) + } + if tt.wantErr == "" && err != nil { + t.Fatalf("Wait: %v", err) + } + if tt.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tt.wantErr)) { + t.Fatalf("Wait error = %v, want %q", err, tt.wantErr) + } + if string(response.StructuredOutput) != tt.wantOutput { + t.Fatalf("StructuredOutput = %q, want %q", response.StructuredOutput, tt.wantOutput) + } + if !reflect.DeepEqual(response.Usage, tt.wantUsage) { + t.Fatalf("Usage = %s, want %s", formatPiRPCTestUsage(response.Usage), formatPiRPCTestUsage(tt.wantUsage)) + } + }) + } +} + +func TestParsePiRPCUsageCacheWriteRetention(t *testing.T) { + for _, tt := range []struct { + name string + usage string + want Usage + }{ + { + name: "one-hour subset is kept beside the aggregate", + usage: `{"input":10,"output":2,"cacheRead":3,"cacheWrite":7,"cacheWrite1h":4,"cost":{"total":0.5}}`, + want: Usage{TokensIn: piRPCTestPtr(10), TokensOut: piRPCTestPtr(2), CacheRead: piRPCTestPtr(3), CacheCreate: piRPCTestPtr(7), CacheCreate1h: piRPCTestPtr(4), CostUSD: piRPCTestPtr(0.5)}, + }, + { + name: "zero one-hour subset is a reported zero", + usage: `{"input":10,"output":2,"cacheRead":0,"cacheWrite":7,"cacheWrite1h":0,"cost":{"total":0.5}}`, + want: Usage{TokensIn: piRPCTestPtr(10), TokensOut: piRPCTestPtr(2), CacheRead: piRPCTestPtr(0), CacheCreate: piRPCTestPtr(7), CacheCreate1h: piRPCTestPtr(0), CostUSD: piRPCTestPtr(0.5)}, + }, + { + name: "absent one-hour subset stays unknown", + usage: `{"input":10,"output":2,"cacheRead":0,"cacheWrite":7,"cost":{"total":0.5}}`, + want: Usage{TokensIn: piRPCTestPtr(10), TokensOut: piRPCTestPtr(2), CacheRead: piRPCTestPtr(0), CacheCreate: piRPCTestPtr(7), CostUSD: piRPCTestPtr(0.5)}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + got := parsePiRPCUsage(map[string]json.RawMessage{"usage": json.RawMessage(tt.usage)}) + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("parsePiRPCUsage = %s, want %s", formatPiRPCTestUsage(got), formatPiRPCTestUsage(tt.want)) + } + }) + } +} + +func TestPiRPCReviewerSettlementPreservesToolEvidence(t *testing.T) { + tempDir := t.TempDir() + repoDir := filepath.Join(tempDir, "repo") + scratchDir := filepath.Join(tempDir, "scratch") + for _, dir := range []string{repoDir, scratchDir} { + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatalf("MkdirAll(%s): %v", dir, err) + } + } + diffPath := filepath.Join(tempDir, "diff.patch") + if err := os.WriteFile(diffPath, []byte("fixed diff\n"), 0o600); err != nil { + t.Fatalf("WriteFile(diff): %v", err) + } + commentary := piRPCTestTurn{text: "Checking the diff.", toolCall: true, stopReason: "toolUse", usage: piRPCTestUsage(100, 10, nil, 0.25)} + overloaded := piRPCTestTurn{stopReason: "error", errorMessage: "503 overloaded", usage: piRPCTestUsage(0, 0, nil, 0)} + final := piRPCTestTurn{text: `{"ok":true}`, stopReason: "stop", usage: piRPCTestUsage(200, 20, nil, 0.5)} + adapter := piRPCScriptAdapter(t, 5*time.Second, + piRPCTestPromptStarted, + commentary.end(), + `{"type":"tool_execution_start","toolCallId":"diff-1","toolName":"cr_diff","args":{}}`, + `{"type":"tool_execution_end","toolCallId":"diff-1","toolName":"cr_diff","result":{"content":[{"type":"text","text":"diff"}]},"isError":false}`, + overloaded.end(), + piRPCTestAgentEnd(true, commentary, overloaded), + `#pause`, + `{"type":"auto_retry_start","attempt":1,"maxAttempts":3,"delayMs":100,"errorMessage":"503 overloaded"}`, + final.end(), + `{"type":"auto_retry_end","success":true,"attempt":1}`, + piRPCTestAgentEnd(false, final), + `{"type":"agent_settled"}`, + ) + stream, err := adapter.Start(context.Background(), Request{ + Prompt: "review", + ReviewerWorkspace: &ReviewerWorkspaceRequest{ + RepoDir: repoDir, ScratchDir: scratchDir, DiffPath: diffPath, MaxToolOutputBytes: 2048, + }, + }) + if err != nil { + t.Fatalf("Start: %v", err) + } + response, err := stream.Wait(context.Background()) + if err != nil { + t.Fatalf("Wait: %v", err) + } + if string(response.StructuredOutput) != `{"ok":true}` { + t.Fatalf("StructuredOutput = %q, want recovered final answer", response.StructuredOutput) + } + if response.ReviewerToolEvidence == nil || response.ReviewerToolEvidence.DiffStatus != llm.DiffToolStatusSucceeded { + t.Fatalf("reviewer tool evidence = %#v, want succeeded cr_diff", response.ReviewerToolEvidence) + } + want := Usage{TokensIn: piRPCTestPtr(300), TokensOut: piRPCTestPtr(30), CostUSD: piRPCTestPtr(0.75)} + if !reflect.DeepEqual(response.Usage, want) { + t.Fatalf("Usage = %s, want %s", formatPiRPCTestUsage(response.Usage), formatPiRPCTestUsage(want)) + } +} + +func TestPiRPCPendingRetryHonorsDeadlineAndCancellation(t *testing.T) { + pendingRetry := []string{ + piRPCTestPromptStarted, + piRPCTestTurn{stopReason: "error", errorMessage: "503 overloaded", usage: piRPCTestUsage(0, 0, nil, 0)}.end(), + `{"type":"agent_end","messages":[],"willRetry":true}`, + `{"type":"auto_retry_start","attempt":1,"maxAttempts":3,"delayMs":60000,"errorMessage":"503 overloaded"}`, + } + t.Run("deadline", func(t *testing.T) { + adapter := piRPCScriptAdapter(t, 300*time.Millisecond, pendingRetry...) + stream, err := adapter.Start(context.Background(), Request{Model: "opencode-go/kimi-k2.6", Prompt: "prompt"}) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err := stream.Wait(context.Background()); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Wait error = %v, want deadline exceeded while retry is pending", err) + } + }) + t.Run("cancellation", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + adapter := piRPCScriptAdapter(t, 5*time.Second, pendingRetry...) + stream, err := adapter.Start(ctx, Request{Model: "opencode-go/kimi-k2.6", Prompt: "prompt"}) + if err != nil { + t.Fatalf("Start: %v", err) + } + time.Sleep(300 * time.Millisecond) + cancel() + if _, err := stream.Wait(context.Background()); !errors.Is(err, context.Canceled) { + t.Fatalf("Wait error = %v, want context canceled while retry is pending", err) + } + }) +} + +func TestPiRPCMissingSettlementFailsAsIncompatibleRuntime(t *testing.T) { + final := piRPCTestTurn{text: `{"ok":true}`, stopReason: "stop", usage: piRPCTestUsage(100, 10, nil, 0.5)} + adapter := piRPCScriptAdapter(t, 5*time.Second, + `{"id":"prompt-1","type":"response","command":"prompt","success":true}`, + final.end(), + `{"type":"agent_end","messages":[]}`, + ) + adapter.settlementGrace = 100 * time.Millisecond + stream, err := adapter.Start(context.Background(), Request{Model: "opencode-go/kimi-k2.6", Prompt: "prompt"}) + if err != nil { + t.Fatalf("Start: %v", err) + } + start := time.Now() + response, err := stream.Wait(context.Background()) + if !errors.Is(err, ErrPiRPCIncompatible) || !strings.Contains(err.Error(), "agent_settled") { + t.Fatalf("Wait error = %v, want incompatible runtime without agent_settled", err) + } + if elapsed := time.Since(start); elapsed > 3*time.Second { + t.Fatalf("Wait took %s, want bounded settlement grace", elapsed) + } + if len(response.StructuredOutput) != 0 { + t.Fatalf("StructuredOutput = %q, want none without settlement", response.StructuredOutput) + } +} + +func TestPiRPCSettlementGraceRestartsOnFollowUpWork(t *testing.T) { + final := piRPCTestTurn{text: `{"ok":true}`, stopReason: "stop", usage: piRPCTestUsage(100, 10, nil, 0.5)} + adapter := piRPCScriptAdapter(t, 5*time.Second, + piRPCTestPromptStarted, + final.end(), + piRPCTestAgentEnd(false, final), + `#pause`, + `{"type":"compaction_start","reason":"threshold"}`, + `#pause`, `#pause`, `#pause`, `#pause`, `#pause`, `#pause`, `#pause`, `#pause`, + `{"type":"compaction_end","reason":"threshold","result":{"summary":"s","tokensBefore":9000},"aborted":false,"willRetry":false}`, + `{"type":"agent_settled"}`, + ) + adapter.settlementGrace = 500 * time.Millisecond + stream, err := adapter.Start(context.Background(), Request{Model: "opencode-go/kimi-k2.6", Prompt: "prompt"}) + if err != nil { + t.Fatalf("Start: %v", err) + } + response, err := stream.Wait(context.Background()) + if err != nil { + t.Fatalf("Wait: %v", err) + } + if string(response.StructuredOutput) != `{"ok":true}` { + t.Fatalf("StructuredOutput = %q, want answer after threshold compaction settles", response.StructuredOutput) + } +} + +type piRPCTestTurn struct { + text string + toolCall bool + stopReason string + errorMessage string + usage map[string]any +} + +func (turn piRPCTestTurn) message() map[string]any { + content := []map[string]any{} + if turn.text != "" { + content = append(content, map[string]any{"type": "text", "text": turn.text}) + } + if turn.toolCall { + content = append(content, map[string]any{"type": "toolCall", "id": "call-1", "name": "cr_diff", "arguments": map[string]any{}}) + } + message := map[string]any{"role": "assistant", "content": content, "usage": turn.usage, "stopReason": turn.stopReason} + if turn.errorMessage != "" { + message["errorMessage"] = turn.errorMessage + } + return message +} + +func (turn piRPCTestTurn) end() string { + return piRPCTestLine(map[string]any{"type": "message_end", "message": turn.message()}) +} + +func piRPCTestAgentEnd(willRetry bool, turns ...piRPCTestTurn) string { + messages := make([]map[string]any, 0, len(turns)) + for _, turn := range turns { + messages = append(messages, turn.message()) + } + return piRPCTestLine(map[string]any{"type": "agent_end", "messages": messages, "willRetry": willRetry}) +} + +func piRPCTestUsage(input, output int, cache map[string]any, cost float64) map[string]any { + usage := map[string]any{"input": input, "output": output, "cost": map[string]any{"total": cost}} + for key, value := range cache { + usage[key] = value + } + return usage +} + +func piRPCTestLine(event map[string]any) string { + data, err := json.Marshal(event) + if err != nil { + panic(err) + } + return string(data) +} + +func piRPCTestPtr[T any](value T) *T { return &value } + +func formatPiRPCTestUsage(usage Usage) string { + data, _ := json.Marshal(usage) + return string(data) +} + +func piRPCScriptAdapter(t *testing.T, timeout time.Duration, script ...string) *PiRPCAdapter { + t.Helper() + return NewPiRPCAdapter(PiRPCOptions{ + Command: os.Args[0], + commandArgsPrefix: piRPCHelperPrefix(), + Env: append(piRPCHelperEnv("script", filepath.Join(t.TempDir(), "record.json")), "LLM_PI_RPC_SCRIPT="+strings.Join(script, "\n")), + Timeout: timeout, + }) +} + func TestPiRPCRejectsUnsafeSpecs(t *testing.T) { adapter := NewPiRPCAdapter(PiRPCOptions{}) req := Request{Model: "opencode-go/kimi-k2.6", Prompt: "prompt"} @@ -1302,6 +1828,10 @@ func TestPiRPCRejectsUnsafeSpecs(t *testing.T) { {name: "tools allowlist", args: append([]string{"--tools", "read"}, args...)}, {name: "missing no tools", args: removeFlag(args, "--no-tools")}, {name: "missing no extensions", args: removeFlag(args, "--no-extensions")}, + {name: "missing context disable", args: removeFlag(args, "--no-context-files")}, + {name: "missing append override", args: removeFlagWithValue(args, "--append-system-prompt")}, + {name: "nonempty append override", args: replaceFlagValue(args, "--append-system-prompt", "ambient instructions")}, + {name: "duplicate append override", args: append(append([]string(nil), args...), "--append-system-prompt", "")}, {name: "unexpected flag", args: append([]string{"--unexpected"}, args...)}, {name: "text mode", args: replaceFlagValue(args, "--mode", "text")}, {name: "wrong system prompt", args: replaceFlagValue(args, "--system-prompt", "be loose")}, @@ -1328,6 +1858,9 @@ func TestPiRPCRejectsUnsafeReviewerSpecs(t *testing.T) { }{ {name: "missing builtin disable", args: removeFlag(args, "--no-builtin-tools")}, {name: "missing context disable", args: removeFlag(args, "--no-context-files")}, + {name: "missing append override", args: removeFlagWithValue(args, "--append-system-prompt")}, + {name: "nonempty append override", args: replaceFlagValue(args, "--append-system-prompt", "ambient instructions")}, + {name: "duplicate append override", args: append(append([]string(nil), args...), "--append-system-prompt", "")}, {name: "missing project approval disable", args: removeFlag(args, "--no-approve")}, {name: "all tools disabled", args: append(removeFlagWithValue(args, "--tools"), "--no-tools")}, {name: "native bash added", args: replaceFlagValue(args, "--tools", piRPCReviewerToolNames+",bash")}, @@ -1417,17 +1950,62 @@ func TestPiRPCHelperProcess(_ *testing.T) { } os.Exit(0) } + if os.Getenv("LLM_PI_RPC_SYNTHETIC_AUTH_REFRESH") == "1" { + // Only this opt-in fixture passes a synthetic selected store and home. + record.Env["PI_CODING_AGENT_DIR"] = os.Getenv("PI_CODING_AGENT_DIR") + record.Env["HOME"] = os.Getenv("HOME") + data, _ := json.Marshal(record) + _ = os.WriteFile(recordPath, data, 0o600) // #nosec G703 -- fixture-owned record path. + _ = os.WriteFile(filepath.Join(record.Env["PI_CODING_AGENT_DIR"], "auth.json"), []byte(`{"fixture":"refreshed synthetic auth state"}`), 0o600) // #nosec G703 -- opt-in synthetic fixture store, never real credentials. + } + // Pi RPC stays alive after agent_settled and shuts down when stdin closes. + stdinClosed := make(chan struct{}) + go func() { + for scanner.Scan() { + } + close(stdinClosed) + }() + awaitStdinClose := func() { + select { + case <-stdinClosed: + case <-time.After(10 * time.Second): + } + } + settle := func() { + fmt.Println(`{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}],"stopReason":"stop"}],"willRetry":false}`) + fmt.Println(`{"type":"agent_settled"}`) + awaitStdinClose() + } switch os.Getenv("LLM_HELPER_MODE") { case "success", "slow-success": if os.Getenv("LLM_HELPER_MODE") == "slow-success" { time.Sleep(slowSuccessSleep) } - fmt.Println(`{"id":"prompt-1","type":"response","command":"prompt","success":true}`) + fmt.Println(`{"id":"prompt-1","type":"response","command":"prompt","success":true,"data":{"disposition":"started"}}`) fmt.Println(`{"type":"agent_start","sessionId":"session-1"}`) fmt.Println(`{"type":"message_end","message":{"role":"user","content":"review this diff"}}`) - fmt.Println(`{"type":"message_end","message":{"role":"assistant","content":[{"type":"thinking","text":"ignored"},{"type":"text","text":"{\"ok\":true}"}],"usage":{"tokensIn":3915,"tokensOut":50,"cacheRead":5,"cacheWrite":7,"cost":{"total":0.00392005}}}}`) - fmt.Println(`{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}]}]}`) + fmt.Println(`{"type":"message_end","message":{"role":"assistant","content":[{"type":"thinking","text":"ignored"},{"type":"text","text":"{\"ok\":true}"}],"usage":{"tokensIn":3915,"tokensOut":50,"cacheRead":5,"cacheWrite":7,"cost":{"total":0.00392005}},"stopReason":"stop"}}`) + fmt.Println(`{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}],"usage":{"tokensIn":3915,"tokensOut":50,"cacheRead":5,"cacheWrite":7,"cost":{"total":0.00392005}},"stopReason":"stop"}],"willRetry":false}`) + fmt.Println(`{"type":"agent_settled"}`) + awaitStdinClose() + case "script": + for _, line := range strings.Split(os.Getenv("LLM_PI_RPC_SCRIPT"), "\n") { + switch line { + case "#pause": + // Emulate retry delay; Pi shuts down instead of retrying once stdin closes. + select { + case <-stdinClosed: + os.Exit(0) + case <-time.After(100 * time.Millisecond): + } + case "#exit": + os.Exit(0) + default: + fmt.Println(line) + } + } + awaitStdinClose() case "streaming-partials": fmt.Println(`{"id":"prompt-1","type":"response","command":"prompt","success":true}`) fmt.Println(`{"type":"agent_start","sessionId":"session-1"}`) @@ -1472,7 +2050,9 @@ func TestPiRPCHelperProcess(_ *testing.T) { fmt.Println(string(data)) } fmt.Println(`{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}],"api":"openai-completions","provider":"opencode-go","model":"deepseek-v4-pro","usage":{"input":100,"output":50,"cacheRead":0,"cacheWrite":0,"totalTokens":150,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1700000000000}}`) - fmt.Println(`{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}]}]}`) + fmt.Println(`{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}]}],"willRetry":false}`) + fmt.Println(`{"type":"agent_settled"}`) + awaitStdinClose() case "prompt-failure": fmt.Println(`{"id":"prompt-1","type":"response","command":"prompt","success":false,"error":"No API key found for opencode-go"}`) case "tool": @@ -1485,28 +2065,28 @@ func TestPiRPCHelperProcess(_ *testing.T) { fmt.Printf("{\"type\":\"tool_execution_start\",\"toolCallId\":\"%s\",\"toolName\":%q,\"args\":{}}\n", tool, tool) fmt.Printf("{\"type\":\"tool_execution_end\",\"toolCallId\":\"%s\",\"toolName\":%q,\"result\":{\"content\":[{\"type\":\"text\",\"text\":\"ok\"}]}}\n", tool, tool) } - fmt.Println(`{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}]}}`) - fmt.Println(`{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}]}]}`) + fmt.Println(`{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}],"stopReason":"stop"}}`) + settle() case "reviewer-log-flood": fmt.Fprintln(os.Stderr, strings.Repeat("stderr flood\n", 1000)) fmt.Println(`{"id":"prompt-1","type":"response","command":"prompt","success":true}`) for i := 0; i < 20; i++ { fmt.Printf("{\"type\":\"tool_execution_end\",\"toolCallId\":\"tool-%d\",\"toolName\":\"cr_read\",\"result\":{\"content\":[{\"type\":\"text\",\"text\":%q}]}}\n", i, strings.Repeat("tool output ", 500)) } - fmt.Println(`{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}]}}`) - fmt.Println(`{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}]}]}`) + fmt.Println(`{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}],"stopReason":"stop"}}`) + settle() case "reviewer-diff-failure-log-flood": fmt.Fprintln(os.Stderr, strings.Repeat("stderr flood\n", 1000)) fmt.Println(`{"id":"prompt-1","type":"response","command":"prompt","success":true}`) fmt.Println(`{"type":"tool_execution_start","toolCallId":"diff-1","toolName":"cr_diff","args":{}}`) fmt.Println(`{"type":"tool_execution_end","toolCallId":"diff-1","toolName":"cr_diff","result":{"content":[{"type":"text","text":"fixed diff unavailable"}],"isError":true}}`) - fmt.Println(`{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}]}}`) - fmt.Println(`{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}]}]}`) + fmt.Println(`{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}],"stopReason":"stop"}}`) + settle() case "reviewer-diff-incomplete": fmt.Println(`{"id":"prompt-1","type":"response","command":"prompt","success":true}`) fmt.Println(`{"type":"tool_execution_start","toolCallId":"diff-1","toolName":"cr_diff","args":{}}`) - fmt.Println(`{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}]}}`) - fmt.Println(`{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}]}]}`) + fmt.Println(`{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}],"stopReason":"stop"}}`) + settle() case "sleep": time.Sleep(10 * time.Second) case "malformed": @@ -1514,10 +2094,12 @@ func TestPiRPCHelperProcess(_ *testing.T) { case "no-agent-end": fmt.Println(`{"id":"prompt-1","type":"response","command":"prompt","success":true}`) fmt.Println(`{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}]}}`) - case "agent-end-exit-failure": + case "settled-exit-failure": fmt.Println(`{"id":"prompt-1","type":"response","command":"prompt","success":true}`) - fmt.Println(`{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}]}}`) - fmt.Println(`{"type":"agent_end","messages":[{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}]}]}`) + fmt.Println(`{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"{\"ok\":true}"}],"stopReason":"stop"}}`) + fmt.Println(`{"type":"agent_end","messages":[],"willRetry":false}`) + fmt.Println(`{"type":"agent_settled"}`) + awaitStdinClose() os.Exit(42) case "exit-before-agent-end": fmt.Println(`{"id":"prompt-1","type":"response","command":"prompt","success":true}`) @@ -1659,3 +2241,12 @@ func TestPiRPCToolUseKillsProcessGroup(t *testing.T) { return !processExists(pid) }) } + +// Absence is distinct from an explicitly empty append source in Pi. +func assertPiRPCEmptyAppend(t *testing.T, args []string) { + t.Helper() + value, present := flagValueOK(args, "--append-system-prompt") + if !present || value != "" || countPiRPCFlag(args, "--append-system-prompt") != 1 { + t.Fatalf("append system prompt = %q (present %v) in %#v, want exactly one empty source", value, present, args) + } +} diff --git a/internal/llmadapters/pi_rpc_tool_evidence.go b/internal/llmadapters/pi_rpc_tool_evidence.go new file mode 100644 index 00000000..5e189619 --- /dev/null +++ b/internal/llmadapters/pi_rpc_tool_evidence.go @@ -0,0 +1,287 @@ +package llmadapters + +import ( + "bytes" + "encoding/json" + "strconv" + "strings" + "unicode/utf8" + + "github.com/open-cli-collective/codereview-cli/internal/llm" +) + +const ( + piRPCTraceMaxCalls = 256 + piRPCTraceMaxIdentityBytes = 64 * 1024 + piRPCTraceMaxCallIDBytes = 256 + piRPCTraceMaxPathBytes = 4096 +) + +type piRPCReviewerToolTrace struct { + entries []piRPCReviewerToolCall + byID map[string]int + identityBytes int + droppedEvents uint64 + truncated bool + streamComplete bool +} + +type piRPCReviewerToolCall struct { + evidence llm.ReviewerToolCallEvidence + ambiguous bool +} + +func (trace *piRPCReviewerToolTrace) observe(event piRPCEvent) { + if (!event.toolStarted && !event.toolCompleted) || (event.toolName != "cr_read" && event.toolName != "cr_diff") { + return + } + id := event.toolCallID + if len(id) > piRPCTraceMaxCallIDBytes { + trace.truncated = true + id = "" + } + if !utf8.ValidString(id) || strings.TrimSpace(id) == "" { + id = "" + } + if index, exists := trace.byID[id]; id != "" && exists { + entry := &trace.entries[index] + if event.toolStarted || entry.evidence.EndObserved || entry.evidence.Tool != event.toolName { + entry.ambiguous = true + entry.evidence.Status = llm.ReviewerToolCallIncomplete + entry.evidence.ProvenanceIssue = "ambiguous_call_id" + entry.evidence.Output = nil + } + if event.toolCompleted { + entry.evidence.EndObserved = true + if !entry.ambiguous { + finishPiRPCToolCall(&entry.evidence, event) + } + } + return + } + if len(trace.entries) >= piRPCTraceMaxCalls || trace.identityBytes+len(id) > piRPCTraceMaxIdentityBytes { + trace.droppedEvents++ + trace.truncated = true + return + } + entry := piRPCReviewerToolCall{evidence: llm.ReviewerToolCallEvidence{ + CallID: id, Tool: event.toolName, Status: llm.ReviewerToolCallIncomplete, + StartObserved: event.toolStarted, EndObserved: event.toolCompleted, + }} + if event.toolStarted { + trace.addArguments(&entry.evidence, event.toolArgs) + } else { + entry.ambiguous = true + entry.evidence.ProvenanceIssue = "missing_start" + } + if id == "" { + entry.ambiguous = true + entry.evidence.ProvenanceIssue = "call_id_unavailable" + } else { + if trace.byID == nil { + trace.byID = make(map[string]int) + } + trace.byID[id] = len(trace.entries) + } + trace.identityBytes += len(id) + len(entry.evidence.Path) + trace.entries = append(trace.entries, entry) +} + +func (trace *piRPCReviewerToolTrace) addArguments(call *llm.ReviewerToolCallEvidence, raw json.RawMessage) { + var args map[string]json.RawMessage + if err := json.Unmarshal(raw, &args); err != nil || args == nil { + call.ProvenanceIssue = "arguments_unavailable" + return + } + // The helper decodes into a Go struct, which accepts case-fold aliases. + // Do not attribute a default view or exact path/range from a map that could + // disagree with the helper about Tool, View, Path, Offset, or Limit. A Tool + // alias can even override the extension's lowercase tool field depending on + // the original parameter insertion order. + for key := range args { + for _, canonical := range []string{"tool", "path", "view", "offset", "limit"} { + if key != canonical && strings.EqualFold(key, canonical) { + call.ProvenanceIssue = "arguments_ambiguous" + return + } + } + } + requestedRange := &llm.ReviewerToolRange{} + for name, target := range map[string]*int64{"offset": &requestedRange.Offset, "limit": &requestedRange.Limit} { + if value, present := args[name]; present { + if bytes.Equal(bytes.TrimSpace(value), []byte("null")) || json.Unmarshal(value, target) != nil || *target < 0 { + call.ProvenanceIssue = "range_unavailable" + requestedRange = nil + break + } + } + } + call.Range = requestedRange + if call.Tool != "cr_read" { + return // cr_diff is a fixed whole-diff artifact, never a per-file read. + } + call.ReadView = llm.ReviewerToolReadFile + if rawView, present := args["view"]; present { + view, exact := piRPCExactString(rawView) + switch { + case exact && view == "symlink": + call.ReadView = llm.ReviewerToolReadSymlink + case exact && view == "": + // An empty view selects the helper's ordinary-file branch. + default: + call.ReadView = "" + call.ProvenanceIssue = "read_view_unavailable" + } + } + path, exact := piRPCExactString(args["path"]) + if !exact || !canonicalPiRPCEvidencePath(path) { + call.ProvenanceIssue = "path_unavailable" + return + } + if len(path) > piRPCTraceMaxPathBytes || trace.identityBytes+len(call.CallID)+len(path) > piRPCTraceMaxIdentityBytes { + trace.truncated = true + call.ProvenanceIssue = "path_omitted" + return + } + call.Path = path +} + +func canonicalPiRPCEvidencePath(path string) bool { + // The helper cleans paths and trims whitespace. Attribute only an already + // canonical argument so a normalization cannot silently name another file. + if path == "" || strings.TrimSpace(path) != path || !utf8.ValidString(path) || strings.ContainsAny(path, "\\\x00") || strings.HasPrefix(path, "/") { + return false + } + if len(path) >= 2 && path[1] == ':' { + return false + } + for _, part := range strings.Split(path, "/") { + if part == "" || part == "." || part == ".." { + return false + } + switch strings.ToLower(part) { + case ".git", ".hg", ".svn": + return false + } + } + return true +} + +func finishPiRPCToolCall(call *llm.ReviewerToolCallEvidence, event piRPCEvent) { + if !event.toolOutcomeKnown { + call.ProvenanceIssue = "outcome_unavailable" + return + } + call.Status = llm.ReviewerToolCallSucceeded + if event.toolFailed { + call.Status = llm.ReviewerToolCallFailed + } + call.Output = event.toolOutput +} + +func (trace *piRPCReviewerToolTrace) snapshot() *llm.ReviewerToolTrace { + result := &llm.ReviewerToolTrace{ + Version: 1, Source: "pi_rpc", StreamComplete: trace.streamComplete, + Truncated: trace.truncated, DroppedEvents: trace.droppedEvents, + Calls: make([]llm.ReviewerToolCallEvidence, len(trace.entries)), + } + for i, entry := range trace.entries { + result.Calls[i] = entry.evidence + if !entry.evidence.EndObserved && entry.evidence.ProvenanceIssue == "" { + result.Calls[i].ProvenanceIssue = "missing_end" + } + } + return llm.CloneReviewerToolEvidence(&llm.ReviewerToolEvidence{Trace: result}).Trace +} + +func piRPCBoolPresent(raw json.RawMessage) bool { + value := bytes.TrimSpace(raw) + return bytes.Equal(value, []byte("true")) || bytes.Equal(value, []byte("false")) +} + +func piRPCToolOutcomeKnown(event, result map[string]json.RawMessage, failed bool) bool { + if failed { + return true // A malformed field must never erase an explicit failure. + } + known := false + for _, object := range []map[string]json.RawMessage{event, result} { + if value, present := object["isError"]; present { + if !piRPCBoolPresent(value) { + return false + } + known = true + } + } + return known +} + +func piRPCToolCallID(raw map[string]json.RawMessage) string { + var id string + for _, key := range []string{"toolCallId", "tool_call_id"} { + if value, present := raw[key]; present { + decoded, exact := piRPCExactString(value) + if !exact || decoded == "" || (id != "" && id != decoded) { + return "" + } + id = decoded + } + } + return id +} + +// encoding/json replaces invalid UTF-8 and unpaired UTF-16 surrogates. Identity +// fields must reject those inputs rather than collide with a real U+FFFD name. +func piRPCExactString(raw json.RawMessage) (string, bool) { + raw = bytes.TrimSpace(raw) + var decoded string + if len(raw) < 2 || raw[0] != '"' || !utf8.Valid(raw) || json.Unmarshal(raw, &decoded) != nil { + return "", false + } + for i := 1; i < len(raw)-1; i++ { + if raw[i] != '\\' { + continue + } + i++ + if raw[i] != 'u' { + continue + } + unit, err := strconv.ParseUint(string(raw[i+1:i+5]), 16, 16) + if err != nil { + return "", false + } + i += 4 + if unit >= 0xdc00 && unit <= 0xdfff { + return "", false + } + if unit < 0xd800 || unit > 0xdbff { + continue + } + if i+7 >= len(raw) || raw[i+1] != '\\' || raw[i+2] != 'u' { + return "", false + } + low, err := strconv.ParseUint(string(raw[i+3:i+7]), 16, 16) + if err != nil || low < 0xdc00 || low > 0xdfff { + return "", false + } + i += 6 + } + return decoded, true +} + +func parsePiRPCToolOutput(raw json.RawMessage) *llm.ReviewerToolOutput { + var details struct { + Output *struct { + Version int `json:"version"` + Bytes *int64 `json:"bytes"` + Truncated *bool `json:"truncated"` + } `json:"codereview_tool_output"` + } + if json.Unmarshal(raw, &details) != nil || details.Output == nil { + return nil + } + output := details.Output + if output.Version != 1 || output.Bytes == nil || *output.Bytes < 0 || *output.Bytes > 1024*1024 || output.Truncated == nil { + return nil + } + return &llm.ReviewerToolOutput{Bytes: *output.Bytes, Truncated: *output.Truncated} +} diff --git a/internal/llmadapters/pi_rpc_tool_evidence_test.go b/internal/llmadapters/pi_rpc_tool_evidence_test.go new file mode 100644 index 00000000..739dc552 --- /dev/null +++ b/internal/llmadapters/pi_rpc_tool_evidence_test.go @@ -0,0 +1,303 @@ +package llmadapters + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/open-cli-collective/codereview-cli/internal/llm" +) + +func TestPiRPCTraceCorrelatesInterleavedReadAndDiffCalls(t *testing.T) { + var trace piRPCReviewerToolTrace + observePiTraceJSON(t, &trace, + `{"type":"tool_execution_start","toolCallId":"read-1","toolName":"cr_read","args":{"path":"src/世界 file.go","offset":10,"limit":20}}`, + `{"type":"tool_execution_start","toolCallId":"diff-1","toolName":"cr_diff","args":{"offset":30,"limit":40}}`, + `{"type":"tool_execution_end","toolCallId":"diff-1","toolName":"cr_diff","isError":false,"result":{"details":{"codereview_tool_output":{"version":1,"bytes":40,"truncated":true}}}}`, + `{"type":"tool_execution_end","toolCallId":"read-1","toolName":"cr_read","isError":false,"result":{"content":[{"type":"text","text":"private source text"}],"details":{"codereview_tool_output":{"version":1,"bytes":20,"truncated":false}}}}`, + `{"type":"tool_execution_start","toolCallId":"read-2","toolName":"cr_read","args":{"path":"src/世界 file.go"}}`, + `{"type":"tool_execution_end","toolCallId":"read-2","toolName":"cr_read","result":{"isError":true,"content":[{"type":"text","text":"private failure text"}]}}`, + ) + trace.streamComplete = true + got := trace.snapshot() + want := &llm.ReviewerToolTrace{Version: 1, Source: "pi_rpc", StreamComplete: true, Calls: []llm.ReviewerToolCallEvidence{ + {CallID: "read-1", Tool: "cr_read", Path: "src/世界 file.go", ReadView: llm.ReviewerToolReadFile, Status: llm.ReviewerToolCallSucceeded, StartObserved: true, EndObserved: true, Range: &llm.ReviewerToolRange{Offset: 10, Limit: 20}, Output: &llm.ReviewerToolOutput{Bytes: 20}}, + {CallID: "diff-1", Tool: "cr_diff", Status: llm.ReviewerToolCallSucceeded, StartObserved: true, EndObserved: true, Range: &llm.ReviewerToolRange{Offset: 30, Limit: 40}, Output: &llm.ReviewerToolOutput{Bytes: 40, Truncated: true}}, + {CallID: "read-2", Tool: "cr_read", Path: "src/世界 file.go", ReadView: llm.ReviewerToolReadFile, Status: llm.ReviewerToolCallFailed, StartObserved: true, EndObserved: true, Range: &llm.ReviewerToolRange{}}, + }} + if !reflect.DeepEqual(got, want) { + t.Fatalf("trace = %#v, want %#v", got, want) + } + data, err := json.Marshal(got) + if err != nil || strings.Contains(string(data), "private") { + t.Fatalf("trace leaked tool output: %s, err=%v", data, err) + } + got.Calls[0].Range.Offset = 999 + got.Calls[0].Output.Bytes = 999 + if !reflect.DeepEqual(trace.snapshot(), want) { + t.Fatal("snapshot mutation changed collected trace") + } +} + +func TestPiRPCTraceDoesNotInferMissingOrAmbiguousExecution(t *testing.T) { + start := `{"type":"tool_execution_start","toolCallId":"call","toolName":"cr_read","args":{"path":"file.go"}}` + end := `{"type":"tool_execution_end","toolCallId":"call","toolName":"cr_read","isError":false}` + for _, tt := range []struct { + name string + lines []string + issue string + }{ + {"start only", []string{start}, "missing_end"}, + {"end only", []string{end}, "missing_start"}, + {"end before start", []string{end, start}, "ambiguous_call_id"}, + {"duplicate start", []string{start, start, end}, "ambiguous_call_id"}, + {"duplicate end", []string{start, end, end}, "ambiguous_call_id"}, + {"mismatched tool", []string{start, strings.ReplaceAll(end, "cr_read", "cr_diff")}, "ambiguous_call_id"}, + {"missing outcome", []string{start, strings.ReplaceAll(end, `,"isError":false`, "")}, "outcome_unavailable"}, + {"missing identity", []string{strings.ReplaceAll(start, `"toolCallId":"call",`, "")}, "call_id_unavailable"}, + } { + t.Run(tt.name, func(t *testing.T) { + var trace piRPCReviewerToolTrace + observePiTraceJSON(t, &trace, tt.lines...) + got := trace.snapshot() + if len(got.Calls) != 1 || got.Calls[0].Status != llm.ReviewerToolCallIncomplete || got.Calls[0].ProvenanceIssue != tt.issue { + t.Fatalf("calls = %#v, want incomplete with %s", got.Calls, tt.issue) + } + }) + } +} + +func TestPiRPCTraceIgnoresModelClaimsAndUnattributedDiffPaths(t *testing.T) { + var trace piRPCReviewerToolTrace + observePiTraceJSON(t, &trace, + `{"type":"toolcall_start","toolCallId":"claim","toolName":"cr_read","args":{"path":"claim.go"}}`, + `{"type":"tool_execution_start","toolCallId":"search","toolName":"cr_search","args":{"path":"search.go","query":"private query"}}`, + `{"type":"tool_execution_end","toolCallId":"search","toolName":"cr_search","isError":false}`, + `{"type":"tool_execution_start","toolCallId":"diff","toolName":"cr_diff","args":{"path":"claim.go"}}`, + `{"type":"tool_execution_end","toolCallId":"diff","toolName":"cr_diff","isError":false,"result":{"content":[{"type":"text","text":"[cr-range offset=0 end=100 total=100 next_offset=-1]\n"}]}}`, + ) + got := trace.snapshot() + if len(got.Calls) != 1 || got.Calls[0].Tool != "cr_diff" || got.Calls[0].Path != "" || got.Calls[0].Output != nil { + t.Fatalf("calls = %#v, want one pathless diff with unknown output bounds", got.Calls) + } + if got.StreamComplete { + t.Fatal("a tool end must not establish that the stream ended cleanly") + } +} + +func TestPiRPCTraceDistinguishesSymlinkMetadataFromRegularBodyReads(t *testing.T) { + stream := &piRPCStream{allowReviewerTools: true} + for _, line := range []string{ + `{"type":"tool_execution_start","toolCallId":"metadata","toolName":"cr_read","args":{"path":"changed-link","view":"symlink","offset":20,"limit":40}}`, + `{"type":"tool_execution_end","toolCallId":"metadata","toolName":"cr_read","isError":false,"result":{"content":[{"type":"text","text":"{\"payload_omitted_reason\":\"oversize\",\"resolved_path\":\"private-target\"}"}]}}`, + `{"type":"tool_execution_start","toolCallId":"body","toolName":"cr_read","args":{"path":"changed-link"}}`, + `{"type":"tool_execution_end","toolCallId":"body","toolName":"cr_read","isError":false}`, + } { + event, err := parsePiRPCEvent([]byte(line)) + if err != nil { + t.Fatal(err) + } + stream.observeReviewerToolEvent(event) + } + got := stream.reviewerToolEvidence() + if got.DiffStatus != llm.DiffToolStatusNotInvoked { + t.Fatal("successful metadata/body reads must not satisfy the existing required-diff gate") + } + calls := got.Trace.Calls + if len(calls) != 2 || calls[0].Path != "changed-link" || calls[1].Path != "changed-link" || calls[0].ReadView != llm.ReviewerToolReadSymlink || calls[1].ReadView != llm.ReviewerToolReadFile { + t.Fatalf("calls = %#v, want separate metadata and file views for the same path", calls) + } + if calls[0].Range == nil || calls[0].Range.Offset != 20 || calls[0].Range.Limit != 40 || calls[0].Status != llm.ReviewerToolCallSucceeded || calls[0].Output != nil { + t.Fatalf("metadata call = %#v, want observed success with requested metadata range and unknown transport bounds", calls[0]) + } + data, err := json.Marshal(got.Trace) + if err != nil || strings.Contains(string(data), "private-target") || strings.Contains(string(data), "payload_omitted_reason") { + t.Fatalf("trace copied symlink artifact content: %s, err %v", data, err) + } +} + +func TestPiRPCTraceDoesNotAssumeAnInvalidViewIsAFileRead(t *testing.T) { + for _, rawView := range []string{`null`, `false`, `1`, `"unknown"`, `"symlink "`, `"\ud800"`} { + var trace piRPCReviewerToolTrace + observePiTraceJSON(t, &trace, + `{"type":"tool_execution_start","toolCallId":"call","toolName":"cr_read","args":{"path":"file.go","view":`+rawView+`}}`, + `{"type":"tool_execution_end","toolCallId":"call","toolName":"cr_read","isError":true}`, + ) + call := trace.snapshot().Calls[0] + if call.ReadView != "" || call.ProvenanceIssue != "read_view_unavailable" || call.Status != llm.ReviewerToolCallFailed { + t.Fatalf("view %s became a file read: %#v", rawView, call) + } + } + var trace piRPCReviewerToolTrace + observePiTraceJSON(t, &trace, `{"type":"tool_execution_start","toolCallId":"call","toolName":"cr_read","args":{"path":"file.go","view":""}}`) + if call := trace.snapshot().Calls[0]; call.ReadView != llm.ReviewerToolReadFile { + t.Fatalf("empty helper view = %#v, want ordinary file request", call) + } +} + +func TestPiRPCTraceRejectsCaseFoldArgumentAliasesBeforeAttribution(t *testing.T) { + for _, args := range []string{ + `{"tool":"ignored","Tool":"cr_diff","path":"file.go"}`, + `{"Tool":"cr_diff","path":"file.go"}`, + `{"path":"changed-link","View":"symlink"}`, + `{"path":"changed-link","view":"","VIEW":"symlink"}`, + `{"path":"file.go","Path":"different.go"}`, + `{"Path":"different.go"}`, + `{"path":"file.go","Offset":100}`, + `{"path":"file.go","limit":100,"LIMIT":1}`, + `{"path":"file.go","offſet":100}`, + } { + var trace piRPCReviewerToolTrace + observePiTraceJSON(t, &trace, + `{"type":"tool_execution_start","toolCallId":"call","toolName":"cr_read","args":`+args+`}`, + `{"type":"tool_execution_end","toolCallId":"call","toolName":"cr_read","isError":false}`, + ) + call := trace.snapshot().Calls[0] + if call.Path != "" || call.ReadView != "" || call.Range != nil || call.ProvenanceIssue != "arguments_ambiguous" { + t.Fatalf("case-fold arguments %s received attribution: %#v", args, call) + } + } +} + +func TestPiRPCTraceDoesNotNormalizeOrPersistUnsafePathArguments(t *testing.T) { + for _, path := range []string{"", "/private/file", "../file", "src/../file", "./file", "src//file", ".git/config", "src/.GIT/config", ".hg/config", "src/.HG/config", ".svn/config", "src/.SVN/config", "C:/private/file", `C:\private\file`, " file.go", "file.go ", "file\x00.go"} { + t.Run(fmt.Sprintf("%q", path), func(t *testing.T) { + var trace piRPCReviewerToolTrace + args, err := json.Marshal(map[string]string{"path": path}) + if err != nil { + t.Fatal(err) + } + trace.observe(piRPCEvent{toolName: "cr_read", toolCallID: "read", toolStarted: true, toolArgs: args}) + trace.observe(piRPCEvent{toolName: "cr_read", toolCallID: "read", toolCompleted: true, toolOutcomeKnown: true}) + call := trace.snapshot().Calls[0] + if call.Path != "" || call.ProvenanceIssue != "path_unavailable" { + t.Fatalf("call = %#v, want unattributed outcome", call) + } + }) + } +} + +func TestPiRPCTraceBoundsCallsAndRetainsCompletionOfEarlierCalls(t *testing.T) { + var trace piRPCReviewerToolTrace + for i := 0; i < piRPCTraceMaxCalls+1; i++ { + trace.observe(piRPCEvent{toolName: "cr_read", toolCallID: fmt.Sprintf("call-%d", i), toolStarted: true, toolArgs: json.RawMessage(`{"path":"file.go"}`)}) + } + trace.observe(piRPCEvent{toolName: "cr_read", toolCallID: "call-0", toolCompleted: true, toolOutcomeKnown: true}) + trace.observe(piRPCEvent{toolName: "cr_read", toolCallID: fmt.Sprintf("call-%d", piRPCTraceMaxCalls), toolCompleted: true, toolOutcomeKnown: true}) + got := trace.snapshot() + if len(got.Calls) != piRPCTraceMaxCalls || !got.Truncated || got.DroppedEvents != 2 || got.Calls[0].Status != llm.ReviewerToolCallSucceeded || len(trace.byID) != piRPCTraceMaxCalls { + t.Fatalf("trace bounds = calls %d, dropped %d, truncated %t, first %#v, identities %d", len(got.Calls), got.DroppedEvents, got.Truncated, got.Calls[0], len(trace.byID)) + } +} + +func TestPiRPCTraceBoundsIdentityBytesWithoutTruncatingIdentity(t *testing.T) { + var trace piRPCReviewerToolTrace + for i := 0; i < 40; i++ { + args, err := json.Marshal(map[string]string{"path": strings.Repeat("x", piRPCTraceMaxPathBytes)}) + if err != nil { + t.Fatal(err) + } + trace.observe(piRPCEvent{toolName: "cr_read", toolCallID: fmt.Sprintf("read-%d", i), toolStarted: true, toolArgs: args}) + } + if got := trace.snapshot(); !got.Truncated || trace.identityBytes > piRPCTraceMaxIdentityBytes { + t.Fatalf("trace = %#v, identity bytes %d", got, trace.identityBytes) + } + last := trace.snapshot().Calls[39] + if last.Path != "" || last.ProvenanceIssue != "path_omitted" { + t.Fatalf("last = %#v, want omitted path rather than a different truncated path", last) + } + trace.observe(piRPCEvent{toolName: "cr_read", toolCallID: strings.Repeat("i", piRPCTraceMaxCallIDBytes+1), toolStarted: true, toolArgs: json.RawMessage(`{"path":"small.go"}`)}) + last = trace.snapshot().Calls[40] + if last.CallID != "" || last.ProvenanceIssue != "call_id_unavailable" { + t.Fatalf("oversized identity = %#v, want unavailable", last) + } +} + +func TestPiRPCTraceMarksMalformedArgumentsAndUnknownOutputBounds(t *testing.T) { + for _, args := range []string{`null`, `[]`, `{}`, `{"path":"file.go","offset":null}`, `{"path":"file.go","limit":-1}`, `{"path":"file.go","offset":"0"}`} { + var trace piRPCReviewerToolTrace + trace.observe(piRPCEvent{toolName: "cr_read", toolCallID: "read", toolStarted: true, toolArgs: json.RawMessage(args)}) + if call := trace.snapshot().Calls[0]; call.ProvenanceIssue == "" { + t.Fatalf("args %s unexpectedly have complete provenance: %#v", args, call) + } + } + for _, details := range []string{`null`, `{}`, `{"codereview_tool_output":{"version":2,"bytes":1,"truncated":false}}`, `{"codereview_tool_output":{"version":1,"bytes":-1,"truncated":false}}`, `{"codereview_tool_output":{"version":1,"bytes":1}}`} { + if got := parsePiRPCToolOutput(json.RawMessage(details)); got != nil { + t.Fatalf("details %s = %#v, want unknown", details, got) + } + } +} + +func TestPiRPCTraceDoesNotChangeLegacyDiffGateOrUnsupportedEvidence(t *testing.T) { + stream := &piRPCStream{allowReviewerTools: true} + stream.observeReviewerToolEvent(piRPCEvent{toolName: "cr_diff", toolCompleted: true}) + got := stream.reviewerToolEvidence() + if got.DiffStatus != llm.DiffToolStatusSucceeded || got.Trace.Calls[0].Status != llm.ReviewerToolCallIncomplete { + t.Fatalf("evidence = %#v, want unchanged legacy gate and conservative new trace", got) + } + unsupported := &piRPCStream{} + unsupported.observeReviewerToolEvent(piRPCEvent{toolName: "cr_read", toolStarted: true}) + if unsupported.reviewerToolEvidence() != nil { + t.Fatal("non-reviewer invocation must not claim trace support") + } +} + +func TestPiRPCTraceRejectsNormalizedIdentityCollisions(t *testing.T) { + for _, raw := range []string{`"\ud800"`, `"\udc00"`, `"x\ud800y"`, `"\ud800\u0041"`, "\"bad\xff\""} { + t.Run(fmt.Sprintf("%q", raw), func(t *testing.T) { + if decoded, exact := piRPCExactString(json.RawMessage(raw)); exact { + t.Fatalf("identity %s normalized to %q", raw, decoded) + } + var trace piRPCReviewerToolTrace + observePiTraceJSON(t, &trace, + `{"type":"tool_execution_start","toolCallId":`+raw+`,"toolName":"cr_read","args":{"path":"file.go"}}`, + `{"type":"tool_execution_end","toolCallId":"�","toolName":"cr_read","isError":false}`, + ) + for _, call := range trace.snapshot().Calls { + if call.Status == llm.ReviewerToolCallSucceeded { + t.Fatalf("malformed identity established success: %#v", call) + } + } + var pathTrace piRPCReviewerToolTrace + observePiTraceJSON(t, &pathTrace, `{"type":"tool_execution_start","toolCallId":"call","toolName":"cr_read","args":{"path":`+raw+`}}`) + if call := pathTrace.snapshot().Calls[0]; call.Path != "" || call.ProvenanceIssue != "path_unavailable" { + t.Fatalf("malformed path established identity: %#v", call) + } + }) + } + for raw, want := range map[string]string{`"�"`: "�", `"\ufffd"`: "�", `"\ud83d\ude00"`: "😀", `"\\ud800"`: `\ud800`} { + if got, exact := piRPCExactString(json.RawMessage(raw)); !exact || got != want { + t.Fatalf("valid identity %s = %q, exact %t, want %q", raw, got, exact, want) + } + } +} + +func TestPiRPCTraceRejectsMalformedOutcomeAndConflictingIDAliases(t *testing.T) { + for _, raw := range []string{`"false"`, `null`, `0`} { + var trace piRPCReviewerToolTrace + observePiTraceJSON(t, &trace, + `{"type":"tool_execution_start","toolCallId":"call","toolName":"cr_read","args":{"path":"file.go"}}`, + `{"type":"tool_execution_end","toolCallId":"call","toolName":"cr_read","isError":`+raw+`,"result":{"isError":false}}`, + ) + if got := trace.snapshot().Calls[0]; got.Status != llm.ReviewerToolCallIncomplete || got.ProvenanceIssue != "outcome_unavailable" { + t.Fatalf("malformed outcome %s = %#v", raw, got) + } + } + event, err := parsePiRPCEvent([]byte(`{"type":"tool_execution_end","toolCallId":"call","tool_call_id":"different","toolName":"cr_read","isError":"malformed","result":{"isError":true}}`)) + if err != nil || event.toolCallID != "" || !event.toolOutcomeKnown || !event.toolFailed { + t.Fatalf("event = %#v, err %v, want missing identity and explicit failure", event, err) + } +} + +func observePiTraceJSON(t *testing.T, trace *piRPCReviewerToolTrace, lines ...string) { + t.Helper() + for _, line := range lines { + event, err := parsePiRPCEvent([]byte(line)) + if err != nil { + t.Fatalf("parsePiRPCEvent: %v", err) + } + trace.observe(event) + } +} diff --git a/internal/llmadapters/request_cost_evidence.go b/internal/llmadapters/request_cost_evidence.go new file mode 100644 index 00000000..7bcf12ac --- /dev/null +++ b/internal/llmadapters/request_cost_evidence.go @@ -0,0 +1,192 @@ +package llmadapters + +import ( + "bytes" + "encoding/json" + "net" + "net/http" + "net/url" + "time" + "unicode/utf8" + + "github.com/open-cli-collective/codereview-cli/internal/llm" +) + +// RequestCostEvidenceSource opts in by concrete adapter ownership, never by +// model prefix. Start's synchronous configuration errors occur before dispatch. +func (a *APIAdapter) RequestCostEvidenceSource() string { + if a.kind == apiOpenAI { + return "openai_responses" + } + return "" +} + +func newSingleSendOpenAIClient() *http.Client { + // Pin a fresh standard transport, not mutable http.DefaultTransport. With + // GetBody cleared and no idempotency header, consumed request bodies cannot + // be replayed. Explicit outer retries produce independent receipts. + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext, + ForceAttemptHTTP2: true, MaxIdleConns: 100, IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: time.Second, + } + return &http.Client{Transport: transport, Timeout: defaultAPIClientTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} +} + +func officialResponsesEndpoint(endpoint string) bool { + u, err := url.Parse(endpoint) + return err == nil && u.Scheme == "https" && u.Host == "api.openai.com" && u.User == nil && u.Path == "/v1/responses" && u.RawPath == "" && u.RawQuery == "" && !u.ForceQuery && u.Fragment == "" +} + +func (a *APIAdapter) newCostAttempt(req Request) *llm.RequestCostAttempt { + if a.kind != apiOpenAI { + return nil + } + attempt := &llm.RequestCostAttempt{RuntimeID: "openai-api-key", RequestedModel: req.Model, EndpointKind: "custom_or_unverified", TransportMode: "opaque", Dispatch: "not_dispatched", BodyState: "missing", Issues: []llm.CostEvidenceGap{}} + if req.Fast { + tier := "fast" + attempt.RequestedServiceTier = &tier + } + if a.ownedOpenAIClient != nil && a.httpClient == a.ownedOpenAIClient { + attempt.TransportMode = "single_send_v1" + } + return attempt +} + +func singleCostEvidence(a *llm.RequestCostAttempt) *llm.RequestCostEvidence { + if a == nil { + return nil + } + return llm.NormalizeRequestCostEvidence(&llm.RequestCostEvidence{Version: 1, Source: "openai_responses", Scope: "invocation", AttemptsObserved: 1, AttemptCountExact: a.Dispatch == "not_dispatched" || a.TransportMode == "single_send_v1", Gaps: []llm.CostEvidenceGap{}, Attempts: []llm.RequestCostAttempt{*a}}) +} + +// costObject isolates known keys, detecting duplicates without last-key-wins. +// Full-document syntax must be validated by the caller first. Unknown keys are +// ignored and no arbitrary provider metadata enters the returned evidence. +func costObject(data []byte, known ...string) (map[string]json.RawMessage, map[string]bool, bool) { + allowed := map[string]bool{} + for _, key := range known { + allowed[key] = true + } + fields := map[string]json.RawMessage{} + duplicates := map[string]bool{} + d := json.NewDecoder(bytes.NewReader(data)) + token, err := d.Token() + if err != nil || token != json.Delim('{') { + return nil, nil, false + } + for d.More() { + token, err = d.Token() + if err != nil { + return nil, nil, false + } + key, ok := token.(string) + if !ok { + return nil, nil, false + } + var raw json.RawMessage + if d.Decode(&raw) != nil { + return nil, nil, false + } + if !allowed[key] { + continue + } + if _, exists := fields[key]; exists { + duplicates[key] = true + } + fields[key] = raw + } + _, err = d.Token() + return fields, duplicates, err == nil +} + +func observeCostEnvelope(a *llm.RequestCostAttempt, body []byte) { + if a == nil { + return + } + if !utf8.Valid(body) || !json.Valid(body) { + a.BodyState = "malformed" + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapInvalid) + return + } + fields, duplicates, ok := costObject(body, "id", "model", "service_tier", "status", "usage") + if !ok { + a.BodyState = "malformed" + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapInvalid) + return + } + for _, field := range []struct { + key string + target **string + limit int + }{{"id", &a.ProviderResponseID, 256}, {"model", &a.ObservedModel, 256}, {"service_tier", &a.ObservedServiceTier, 64}, {"status", &a.ResponseStatus, 64}} { + raw, exists := fields[field.key] + if duplicates[field.key] { + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapDuplicate) + continue + } + if !exists || bytes.Equal(raw, []byte("null")) { + continue + } + var value string + if !llm.CostJSONUnicodeValid(raw) || json.Unmarshal(raw, &value) != nil || !llm.SafeCostString(value, field.limit) { + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapIdentity) + continue + } + *field.target = &value + } + if duplicates["usage"] { + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapDuplicate) + return + } + raw, exists := fields["usage"] + if !exists || bytes.Equal(raw, []byte("null")) { + return + } + usage, dups, ok := costObject(raw, "input_tokens", "output_tokens", "total_tokens", "input_tokens_details") + if !ok { + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapUsage) + return + } + u := &llm.OpenAIRequestUsage{} + a.Usage = u + for _, field := range []struct { + key string + target **int64 + }{{"input_tokens", &u.InputTokens}, {"output_tokens", &u.OutputTokens}, {"total_tokens", &u.TotalTokens}} { + decodeCostCounter(a, usage, dups, field.key, field.target) + } + if dups["input_tokens_details"] { + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapDuplicate) + return + } + details, exists := usage["input_tokens_details"] + if !exists || bytes.Equal(details, []byte("null")) { + return + } + detailFields, detailDups, ok := costObject(details, "cached_tokens", "cache_write_tokens") + if !ok { + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapUsage) + return + } + decodeCostCounter(a, detailFields, detailDups, "cached_tokens", &u.CacheReadTokens) + decodeCostCounter(a, detailFields, detailDups, "cache_write_tokens", &u.CacheWriteTokens) +} + +func decodeCostCounter(a *llm.RequestCostAttempt, fields map[string]json.RawMessage, duplicates map[string]bool, key string, target **int64) { + if duplicates[key] { + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapDuplicate) + return + } + raw, exists := fields[key] + if !exists || bytes.Equal(raw, []byte("null")) { + return + } + var value int64 + if json.Unmarshal(raw, &value) != nil { + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapUsage) + return + } + *target = &value +} diff --git a/internal/llmadapters/request_cost_evidence_test.go b/internal/llmadapters/request_cost_evidence_test.go new file mode 100644 index 00000000..13bb75f8 --- /dev/null +++ b/internal/llmadapters/request_cost_evidence_test.go @@ -0,0 +1,360 @@ +package llmadapters + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync/atomic" + "testing" + + "github.com/open-cli-collective/codereview-cli/internal/llm" +) + +const costWire = `{"id":"resp-synthetic","model":"observed-model","service_tier":"priority","status":"completed","usage":{"input_tokens":10,"output_tokens":3,"total_tokens":13,"input_tokens_details":{"cached_tokens":2,"cache_write_tokens":0}},"output_text":"{\"ok\":true}"}` + +type costRoundTrip func(*http.Request) (*http.Response, error) + +func (f costRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } +func costBodyAdapter(t *testing.T, status int, body io.ReadCloser) *APIAdapter { + t.Helper() + a, err := newAPIAdapter(apiOpenAI, APIOptions{APIKey: "synthetic-test-key", HTTPClient: &http.Client{Transport: costRoundTrip(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: status, Status: fmt.Sprint(status), Body: body, Header: make(http.Header)}, nil + })}}) + if err != nil { + t.Fatal(err) + } + return a +} +func oneCostAttempt(t *testing.T, r Response) llm.RequestCostAttempt { + t.Helper() + if r.RequestCostEvidence == nil || len(r.RequestCostEvidence.Attempts) != 1 { + t.Fatalf("missing bounded receipt: %#v", r.RequestCostEvidence) + } + return r.RequestCostEvidence.Attempts[0] +} + +func TestOpenAIRequestCostRetainsUsageAndObservedIdentity(t *testing.T) { + a := costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(costWire))) + _, response, err := a.execute(context.Background(), Request{Model: "requested-model", Fast: true}) + if err != nil { + t.Fatal(err) + } + r := oneCostAttempt(t, response) + if r.RequestedModel != "requested-model" || *r.ObservedModel != "observed-model" || *r.RequestedServiceTier != "fast" || *r.ObservedServiceTier != "priority" || *r.Usage.CacheWriteTokens != 0 { + t.Fatalf("collapsed request/delivered values: %#v", r) + } + if r.TransportMode != "opaque" || response.RequestCostEvidence.AttemptCountExact { + t.Fatal("injected client gained ownership") + } + *r.Usage.InputTokens = 999 + if *response.RequestCostEvidence.Attempts[0].Usage.InputTokens != 999 { + t.Fatal("unexpected local pointer semantics") + } + clone := llm.CloneRequestCostEvidence(response.RequestCostEvidence) + *clone.Attempts[0].Usage.InputTokens = 1 + if *response.RequestCostEvidence.Attempts[0].Usage.InputTokens != 999 { + t.Fatal("detached copy aliases evidence") + } +} + +func TestOpenAIRequestCostInvalidCountersMissingNullAndDuplicateFields(t *testing.T) { + for _, value := range []string{`null`, `-1`, `10.5`, `"10"`, `true`, `[]`, `9223372036854775808`} { + body := strings.Replace(costWire, `"cache_write_tokens":0`, `"cache_write_tokens":`+value, 1) + a := costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(body))) + _, response, _ := a.execute(context.Background(), Request{Model: "request"}) + r := oneCostAttempt(t, response) + if r.Usage == nil || *r.Usage.InputTokens != 10 || !slices.Contains(r.Issues, llm.CostGapUsage) { + t.Fatalf("invalid counter erased safe usage or became eligible: %s %#v", value, r) + } + if value == `-1` && (r.Usage.CacheWriteTokens == nil || *r.Usage.CacheWriteTokens != -1) { + t.Fatal("negative observation not retained") + } + if value != `-1` && r.Usage.CacheWriteTokens != nil { + t.Fatal("invalid/null counter manufactured a value") + } + } + for _, body := range []string{ + strings.Replace(costWire, `"usage":{`, `"usage":null,"ignored_usage":{`, 1), + strings.Replace(costWire, `"input_tokens_details":{`, `"input_tokens_details":null,"ignored_details":{`, 1), + strings.Replace(costWire, `"cache_write_tokens":0`, `"cache_write_tokens":0,"cache_write_tokens":1`, 1), + strings.Replace(costWire, `"id":"resp-synthetic"`, `"id":"first","id":"second"`, 1), + strings.Replace(costWire, `"input_tokens":10`, `"input_tokens":10,"input_tokens":11`, 1), + strings.Replace(costWire, `"cached_tokens":2`, `"cached_tokens":11`, 1), + strings.Replace(costWire, `"total_tokens":13`, `"total_tokens":14`, 1), + } { + a := costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(body))) + _, response, _ := a.execute(context.Background(), Request{Model: "request"}) + r := oneCostAttempt(t, response) + if !slices.Contains(r.Issues, llm.CostGapUsage) && !slices.Contains(r.Issues, llm.CostGapDuplicate) { + t.Fatalf("malformed evidence became eligible: %#v", r) + } + } +} + +func TestOpenAIRequestCostNoTextMalformedOutputAndErrorStatus(t *testing.T) { + for _, tc := range []struct { + name, body string + status int + bodyState string + }{ + {"completed refusal", strings.Replace(costWire, `"output_text":"{\"ok\":true}"`, `"output":[{"type":"message","content":[{"type":"refusal","refusal":"synthetic-secret-canary"}]}]`, 1), 200, "complete"}, + {"incomplete", strings.Replace(strings.Replace(costWire, `"status":"completed"`, `"status":"incomplete"`, 1), `"output_text":"{\"ok\":true}"`, `"output":[]`, 1), 200, "complete"}, + {"output shape", strings.Replace(costWire, `"output_text":"{\"ok\":true}"`, `"output":42`, 1), 200, "complete"}, + {"HTTP error", costWire, 429, "complete"}, + {"truncated", costWire[:len(costWire)-1], 200, "malformed"}, + } { + t.Run(tc.name, func(t *testing.T) { + a := costBodyAdapter(t, tc.status, io.NopCloser(strings.NewReader(tc.body))) + _, response, err := a.execute(context.Background(), Request{Model: "request"}) + if err == nil { + t.Fatal("task error converted to success") + } + r := oneCostAttempt(t, response) + if r.BodyState != tc.bodyState || *r.HTTPStatus != tc.status { + t.Fatalf("receipt=%#v", r) + } + if tc.bodyState == "complete" && (r.Usage == nil || *r.Usage.InputTokens != 10) { + t.Fatal("no-text/error discarded usage") + } + if tc.bodyState != "complete" && r.Usage != nil { + t.Fatal("malformed JSON prefix scraped") + } + encoded, _ := json.Marshal(response.RequestCostEvidence) + if strings.Contains(string(encoded), "synthetic-secret-canary") { + t.Fatal("refusal text leaked into cost evidence") + } + }) + } +} + +type failingCostBody struct{} + +func (failingCostBody) Read([]byte) (int, error) { return 0, io.ErrUnexpectedEOF } +func (failingCostBody) Close() error { return nil } +func TestOpenAIRequestCostTransportBodyLimitAndCancellation(t *testing.T) { + for _, tc := range []struct { + body io.ReadCloser + state string + }{{failingCostBody{}, "read_error"}, {io.NopCloser(strings.NewReader(strings.Repeat("x", apiResponseLogLimit+1))), "over_limit"}} { + a := costBodyAdapter(t, 200, tc.body) + _, r, err := a.execute(context.Background(), Request{Model: "request"}) + if err == nil { + t.Fatal("body error lost") + } + receipt := oneCostAttempt(t, r) + if receipt.BodyState != tc.state || receipt.Dispatch != "possibly_dispatched" || receipt.Usage != nil { + t.Fatalf("receipt=%#v", receipt) + } + } + var calls atomic.Int32 + a, err := newAPIAdapter(apiOpenAI, APIOptions{APIKey: "synthetic-test-key", HTTPClient: &http.Client{Transport: costRoundTrip(func(*http.Request) (*http.Response, error) { calls.Add(1); return nil, context.DeadlineExceeded })}}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, r, _ := a.execute(ctx, Request{Model: "request"}) + if calls.Load() != 0 || oneCostAttempt(t, r).Dispatch != "not_dispatched" { + t.Fatal("pre-Do cancel dispatched") + } + _, r, _ = a.execute(context.Background(), Request{Model: "request"}) + if calls.Load() != 1 || oneCostAttempt(t, r).Dispatch != "possibly_dispatched" { + t.Fatal("Do error incorrectly free") + } + ctx, cancel = context.WithCancel(context.Background()) + a.httpClient = &http.Client{Transport: costRoundTrip(func(*http.Request) (*http.Response, error) { + cancel() + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(costWire)), Header: make(http.Header)}, nil + })} + _, r, _ = a.execute(ctx, Request{Model: "request"}) + if oneCostAttempt(t, r).Usage == nil { + t.Fatal("complete response lost on cancellation") + } +} + +// This _test.go-only assembly seam redirects the adapter-owned standard TLS +// transport to a synthetic local server. No production HTTPClient trust toggle +// is exposed, and arbitrary injected clients remain opaque. +func ownedCostTestAdapter(t *testing.T, handler http.Handler) (*APIAdapter, *httptest.Server) { + t.Helper() + server := httptest.NewTLSServer(handler) + t.Cleanup(server.Close) + a, err := newAPIAdapter(apiOpenAI, APIOptions{APIKey: "synthetic-test-key"}) + if err != nil { + t.Fatal(err) + } + transport := a.ownedOpenAIClient.Transport.(*http.Transport) + transport.Proxy = nil // Route this synthetic owned-transport fixture locally even when HTTPS_PROXY is set. + pool := x509.NewCertPool() + pool.AddCert(server.Certificate()) + transport.TLSClientConfig = &tls.Config{RootCAs: pool, ServerName: server.Certificate().DNSNames[0], MinVersion: tls.VersionTLS12} + transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, server.Listener.Addr().String()) + } + t.Cleanup(transport.CloseIdleConnections) + return a, server +} +func TestOpenAIOwnedClientStopsRedirectsAndHasNoReplayBody(t *testing.T) { + for _, status := range []int{301, 302, 303, 307, 308} { + var calls atomic.Int32 + a, _ := ownedCostTestAdapter(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + w.Header().Set("Location", "https://api.openai.com/again") + w.WriteHeader(status) + })) + _, response, err := a.execute(context.Background(), Request{Model: "requested", Prompt: "synthetic"}) + if err == nil || calls.Load() != 1 { + t.Fatalf("redirect %d calls=%d err=%v", status, calls.Load(), err) + } + r := oneCostAttempt(t, response) + if r.TransportMode != "single_send_v1" || r.EndpointKind != "official_global" { + t.Fatal("owned transport classification missing") + } + } + a, _ := ownedCostTestAdapter(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + _, _ = io.WriteString(w, costWire) + })) + original := a.httpClient + // Inspect the outgoing GetBody through a test-only opaque wrapper. The + // wrapper is intentionally NOT eligible as a single-send producer. + a.httpClient = &http.Client{Transport: costRoundTrip(func(r *http.Request) (*http.Response, error) { + if r.GetBody != nil { + t.Error("replayable body") + } + return original.Transport.RoundTrip(r) + })} + _, response, err := a.execute(context.Background(), Request{Model: "requested"}) + if err != nil { + t.Fatal(err) + } + if oneCostAttempt(t, response).TransportMode != "opaque" { + t.Fatal("test wrapper inherited owned trust") + } +} + +func TestOpenAICostEnvelopeUnknownFieldsAndUnsafeValues(t *testing.T) { + for _, field := range []string{"id", "model", "service_tier", "status"} { + var body map[string]json.RawMessage + if json.Unmarshal([]byte(costWire), &body) != nil { + t.Fatal("fixture") + } + delete(body, field) + body["unknown_future"] = json.RawMessage(`{"any":[1,2,3]}`) + encoded, _ := json.Marshal(body) + a := costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(string(encoded)))) + _, response, _ := a.execute(context.Background(), Request{Model: "requested"}) + r := oneCostAttempt(t, response) + if len(r.Issues) == 0 || r.Usage == nil { + t.Fatal("unknown fields discarded known usage or missing identity inferred") + } + } + for _, unsafe := range []string{strings.Repeat("x", 257), "model\nsecret", "model\x00secret"} { + encoded, _ := json.Marshal(unsafe) + body := strings.Replace(costWire, `"observed-model"`, string(encoded), 1) + a := costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(body))) + _, response, _ := a.execute(context.Background(), Request{Model: "requested"}) + if oneCostAttempt(t, response).ObservedModel != nil { + t.Fatal("unsafe identity retained/truncated") + } + } + if officialResponsesEndpoint("https://api.openai.com/v1/responses?secret=x") || officialResponsesEndpoint("https://gateway.example/v1/responses") || officialResponsesEndpoint("http://api.openai.com/v1/responses") { + t.Fatal("unverified route qualified") + } +} + +func TestOpenAIRequestCostEachMissingOrNullCounterRemainsUnknown(t *testing.T) { + for _, field := range []string{"input_tokens", "output_tokens", "cached_tokens", "cache_write_tokens"} { + for _, missing := range []bool{false, true} { + var body map[string]any + if err := json.Unmarshal([]byte(costWire), &body); err != nil { + t.Fatal(err) + } + usage := body["usage"].(map[string]any) + target := usage + if field == "cached_tokens" || field == "cache_write_tokens" { + target = usage["input_tokens_details"].(map[string]any) + } + if missing { + delete(target, field) + } else { + target[field] = nil + } + encoded, _ := json.Marshal(body) + a := costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(string(encoded)))) + _, response, _ := a.execute(context.Background(), Request{Model: "request"}) + receipt := oneCostAttempt(t, response) + if receipt.Usage == nil || !slices.Contains(receipt.Issues, llm.CostGapUsage) { + t.Fatalf("%s absence manufactured zero", field) + } + } + } +} +func TestOpenAIRequestCostConstructionAndUnknownDeliveredTier(t *testing.T) { + a := costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(costWire))) + _, response, err := a.execute(nil, Request{Model: "request"}) //nolint:staticcheck // SA1012: nil deliberately tests request-construction failure before dispatch. + if err == nil || oneCostAttempt(t, response).Dispatch != "not_dispatched" { + t.Fatal("construction failure dispatched") + } + body := strings.Replace(costWire, `"service_tier":"priority"`, `"service_tier":"future-tier"`, 1) + a = costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(body))) + _, response, err = a.execute(context.Background(), Request{Model: "request"}) + if err != nil { + t.Fatal(err) + } + receipt := oneCostAttempt(t, response) + if receipt.ObservedServiceTier == nil || *receipt.ObservedServiceTier != "future-tier" || !slices.Contains(receipt.Issues, llm.CostGapIdentity) { + t.Fatal("unknown delivered tier inferred or discarded") + } +} +func TestOpenAIOwnedExplicitRetryProducesIndependentReceipts(t *testing.T) { + var calls atomic.Int32 + a, _ := ownedCostTestAdapter(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if calls.Add(1) == 1 { + w.WriteHeader(429) + _, _ = io.WriteString(w, `{"error":{"message":"synthetic transient"}}`) + return + } + _, _ = io.WriteString(w, costWire) + })) + result, err := llm.RunStructuredWithSessionResume(context.Background(), a, "", Request{Model: "requested"}, func(b []byte) (bool, error) { + var payload struct { + OK bool `json:"ok"` + } + err := json.Unmarshal(b, &payload) + return payload.OK, err + }) + if err != nil || !result.Value || calls.Load() != 2 || len(result.Response.RequestCostEvidence.Attempts) != 2 { + t.Fatalf("explicit retry err=%v history=%#v", err, result.Response.RequestCostEvidence) + } + if !slices.Contains(result.Response.RequestCostEvidence.Attempts[0].Issues, llm.CostGapDispatch) { + t.Fatal("HTTP error erased by later success") + } +} + +func TestOpenAIRequestCostUnicodeIdentityIsNeverReplaced(t *testing.T) { + for _, raw := range []string{`"\ud800"`, `"\udc00"`, `"\ud800\u0041"`} { + body := strings.Replace(costWire, `"observed-model"`, raw, 1) + a := costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(body))) + _, response, _ := a.execute(context.Background(), Request{Model: "request"}) + receipt := oneCostAttempt(t, response) + if receipt.ObservedModel != nil || receipt.Usage == nil || !slices.Contains(receipt.Issues, llm.CostGapIdentity) { + t.Fatal("surrogate changed identity or erased unrelated usage") + } + } + body := strings.Replace(costWire, `"observed-model"`, "\"\xff\"", 1) + a := costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(body))) + _, response, _ := a.execute(context.Background(), Request{Model: "request"}) + receipt := oneCostAttempt(t, response) + if receipt.BodyState != "malformed" || receipt.ObservedModel != nil || receipt.Usage != nil { + t.Fatal("invalid UTF-8 whole document prefix parsed") + } +} diff --git a/internal/llmadapters/subprocess.go b/internal/llmadapters/subprocess.go index 35bc12cf..ebd74aba 100644 --- a/internal/llmadapters/subprocess.go +++ b/internal/llmadapters/subprocess.go @@ -12,6 +12,7 @@ import ( "os/exec" "path/filepath" "regexp" + "slices" "strconv" "strings" "sync" @@ -345,6 +346,7 @@ func buildClaudeForegroundArgs(req Request, scratch string, resumeSessionID stri if workspace := req.ReviewerWorkspace; workspace != nil { args = append(args, "--add-dir", workspace.RepoDir) } + args = appendClaudeNoNetworkArgs(args, req) if resumeSessionID != "" { args = append(args, "--resume", resumeSessionID) } @@ -360,6 +362,33 @@ func buildClaudeForegroundArgs(req Request, scratch string, resumeSessionID stri return append(args, "--", claudeBGPositionalPrompt(scratch)) } +// claudeNoNetworkDisallowedTools denies the ways a Claude reviewer could look +// up live PR state from its workspace: web tools, the GitHub CLI, HTTP +// clients, and git commands that talk to a remote. Bash can still reach the +// network by other means, so this narrows the tool surface; it is not a sandbox. +var claudeNoNetworkDisallowedTools = strings.Join([]string{ + "WebFetch", + "WebSearch", + "Bash(gh *)", + "Bash(curl *)", + "Bash(wget *)", + "Bash(git fetch *)", + "Bash(git pull *)", + "Bash(git ls-remote *)", + "Bash(git clone *)", + "Bash(git remote *)", + "Bash(git -C * fetch *)", + "Bash(git -C * pull *)", + "Bash(git -C * ls-remote *)", +}, ",") + +func appendClaudeNoNetworkArgs(args []string, req Request) []string { + if req.ReviewerWorkspace == nil || !req.ReviewerWorkspace.NoNetwork { + return args + } + return append(args, "--disallowedTools", claudeNoNetworkDisallowedTools) +} + // startClaude picks the transport for a Claude task. Background mode is the // default and foreground is the documented-sturdier fallback, so a background // job that fails without producing a result is retried once in foreground @@ -774,6 +803,7 @@ func (a *SubprocessAdapter) buildArgsForSession(req Request, scratch string, res if workspace := req.ReviewerWorkspace; workspace != nil { args = append(args, "--add-dir", workspace.RepoDir) } + args = appendClaudeNoNetworkArgs(args, req) if resumeSessionID != "" { args = append(args, "--resume", resumeSessionID) } @@ -803,6 +833,7 @@ func (a *SubprocessAdapter) buildArgsForSession(req Request, scratch string, res if req.Fast { args = append(args, "-c", `service_tier="fast"`) } + args = appendCodexNoNetworkArgs(args, req) if resumeSessionID != "" { args = subprocessCodexResumeArgs() if req.Model != "" { @@ -814,6 +845,7 @@ func (a *SubprocessAdapter) buildArgsForSession(req Request, scratch string, res if req.Fast { args = append(args, "-c", `service_tier="fast"`) } + args = appendCodexNoNetworkArgs(args, req) args = append(args, resumeSessionID) } return append(args, "--", req.Prompt), nil @@ -822,6 +854,24 @@ func (a *SubprocessAdapter) buildArgsForSession(req Request, scratch string, res } } +// codexNoNetworkConfig turns off Codex web search (on by default, in cached +// mode) and pins the workspace-write sandbox's network access off, so a +// reviewer cannot look up live PR state. +var codexNoNetworkConfig = []string{ + `web_search="disabled"`, + "sandbox_workspace_write.network_access=false", +} + +func appendCodexNoNetworkArgs(args []string, req Request) []string { + if req.ReviewerWorkspace == nil || !req.ReviewerWorkspace.NoNetwork { + return args + } + for _, value := range codexNoNetworkConfig { + args = append(args, "-c", value) + } + return args +} + func subprocessCodexBaseArgs(prefix []string, sandbox, cwd string, durable bool) []string { args := append([]string(nil), prefix...) args = append(args, "--json") @@ -874,9 +924,13 @@ func (a *SubprocessAdapter) validateArgs(args []string, scratch string, req Requ "--model": true, "--effort": true, "--settings": true, + "--disallowedTools": true, }); err != nil { return err } + if err := validateClaudeNoNetworkArgs(checkedArgs, req); err != nil { + return err + } // Exactly one transport: the detached job service (--bg) or headless // foreground print mode (-p, see claudeForegroundEnabled). if containsFlag(checkedArgs, "--bg") == containsFlag(checkedArgs, "-p") { @@ -950,6 +1004,28 @@ func (a *SubprocessAdapter) validateArgs(args []string, scratch string, req Requ return fmt.Errorf("%w: missing %s", ErrUnsafeSubprocessConfig, flag) } } + if workspace != nil && workspace.NoNetwork { + configs := flagValues(checkedArgs, "-c") + for _, want := range codexNoNetworkConfig { + if !slices.Contains(configs, want) { + return fmt.Errorf("%w: codex_cli offline reviewer must pass -c %s", ErrUnsafeSubprocessConfig, want) + } + } + } + } + return nil +} + +func validateClaudeNoNetworkArgs(args []string, req Request) error { + got, ok := flagValueOK(args, "--disallowedTools") + if req.ReviewerWorkspace != nil && req.ReviewerWorkspace.NoNetwork { + if !ok || got != claudeNoNetworkDisallowedTools { + return fmt.Errorf("%w: claude_cli offline reviewer must deny network tools", ErrUnsafeSubprocessConfig) + } + return nil + } + if ok { + return fmt.Errorf("%w: claude_cli must not pass --disallowedTools without an offline reviewer workspace", ErrUnsafeSubprocessConfig) } return nil } @@ -1107,7 +1183,12 @@ func (s *subprocessStream) runClaudeBG(ctx context.Context, adapter *SubprocessA } s.CloseProcessGroup() + s.finishClaudeBG(ctx, adapter, jobID, scratch, workDir, start, result) +} +// finishClaudeBG owns result polling and cleanup after the launcher has exited. +// The caller keeps the launch and this phase under the same task context. +func (s *subprocessStream) finishClaudeBG(ctx context.Context, adapter *SubprocessAdapter, jobID string, scratch string, workDir string, start time.Time, result subprocessResult) { if result.err == nil { var sessionID string result.response, sessionID, result.err = adapter.waitForClaudeBGResult(ctx, jobID, scratch) @@ -1116,7 +1197,6 @@ func (s *subprocessStream) runClaudeBG(ctx context.Context, adapter *SubprocessA } recordRequestDuration(&result.response, start, result.err) } - s.runCleanup() if jobID != "" { if err := adapter.cleanupClaudeBGJob(context.WithoutCancel(ctx), jobID, result.err, workDir); err != nil { s.writeCleanupWarning(err) diff --git a/internal/llmadapters/subprocess_cleanup_order_test.go b/internal/llmadapters/subprocess_cleanup_order_test.go new file mode 100644 index 00000000..caf9b568 --- /dev/null +++ b/internal/llmadapters/subprocess_cleanup_order_test.go @@ -0,0 +1,151 @@ +package llmadapters + +import ( + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// This covers adapter teardown ordering only. It does not establish process +// ownership or claim that the background service cleans up detached children. +func TestSubprocessClaudeJobCleanupPrecedesScratchRemoval(t *testing.T) { + for _, tt := range []struct { + name string + mode string + extraEnv []string + wantErr string + wantStop bool + wantWarning string + wantJobDir bool + }{ + {name: "success", mode: "success"}, + { + name: "task failure", + mode: "bg-failed", + wantErr: ErrClaudeBGTransport.Error() + ": job failed: model failed", + wantStop: true, + }, + { + name: "stop failure preserves task error", + mode: "bg-stop-fails", + wantErr: ErrClaudeBGTransport.Error() + ": job blocked: stop will fail", + wantStop: true, + wantWarning: "warning: llm subprocess: Claude bg cleanup stop failed: exit status 44: stop refused", + }, + { + name: "remove failure preserves success", + mode: "success", + extraEnv: []string{"LLM_HELPER_CLAUDE_FAIL_RM_IDS=job-1"}, + wantWarning: "warning: llm subprocess: Claude bg cleanup rm failed: exit status 45: rm refused", + wantJobDir: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + tempDir := t.TempDir() + root, err := os.OpenRoot(tempDir) + if err != nil { + t.Fatalf("OpenRoot: %v", err) + } + t.Cleanup(func() { + if err := root.Close(); err != nil { + t.Errorf("Close root: %v", err) + } + }) + recordPath := filepath.Join(tempDir, "records.jsonl") + logPath := filepath.Join(tempDir, "events.log") + configDir := filepath.Join(tempDir, "claude") + scratch := filepath.Join(tempDir, "scratch") + if err := root.Mkdir("scratch", 0o700); err != nil { + t.Fatal(err) + } + var cleanupRecords []byte + var cleanupRecordErr error + var cleanupLog []byte + var cleanupLogErr error + var cleanupJobInfo os.FileInfo + var cleanupJobErr error + var removalErr error + cleanupCalls := 0 + adapter := NewClaudeCLIAdapter(SubprocessOptions{ + Command: os.Args[0], + commandArgsPrefix: helperPrefix(), + Env: append(helperClaudeEnv(tt.mode, recordPath, configDir), tt.extraEnv...), + Timeout: 5 * time.Second, + ScratchDirFactory: func() (string, func() error, error) { + return scratch, func() error { + cleanupCalls++ + // Records identify invocations; log and file snapshots assert + // their visible effects at cleanup. Synchronous command-return + // ordering is a separate source contract. Read these snapshots + // only after Wait synchronizes with stream completion. + cleanupRecords, cleanupRecordErr = root.ReadFile("records.jsonl") + cleanupLog, cleanupLogErr = root.ReadFile("events.log") + cleanupJobInfo, cleanupJobErr = root.Stat(filepath.Join("claude", "jobs", "job-1")) + removalErr = root.RemoveAll("scratch") + return removalErr + }, nil + }, + }) + // Exercise one background invocation, independently of the existing + // fallback tests and the caller's foreground environment preference. + stream, err := adapter.startClaudeBG(context.Background(), Request{Prompt: "prompt", LogPath: logPath}, "") + if err != nil { + t.Fatalf("startClaudeBG: %v", err) + } + response, err := stream.Wait(context.Background()) + if tt.wantErr != "" { + if err == nil || err.Error() != tt.wantErr { + t.Fatalf("Wait error = %v, want exactly %q", err, tt.wantErr) + } + if !errors.Is(err, ErrClaudeBGTransport) { + t.Fatalf("Wait error = %v, want ErrClaudeBGTransport", err) + } + } else if err != nil || string(response.StructuredOutput) != `{"ok":true}` { + t.Fatalf("Wait = (%s, %v), want success", response.StructuredOutput, err) + } + if cleanupCalls != 1 || cleanupRecordErr != nil || removalErr != nil { + t.Fatalf("cleanup calls=%d record error=%v removal error=%v", cleanupCalls, cleanupRecordErr, removalErr) + } + if cleanupLogErr != nil { + t.Fatalf("read log at scratch removal: %v", cleanupLogErr) + } + var warnings []string + for _, line := range strings.Split(string(cleanupLog), "\n") { + if strings.HasPrefix(line, "warning: ") { + warnings = append(warnings, line) + } + } + if got := strings.Join(warnings, "\n"); got != tt.wantWarning { + t.Fatalf("warnings at scratch removal = %q, want %q", got, tt.wantWarning) + } + if tt.wantJobDir { + if cleanupJobErr != nil || cleanupJobInfo == nil || !cleanupJobInfo.IsDir() { + t.Fatalf("refused rm did not preserve job directory at scratch removal: %v", cleanupJobErr) + } + } else if !errors.Is(cleanupJobErr, os.ErrNotExist) { + t.Fatalf("successful rm left job directory at scratch removal: %v", cleanupJobErr) + } + var records []helperRecord + decoder := json.NewDecoder(strings.NewReader(string(cleanupRecords))) + for { + var record helperRecord + if err := decoder.Decode(&record); errors.Is(err, io.EOF) { + break + } else if err != nil { + t.Fatalf("decode records at scratch removal: %v", err) + } + records = append(records, record) + } + assertClaudeCleanup(t, records, "job-1", tt.wantStop, configDir) + if _, err := root.Stat("scratch"); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("scratch remains after Wait: %v", err) + } + }) + } +} diff --git a/internal/llmadapters/subprocess_test.go b/internal/llmadapters/subprocess_test.go index 760a4bd2..aa683173 100644 --- a/internal/llmadapters/subprocess_test.go +++ b/internal/llmadapters/subprocess_test.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "strconv" "strings" "testing" @@ -353,9 +354,7 @@ func TestSubprocessClaudeBackgroundStatesAndCleanup(t *testing.T) { wantOutput string wantSession string wantErr string - wantErrIs error wantStop bool - timeout time.Duration wantRawResult bool // wantRetry marks the states startClaude treats as transport // failures. Asserting the launch count is what keeps this table @@ -372,29 +371,20 @@ func TestSubprocessClaudeBackgroundStatesAndCleanup(t *testing.T) { {name: "stop fails still removes", mode: "bg-stop-fails", wantErr: "blocked: stop will fail", wantStop: true, wantRetry: true}, {name: "missing result", mode: "bg-missing-result", wantErr: "background job: no result file", wantSession: "session-missing", wantStop: true}, {name: "empty result", mode: "bg-empty-result", wantErr: "background job: result file is empty", wantSession: "session-empty", wantStop: true}, - {name: "timeout", mode: "bg-running", wantErrIs: context.DeadlineExceeded, wantStop: true, timeout: 50 * time.Millisecond}, } { t.Run(tt.name, func(t *testing.T) { tempDir := t.TempDir() recordPath := filepath.Join(tempDir, "records.jsonl") configDir := filepath.Join(tempDir, "claude") - timeout := tt.timeout - if timeout == 0 { - timeout = 5 * time.Second - } - adapter := newClaudeHelperAdapter(tt.mode, recordPath, configDir, timeout) + adapter := newClaudeHelperAdapter(tt.mode, recordPath, configDir, 5*time.Second) stream, err := adapter.Start(context.Background(), Request{Prompt: "prompt"}) if err != nil { t.Fatalf("Start: %v", err) } response, err := stream.Wait(context.Background()) - if tt.wantErr != "" || tt.wantErrIs != nil { - if tt.wantErrIs != nil { - if !errors.Is(err, tt.wantErrIs) { - t.Fatalf("Wait error = %v, want errors.Is %v", err, tt.wantErrIs) - } - } else if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { t.Fatalf("Wait error = %v, want containing %q", err, tt.wantErr) } if tt.wantSession != "" && stream.SessionID() != tt.wantSession { @@ -423,6 +413,78 @@ func TestSubprocessClaudeBackgroundStatesAndCleanup(t *testing.T) { assertClaudeCleanup(t, readHelperRecords(t, recordPath), "job-1", false, configDir) }) } + t.Run("timeout", testSubprocessClaudeBackgroundTimeoutCleanup) +} + +func testSubprocessClaudeBackgroundTimeoutCleanup(t *testing.T) { + for _, startupDelay := range []time.Duration{0, 100 * time.Millisecond} { + t.Run("startup="+startupDelay.String(), func(t *testing.T) { + tempDir := t.TempDir() + recordPath := filepath.Join(tempDir, "records.jsonl") + configDir := filepath.Join(tempDir, "claude") + adapter := newClaudeHelperAdapterWithEnv("bg-running", recordPath, configDir, 5*time.Second, + "LLM_HELPER_CLAUDE_LAUNCH_DELAY="+startupDelay.String()) + req := Request{Prompt: "prompt"} + scratch, cleanup, err := adapter.invocationScratchDir(req) + if err != nil { + t.Fatalf("invocationScratchDir: %v", err) + } + t.Cleanup(func() { _ = cleanup() }) + if err := writeClaudeBGPromptFile(req.Prompt, scratch, req.ReviewerWorkspace); err != nil { + t.Fatalf("writeClaudeBGPromptFile: %v", err) + } + args, err := adapter.buildArgs(req, scratch) + if err != nil { + t.Fatalf("buildArgs: %v", err) + } + if err := adapter.validateArgs(args, scratch, req); err != nil { + t.Fatalf("validateArgs: %v", err) + } + workDir, err := claudeBGWorkingDir(adapter.env) + if err != nil { + t.Fatalf("claudeBGWorkingDir: %v", err) + } + + // Startup is not the behavior under test. Await the real helper's + // exit and job ID before starting the short polling deadline. The + // delayed variant exceeds that deadline before writing its record. + startupCtx, startupCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer startupCancel() + launchStdout, launchStderr, err := adapter.runClaudeBGCommand(startupCtx, workDir, args...) + if err != nil { + t.Fatalf("launch helper: %v; stderr: %s", err, launchStderr) + } + jobID := extractClaudeBGJobID(string(launchStdout)) + if jobID != "job-1" { + t.Fatalf("job ID = %q, want job-1; stdout: %s", jobID, launchStdout) + } + assertClaudeLaunchCount(t, readHelperRecords(t, recordPath), false) + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + primary := &subprocessStream{baseStream: llm.NewBaseStream(cancel)} + stream := &claudeFallbackStream{ + adapter: adapter, req: req, primary: primary, + // Keep retry budget available so an erroneous transport error + // would launch a foreground helper and fail the launch count. + deadline: taskDeadline(context.Background(), adapter.timeout), + } + go primary.finishClaudeBG(ctx, adapter, jobID, scratch, workDir, time.Now(), subprocessResult{}) + _, err = stream.Wait(context.Background()) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Wait error = %v, want context.DeadlineExceeded", err) + } + if stream.SessionID() != "session-running" { + t.Fatalf("SessionID = %q, want session-running", stream.SessionID()) + } + records := readHelperRecords(t, recordPath) + assertClaudeCleanup(t, records, jobID, true, configDir) + assertClaudeLaunchCount(t, records, false) + if _, err := os.Stat(filepath.Join(configDir, "jobs", jobID)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("job directory exists after cleanup: stat err = %v", err) + } + }) + } } func TestSubprocessClaudeStaleJobGC(t *testing.T) { @@ -1434,10 +1496,20 @@ func TestSubprocessHelperProcess(_ *testing.T) { return } recordPath := os.Getenv("LLM_HELPER_RECORD") + args := adapterArgsFromHelper() + if containsFlag(args, "--bg") { + if delay := os.Getenv("LLM_HELPER_CLAUDE_LAUNCH_DELAY"); delay != "" { + duration, err := time.ParseDuration(delay) + if err != nil { + fmt.Fprintf(os.Stderr, "parse launch delay: %v\n", err) + os.Exit(48) + } + time.Sleep(duration) + } + } cwd, _ := os.Getwd() entries, _ := os.ReadDir(cwd) stdin, _ := io.ReadAll(os.Stdin) - args := adapterArgsFromHelper() record := helperRecord{ AdapterArgs: args, Cwd: cwd, @@ -2584,3 +2656,93 @@ func TestSubprocessClaudeBackgroundNothingLeftBehindIsRetried(t *testing.T) { // attempted. assertClaudeLaunchCount(t, readHelperRecords(t, recordPath), true) } + +func offlineTestRequest(t *testing.T, noNetwork bool) (Request, string) { + t.Helper() + tempDir := t.TempDir() + scratch := filepath.Join(tempDir, "scratch") + repoRoot := filepath.Join(tempDir, "repo") + for _, dir := range []string{scratch, repoRoot} { + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatalf("MkdirAll(%s): %v", dir, err) + } + } + return Request{ + Prompt: "prompt", + ReviewerWorkspace: &ReviewerWorkspaceRequest{ + RepoDir: repoRoot, + ScratchDir: scratch, + NoNetwork: noNetwork, + }, + }, scratch +} + +func TestSubprocessClaudeDeniesNetworkToolsOnlyForOfflineReviewer(t *testing.T) { + adapter := NewClaudeCLIAdapter(SubprocessOptions{}) + for _, noNetwork := range []bool{false, true} { + req, scratch := offlineTestRequest(t, noNetwork) + background, err := adapter.buildArgsForSession(req, scratch, "") + if err != nil { + t.Fatalf("buildArgsForSession: %v", err) + } + resumed, err := adapter.buildArgsForSession(req, scratch, "session-1") + if err != nil { + t.Fatalf("buildArgsForSession(resume): %v", err) + } + foreground := buildClaudeForegroundArgs(req, scratch, "") + for name, args := range map[string][]string{"background": background, "resumed": resumed, "foreground": foreground} { + got, ok := flagValueOK(argsBeforePrompt(args), "--disallowedTools") + if noNetwork { + if !ok { + t.Fatalf("%s offline args = %v, want --disallowedTools", name, args) + } + for _, rule := range []string{"WebFetch", "WebSearch", "Bash(gh *)", "Bash(curl *)", "Bash(wget *)", "Bash(git fetch *)", "Bash(git pull *)", "Bash(git ls-remote *)"} { + if !slices.Contains(strings.Split(got, ","), rule) { + t.Fatalf("%s disallowed tools = %q, missing %q", name, got, rule) + } + } + } else if ok { + t.Fatalf("%s args = %v, want no --disallowedTools without the offline flag", name, args) + } + if err := adapter.validateArgs(args, scratch, req); err != nil { + t.Fatalf("%s validateArgs(noNetwork=%v): %v", name, noNetwork, err) + } + } + if noNetwork { + if err := adapter.validateArgs(removeFlagPair(background, "--disallowedTools"), scratch, req); !errors.Is(err, ErrUnsafeSubprocessConfig) { + t.Fatalf("validateArgs without deny rules = %v, want unsafe config", err) + } + } + } +} + +func TestSubprocessCodexDisablesNetworkOnlyForOfflineReviewer(t *testing.T) { + adapter := NewCodexCLIAdapter(SubprocessOptions{AllowBestEffortNoTools: true}) + for _, noNetwork := range []bool{false, true} { + req, scratch := offlineTestRequest(t, noNetwork) + started, err := adapter.buildArgsForSession(req, scratch, "") + if err != nil { + t.Fatalf("buildArgsForSession: %v", err) + } + resumed, err := adapter.buildArgsForSession(req, scratch, "session-1") + if err != nil { + t.Fatalf("buildArgsForSession(resume): %v", err) + } + for name, args := range map[string][]string{"started": started, "resumed": resumed} { + configs := flagValues(argsBeforePrompt(args), "-c") + for _, want := range []string{`web_search="disabled"`, "sandbox_workspace_write.network_access=false"} { + if slices.Contains(configs, want) != noNetwork { + t.Fatalf("%s -c values = %v, want %q present=%v", name, configs, want, noNetwork) + } + } + if err := adapter.validateArgs(args, scratch, req); err != nil { + t.Fatalf("%s validateArgs(noNetwork=%v): %v", name, noNetwork, err) + } + } + if noNetwork { + if err := adapter.validateArgs(removeFlagPair(started, "-c"), scratch, req); !errors.Is(err, ErrUnsafeSubprocessConfig) { + t.Fatalf("validateArgs without network config = %v, want unsafe config", err) + } + } + } +} diff --git a/internal/llmlifecycle/lifecycle.go b/internal/llmlifecycle/lifecycle.go index 60cb4159..54e0e68c 100644 --- a/internal/llmlifecycle/lifecycle.go +++ b/internal/llmlifecycle/lifecycle.go @@ -23,7 +23,7 @@ import ( ) // SchemaVersion identifies the on-disk LLM task artifact schema. -const SchemaVersion = 3 +const SchemaVersion = 4 // Status is the durable outcome recorded for one structured LLM task. type Status string @@ -202,31 +202,34 @@ func (d SessionDraft) ToLedger(runID string) ledger.Session { // Metadata is the durable descriptor for one structured LLM task. type Metadata struct { - SchemaVersion int `json:"schema_version"` - TaskID string `json:"task_id"` - Phase string `json:"phase"` - DependencyTaskIDs []string `json:"dependency_task_ids,omitempty"` - InputFingerprint string `json:"input_fingerprint"` - AgentID string `json:"agent_id,omitempty"` - Status Status `json:"status"` - SessionRowID string `json:"session_row_id,omitempty"` - ProviderSessionID string `json:"provider_session_id,omitempty"` - Adapter string `json:"adapter"` - Model string `json:"model"` - Effort string `json:"effort,omitempty"` - LogPath string `json:"log_path,omitempty"` - ValidatedOutputPath string `json:"validated_output_path,omitempty"` - Error string `json:"error,omitempty"` - TokensIn *int `json:"tokens_in,omitempty"` - TokensOut *int `json:"tokens_out,omitempty"` - CacheRead *int `json:"cache_read,omitempty"` - CacheCreate *int `json:"cache_create,omitempty"` - CacheCreate5m *int `json:"cache_create_5m,omitempty"` - CacheCreate1h *int `json:"cache_create_1h,omitempty"` - CostUSD *float64 `json:"cost_usd,omitempty"` - Speed string `json:"speed,omitempty"` - ReviewerToolEvidence *llm.ReviewerToolEvidence `json:"reviewer_tool_evidence,omitempty"` - Attempts []AttemptMetadata `json:"attempts,omitempty"` + requestCostScopeValid bool + SchemaVersion int `json:"schema_version"` + TaskID string `json:"task_id"` + Phase string `json:"phase"` + DependencyTaskIDs []string `json:"dependency_task_ids,omitempty"` + InputFingerprint string `json:"input_fingerprint"` + AgentID string `json:"agent_id,omitempty"` + Status Status `json:"status"` + SessionRowID string `json:"session_row_id,omitempty"` + ProviderSessionID string `json:"provider_session_id,omitempty"` + Adapter string `json:"adapter"` + Model string `json:"model"` + Effort string `json:"effort,omitempty"` + LogPath string `json:"log_path,omitempty"` + ValidatedOutputPath string `json:"validated_output_path,omitempty"` + Error string `json:"error,omitempty"` + TokensIn *int `json:"tokens_in,omitempty"` + TokensOut *int `json:"tokens_out,omitempty"` + CacheRead *int `json:"cache_read,omitempty"` + CacheCreate *int `json:"cache_create,omitempty"` + CacheCreate5m *int `json:"cache_create_5m,omitempty"` + CacheCreate1h *int `json:"cache_create_1h,omitempty"` + CostUSD *float64 `json:"cost_usd,omitempty"` + Speed string `json:"speed,omitempty"` + ReviewerToolEvidence *llm.ReviewerToolEvidence `json:"reviewer_tool_evidence,omitempty"` + RequestCostEvidence *llm.RequestCostEvidence `json:"request_cost_evidence,omitempty"` + RequestCostCheckpoint *RequestCostCheckpointBinding `json:"request_cost_checkpoint,omitempty"` + Attempts []AttemptMetadata `json:"attempts,omitempty"` } // AttemptMetadata records one invalid structured-output attempt. @@ -268,7 +271,23 @@ func (e *TaskError) Status() Status { } // RunStructured executes or loads one durable structured LLM task. -func RunStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T]) (Result[T], error) { +func RunStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T]) (result Result[T], resultErr error) { + if err := validateRequest(req, decode); err != nil { + return result, err + } + lock, err := taskCostLock(req.Paths, req.TaskID) + if err != nil { + return result, err + } + defer func() { + if releaseErr := lock.Release(); releaseErr != nil { + resultErr = errors.Join(resultErr, releaseErr) + } + }() + return runStructuredLocked(ctx, req, decode) +} + +func runStructuredLocked[T any](ctx context.Context, req Request, decode llm.Decoder[T]) (Result[T], error) { var zero Result[T] if err := validateRequest(req, decode); err != nil { return zero, err @@ -276,7 +295,7 @@ func RunStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T if strings.TrimSpace(req.RunID) == "" && !req.AllowNoRunCache { return zero, fmt.Errorf("llmlifecycle: structured task %q requires a persisted run", req.TaskID) } - if loaded, ok, err := LoadStructured(ctx, req, decode); err != nil || ok { + if loaded, ok, err := loadStructuredLocked(ctx, req, decode); err != nil || ok { return loaded, err } @@ -288,6 +307,15 @@ func RunStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T resumeSessionID = taskSessionID } } + rowID := req.NewSessionRowID() + var checkpoint *costCheckpoint + if llm.RequestCostSource(req.Adapter) != "" { + var err error + checkpoint, err = beginCostGeneration(req, rowID) + if err != nil { + return zero, err + } + } progressEvent := NewProgressEvent(req, resumeSessionID) progressSpan := startProgress(req.Progress, progressEvent) @@ -302,7 +330,7 @@ func RunStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T structured, runErr := llm.RunStructuredWithSessionResume(ctx, req.Adapter, resumeSessionID, request, decode) completed := now() draft := SessionDraft{ - RowID: req.NewSessionRowID(), + RowID: rowID, ProviderReportedSessionID: structured.SessionID, ProviderSessionID: structured.SessionID, Role: req.Role, @@ -325,9 +353,21 @@ func RunStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T role = ledger.SessionRoleOrchestrator } draft.Role = role + if checkpoint != nil { + finalErr := finishCostGeneration(req.Paths, checkpoint, structured.Response.RequestCostEvidence) + if finalErr != nil { + costGap(checkpoint, llm.CostGapCheckpoint) + } + draft.Response.RequestCostEvidence = llm.CloneRequestCostEvidence(checkpoint.Evidence) + if finalErr != nil { + endProgress(progressSpan, finalErr, ProgressResult{Usage: draft.Response.Usage}) + return Result[T]{Draft: draft}, errors.Join(runErr, fmt.Errorf("llmlifecycle: finalize request-cost checkpoint: %w", finalErr)) + } + } hasRun := strings.TrimSpace(req.RunID) != "" meta := BaseMetadata(req, draft) + meta.RequestCostCheckpoint = checkpointBinding(checkpoint) if !hasRun { meta.SessionRowID = "" } @@ -336,14 +376,14 @@ func RunStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T if req.Store == nil { err := fmt.Errorf("llmlifecycle: store is required") endProgress(progressSpan, err, progressResult(meta, structured, false, structured.Response.Usage)) - return zero, err + return Result[T]{Draft: draft, Session: session}, err } session = draft.ToLedger(req.RunID) if err := req.Store.InsertSession(ctx, session); err != nil { meta.Status = StatusFailedBlocking meta.ProviderSessionID = draft.ProviderSessionID endProgress(progressSpan, err, progressResult(meta, structured, false, structured.Response.Usage)) - return zero, err + return Result[T]{Draft: draft, Session: session}, err } } @@ -353,7 +393,7 @@ func RunStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T meta.ProviderSessionID = draft.ProviderSessionID if err := WriteSuccess(req.Paths, &meta, structured.AcceptedOutput); err != nil { endProgress(progressSpan, err, progressResult(meta, structured, false, structured.Response.Usage)) - return zero, err + return Result[T]{Draft: draft, Session: session}, err } endProgress(progressSpan, nil, progressResult(meta, structured, false, structured.Response.Usage)) return Result[T]{Value: structured.Value, Draft: draft, Session: session}, nil @@ -372,7 +412,20 @@ func RunStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T } // LoadStructured loads a reusable task result without calling the provider. -func LoadStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T]) (Result[T], bool, error) { +func LoadStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T]) (result Result[T], loaded bool, resultErr error) { + lock, err := taskCostLock(req.Paths, req.TaskID) + if err != nil { + return result, false, err + } + defer func() { + if releaseErr := lock.Release(); releaseErr != nil { + resultErr = errors.Join(resultErr, releaseErr) + } + }() + return loadStructuredLocked(ctx, req, decode) +} + +func loadStructuredLocked[T any](ctx context.Context, req Request, decode llm.Decoder[T]) (Result[T], bool, error) { var zero Result[T] meta, ok, err := ReadMetadata(req.Paths, req.TaskID) if err != nil || !ok { @@ -381,6 +434,9 @@ func LoadStructured[T any](ctx context.Context, req Request, decode llm.Decoder[ if err := ValidateMetadata(meta, req, adapterName(req.Adapter)); err != nil { return zero, true, err } + if llm.RequestCostSource(req.Adapter) != "" || meta.RequestCostEvidence != nil || meta.Adapter == "openai_api" { + overlayCostMetadata(req.Paths, req.TaskID, req.RunID, &meta) + } switch meta.Status { case StatusSucceeded: outputPath, err := req.Paths.ValidatedOutput(req.TaskID) @@ -416,7 +472,8 @@ func LoadStructured[T any](ctx context.Context, req Request, decode llm.Decoder[ } draft := SessionDraftFromLedger(session) draft.Response.Usage.Speed = meta.Speed - draft.Response.ReviewerToolEvidence = meta.ReviewerToolEvidence + draft.Response.ReviewerToolEvidence = llm.CloneReviewerToolEvidence(meta.ReviewerToolEvidence) + draft.Response.RequestCostEvidence = llm.CloneRequestCostEvidence(meta.RequestCostEvidence) progress := progressResult(meta, llm.StructuredResult[T]{SessionID: meta.ProviderSessionID}, true, draft.Response.Usage) loadProgress(req.Progress, NewProgressEvent(req, ResumeSessionID(meta)), progress) return Result[T]{Value: value, Draft: draft, Session: session, Cached: true}, true, nil @@ -572,7 +629,7 @@ func ListMetadata(paths Paths) ([]Metadata, error) { return nil, fmt.Errorf("llmlifecycle: read task metadata %q: %w", entry.Name(), err) } var meta Metadata - if err := json.Unmarshal(data, &meta); err != nil { + if err := decodeMetadata(data, &meta); err != nil { return nil, fmt.Errorf("llmlifecycle: decode task metadata %q: %w", entry.Name(), err) } metadata = append(metadata, meta) @@ -593,14 +650,23 @@ func readMetadata(paths Paths, taskID, readError, decodeError string) (Metadata, return Metadata{}, false, fmt.Errorf(readError, taskID, err) } var meta Metadata - if err := json.Unmarshal(data, &meta); err != nil { + if err := decodeMetadata(data, &meta); err != nil { return Metadata{}, false, fmt.Errorf(decodeError, taskID, err) } return meta, true, nil } // ResetIfInputFingerprintChanged removes stale task artifacts when inputs change. -func ResetIfInputFingerprintChanged(paths Paths, taskID, fingerprint string) error { +func ResetIfInputFingerprintChanged(paths Paths, taskID, fingerprint string) (resultErr error) { + lock, err := taskCostLock(paths, taskID) + if err != nil { + return err + } + defer func() { + if releaseErr := lock.Release(); releaseErr != nil { + resultErr = errors.Join(resultErr, releaseErr) + } + }() meta, ok, err := readMetadata(paths, taskID, "pipeline: read LLM task %q metadata: %w", "pipeline: decode LLM task %q metadata: %w", @@ -611,6 +677,13 @@ func ResetIfInputFingerprintChanged(paths Paths, taskID, fingerprint string) err if strings.TrimSpace(meta.InputFingerprint) == strings.TrimSpace(fingerprint) { return nil } + checkpoint, err := reconcileCostCheckpoint(paths, taskID, "", &meta, true) + if err != nil { + return err + } + if err := writeCostCheckpoint(paths, checkpoint); err != nil { + return fmt.Errorf("llmlifecycle: preserve request-cost observations before reset: %w", err) + } taskDir, err := paths.TaskDir(taskID) if err != nil { return err @@ -664,6 +737,12 @@ func WriteMetadata(paths Paths, meta Metadata) error { return err } data, err := json.MarshalIndent(meta, "", " ") + if meta.RequestCostEvidence != nil { + // Admission and the bounded optional-envelope reader use compact JSON. + // Pretty-printing a near-cap cost prefix would inflate it past that + // bound and lose observations when the sidecar is unavailable. + data, err = json.Marshal(meta) + } if err != nil { return err } @@ -701,7 +780,8 @@ func BaseMetadata(req Request, draft SessionDraft) Metadata { CacheCreate1h: draft.Response.Usage.CacheCreate1h, CostUSD: draft.Response.Usage.CostUSD, Speed: draft.Response.Usage.Speed, - ReviewerToolEvidence: draft.Response.ReviewerToolEvidence, + ReviewerToolEvidence: llm.CloneReviewerToolEvidence(draft.Response.ReviewerToolEvidence), + RequestCostEvidence: llm.CloneRequestCostEvidence(draft.Response.RequestCostEvidence), } } @@ -831,7 +911,8 @@ func SessionDraftFromMetadata(meta Metadata) SessionDraft { Model: meta.Model, Effort: meta.Effort, Response: llm.Response{ - ReviewerToolEvidence: meta.ReviewerToolEvidence, + ReviewerToolEvidence: llm.CloneReviewerToolEvidence(meta.ReviewerToolEvidence), + RequestCostEvidence: llm.CloneRequestCostEvidence(meta.RequestCostEvidence), Usage: llm.Usage{ TokensIn: meta.TokensIn, TokensOut: meta.TokensOut, @@ -870,7 +951,8 @@ func loadOptionalTaskSession(ctx context.Context, store Store, runID string, met } draft := SessionDraftFromLedger(session) draft.Response.Usage.Speed = meta.Speed - draft.Response.ReviewerToolEvidence = meta.ReviewerToolEvidence + draft.Response.ReviewerToolEvidence = llm.CloneReviewerToolEvidence(meta.ReviewerToolEvidence) + draft.Response.RequestCostEvidence = llm.CloneRequestCostEvidence(meta.RequestCostEvidence) return session, draft, nil } diff --git a/internal/llmlifecycle/lifecycle_test.go b/internal/llmlifecycle/lifecycle_test.go index bb44917e..d3c13af4 100644 --- a/internal/llmlifecycle/lifecycle_test.go +++ b/internal/llmlifecycle/lifecycle_test.go @@ -34,7 +34,7 @@ func TestFingerprintIncludesCurrentTaskSchemaVersion(t *testing.T) { "review changed code", []string{"orchestrator-selection", "discussion=abc123"}, ) - const want = "119c25ca1cdbaffa98f09019140779be4802f63e96769568fac3d9aa35b4697e" + const want = "8c285dc068b786907f3539d508f4b9c6c81051aa5b9788753c306b9f0de5d839" if got != want { t.Fatalf("Fingerprint() = %q, want current schema-version cache key %q", got, want) } @@ -453,7 +453,7 @@ func TestRunStructuredRejectsPreviousSchemaBeforeProviderCall(t *testing.T) { staleAdapter := &llm.FakeAdapter{NameValue: "fake-llm"} req.Adapter = staleAdapter _, err = RunStructured(ctx, req, decodeLifecyclePayload) - if err == nil || !strings.Contains(err.Error(), "schema version = 2, want 3") { + if err == nil || !strings.Contains(err.Error(), "schema version = 3, want 4") { t.Fatalf("RunStructured previous schema error = %v, want fail-closed schema rejection", err) } if len(staleAdapter.Requests()) != 0 || len(staleAdapter.Resumes()) != 0 { diff --git a/internal/llmlifecycle/request_cost_evidence.go b/internal/llmlifecycle/request_cost_evidence.go new file mode 100644 index 00000000..baec6aae --- /dev/null +++ b/internal/llmlifecycle/request_cost_evidence.go @@ -0,0 +1,665 @@ +package llmlifecycle + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/open-cli-collective/codereview-cli/internal/fsatomic" + "github.com/open-cli-collective/codereview-cli/internal/llm" + "github.com/open-cli-collective/codereview-cli/internal/runlock" +) + +const maxCostGenerations = 64 + +// RequestCostCheckpointBinding binds optional telemetry, never output reuse. +type RequestCostCheckpointBinding struct { + Version int `json:"version"` + ArtifactScope string `json:"artifact_scope"` + RunID string `json:"run_id"` + TaskID string `json:"task_id"` + Sequence uint64 `json:"sequence"` + Digest string `json:"digest"` +} + +type costGeneration struct { + GenerationID string `json:"generation_id"` + InputFingerprint string `json:"input_fingerprint"` + EvidenceDigest string `json:"evidence_digest"` +} + +type pendingCostGeneration struct { + GenerationID string `json:"generation_id"` + InputFingerprint string `json:"input_fingerprint"` +} + +// Map-free fixed-order v1 digest projection; only self-digest is excluded. +type costCheckpointProjection struct { + Version int `json:"version"` + ArtifactScope string `json:"artifact_scope"` + RunID string `json:"run_id"` + TaskID string `json:"task_id"` + Sequence uint64 `json:"sequence"` + FreshOrigin bool `json:"fresh_origin"` + GenerationsObserved uint64 `json:"generations_observed"` + OmittedGenerations uint64 `json:"omitted_generations"` + Evidence *llm.RequestCostEvidence `json:"evidence"` + Generations []costGeneration `json:"generations"` + Pending *pendingCostGeneration `json:"pending"` +} + +type costCheckpoint struct { + costCheckpointProjection + Digest string `json:"digest"` +} + +// RequestCostCheckpoint lives outside the resettable task directory but inside +// the caller/run-owned artifact root covered by existing retention/purge. +func (p Paths) RequestCostCheckpoint(taskID string) (string, error) { + if _, err := p.TaskDir(taskID); err != nil { + return "", err + } + // Keep suffixes and the atomic writer's .tmp inside NAME_MAX even when + // the existing encoded task directory already uses that component budget. + identity := sha256.Sum256([]byte(strings.TrimSpace(taskID))) + return filepath.Join(p.LLMTasksDir, ".request-cost", hex.EncodeToString(identity[:])+".json"), nil +} + +func taskCostLock(paths Paths, taskID string) (*runlock.Lock, error) { + p, err := paths.RequestCostCheckpoint(taskID) + if err != nil { + return nil, err + } + lock, err := runlock.Acquire(strings.TrimSuffix(p, ".json") + ".lock") + if err != nil { + return nil, fmt.Errorf("llmlifecycle: acquire task ownership: %w", err) + } + return lock, nil +} + +func artifactCostScope(paths Paths) (string, error) { + p, err := filepath.Abs(paths.LLMTasksDir) + if err != nil { + return "", err + } + h := sha256.Sum256([]byte(filepath.Clean(p))) + return hex.EncodeToString(h[:]), nil +} + +func newCostCheckpoint(paths Paths, taskID, runID string) (*costCheckpoint, error) { + scope, err := artifactCostScope(paths) + if err != nil { + return nil, err + } + if !llm.SafeCostString(strings.TrimSpace(taskID), 256) || (runID != "" && !llm.SafeCostString(runID, 256)) { + return nil, errors.New("llmlifecycle: invalid request-cost scope identity") + } + e := llm.UnknownRequestCostEvidence(llm.CostGapLegacy) + e.Scope = "task_artifact_history" + e.ArtifactScope = scope + e.TaskID = strings.TrimSpace(taskID) + e.RunID = runID + return &costCheckpoint{costCheckpointProjection: costCheckpointProjection{Version: 1, ArtifactScope: scope, TaskID: e.TaskID, RunID: runID, Evidence: e, Generations: []costGeneration{}}}, nil +} + +func costCheckpointBytes(c *costCheckpoint) ([]byte, error) { + return json.Marshal(c.costCheckpointProjection) +} +func checkpointDigest(c *costCheckpoint) (string, error) { + b, err := costCheckpointBytes(c) + if err != nil { + return "", err + } + h := sha256.New() + _, _ = h.Write([]byte("cr-request-cost-checkpoint/v1\n")) + _, _ = h.Write(b) + return hex.EncodeToString(h.Sum(nil)), nil +} +func costGap(c *costCheckpoint, gap llm.CostEvidenceGap) { + c.Evidence.Gaps = llm.AddCostGap(c.Evidence.Gaps, gap) + c.Evidence.AttemptCountExact = false +} +func isCostDigest(s string) bool { + if len(s) != 64 { + return false + } + _, err := hex.DecodeString(s) + return err == nil +} + +func readCostCheckpoint(paths Paths, taskID, runID string, anyRun bool) (*costCheckpoint, llm.CostEvidenceGap) { + p, err := paths.RequestCostCheckpoint(taskID) + if err != nil { + return nil, llm.CostGapCheckpoint + } + f, err := os.Open(p) // #nosec G304 -- path is derived from the caller-owned root. + if err != nil { + return nil, llm.CostGapCheckpoint + } + defer f.Close() + b, err := io.ReadAll(io.LimitReader(f, llm.MaxRequestCostEvidenceBytes+1)) + if err != nil { + return nil, llm.CostGapCheckpoint + } + if len(b) > llm.MaxRequestCostEvidenceBytes { + return nil, llm.CostGapOverflow + } + var c costCheckpoint + if !llm.RequestCostJSONValid(b) || json.Unmarshal(b, &c) != nil || c.Version != 1 || c.Evidence == nil { + return nil, llm.CostGapCheckpoint + } + var rawScope struct { + RunID *string `json:"run_id"` + } + if json.Unmarshal(b, &rawScope) != nil || rawScope.RunID == nil || (*rawScope.RunID != "" && !llm.SafeCostString(*rawScope.RunID, 256)) { + return nil, llm.CostGapScope + } + scope, err := artifactCostScope(paths) + if err != nil || c.ArtifactScope != scope || c.TaskID != strings.TrimSpace(taskID) || (!anyRun && c.RunID != runID) { + return nil, llm.CostGapScope + } + if len(c.Generations) > maxCostGenerations || len(c.Evidence.Attempts) > llm.MaxRequestCostAttempts || len(c.Evidence.Gaps) > llm.MaxRequestCostGaps { + return nil, llm.CostGapOverflow + } + if c.FreshOrigin || c.Evidence.Version != 1 || c.Evidence.Source != "openai_responses" || c.Evidence.Scope != "task_artifact_history" || c.Evidence.ArtifactScope != c.ArtifactScope || c.Evidence.RunID != c.RunID || c.Evidence.TaskID != c.TaskID { + return nil, llm.CostGapScope + } + digest, err := checkpointDigest(&c) + if err != nil || digest != c.Digest { + return nil, llm.CostGapConflict + } + c.Evidence = llm.NormalizeRequestCostEvidence(c.Evidence) + seen := map[string]bool{} + for _, g := range c.Generations { + if !llm.SafeCostString(g.GenerationID, 256) || (g.InputFingerprint != "" && !llm.SafeCostString(g.InputFingerprint, 256)) || (g.EvidenceDigest != "" && !isCostDigest(g.EvidenceDigest)) { + return nil, llm.CostGapInvalid + } + if seen[g.GenerationID] { + costGap(&c, llm.CostGapDuplicate) + } + seen[g.GenerationID] = true + } + if c.Pending != nil && (!llm.SafeCostString(c.Pending.GenerationID, 256) || !llm.SafeCostString(c.Pending.InputFingerprint, 256)) { + return nil, llm.CostGapInvalid + } + if c.Pending != nil && seen[c.Pending.GenerationID] { + costGap(&c, llm.CostGapDuplicate) + } + if c.GenerationsObserved < uint64(len(c.Generations)) { + costGap(&c, llm.CostGapInvalid) + } + validateCostAttemptIdentities(&c) + costGap(&c, llm.CostGapLegacy) + return &c, "" +} + +func validateCostAttemptIdentities(c *costCheckpoint) { + seen := map[string]bool{} + ids := map[string]string{} + ordinals := map[string]uint32{} + for _, a := range c.Evidence.Attempts { + key := fmt.Sprintf("%s/%d", a.GenerationID, a.Ordinal) + if a.GenerationID == "" || a.Ordinal == 0 || a.AdapterAttempt == 0 || (a.ValidationPhase != "initial" && a.ValidationPhase != "correction") || seen[key] || a.Ordinal != ordinals[a.GenerationID]+1 { + costGap(c, llm.CostGapDuplicate) + } + seen[key] = true + ordinals[a.GenerationID] = a.Ordinal + if a.ProviderResponseID != nil { + if old, ok := ids[*a.ProviderResponseID]; ok && old != key { + costGap(c, llm.CostGapDuplicate) + } + ids[*a.ProviderResponseID] = key + } + } +} +func sameCostAttempt(a, b llm.RequestCostAttempt) bool { + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + return string(x) == string(y) +} + +// Admit only new records into the retained prefix, leaving framing/pending +// capacity. A later generation must not evict an earlier observation to fit. +func appendCostAttempt(c *costCheckpoint, a llm.RequestCostAttempt) bool { + if len(c.Evidence.Attempts) >= llm.MaxRequestCostAttempts { + return false + } + c.Evidence.Attempts = append(c.Evidence.Attempts, a) + b, _ := costCheckpointBytes(c) + if len(b) > llm.MaxRequestCostEvidenceBytes-llm.RequestCostReservedBytes { + c.Evidence.Attempts = c.Evidence.Attempts[:len(c.Evidence.Attempts)-1] + return false + } + return true +} +func appendCostGeneration(c *costCheckpoint, g costGeneration) bool { + if len(c.Generations) >= maxCostGenerations || c.OmittedGenerations != 0 { + return false + } + c.Generations = append(c.Generations, g) + b, _ := costCheckpointBytes(c) + if len(b) > llm.MaxRequestCostEvidenceBytes-llm.RequestCostReservedBytes { + c.Generations = c.Generations[:len(c.Generations)-1] + return false + } + return true +} + +// Repeated snapshots are references, not new charges. A conflicting same-identity +// copy keeps the existing prefix and a sticky ambiguity/loss reason. +func mergeCostObservations(c *costCheckpoint, other *llm.RequestCostEvidence) { + if other == nil { + return + } + e := llm.NormalizeRequestCostEvidence(other) + for _, gap := range e.Gaps { + costGap(c, gap) + } + if e.Scope != "task_artifact_history" || e.ArtifactScope != c.ArtifactScope || e.RunID != c.RunID || e.TaskID != c.TaskID { + costGap(c, llm.CostGapScope) + return + } + index := map[string]llm.RequestCostAttempt{} + for _, a := range c.Evidence.Attempts { + index[fmt.Sprintf("%s/%d", a.GenerationID, a.Ordinal)] = a + } + blocked := c.Evidence.Truncated + var omitted uint64 + for _, a := range e.Attempts { + key := fmt.Sprintf("%s/%d", a.GenerationID, a.Ordinal) + if prior, ok := index[key]; ok { + if !sameCostAttempt(prior, a) { + costGap(c, llm.CostGapConflict) + } + continue + } + if blocked || !appendCostAttempt(c, a) { + blocked = true + c.Evidence.Truncated = true + omitted = llm.SaturatingCostAdd(omitted, 1) + costGap(c, llm.CostGapOverflow) + continue + } + index[key] = a + } + if omitted > c.Evidence.DroppedAttempts { + c.Evidence.DroppedAttempts = omitted + } + if e.AttemptsObserved > c.Evidence.AttemptsObserved { + c.Evidence.AttemptsObserved = e.AttemptsObserved + } + if e.DroppedAttempts > c.Evidence.DroppedAttempts { + c.Evidence.DroppedAttempts = e.DroppedAttempts + } + if c.Evidence.AttemptsObserved < uint64(len(c.Evidence.Attempts)) { + c.Evidence.AttemptsObserved = uint64(len(c.Evidence.Attempts)) + } + c.Evidence.Truncated = c.Evidence.Truncated || e.Truncated + validateCostAttemptIdentities(c) + reconcileKnownCostGenerations(c) +} + +// Metadata-only observations establish a lower bound on observed generations, +// but cannot recreate missing finalized fingerprints/digests. Keep those missing +// descriptors as omissions with a gap rather than reporting a zero count. +func reconcileKnownCostGenerations(c *costCheckpoint) { + known := map[string]bool{} + described := map[string]bool{} + for _, g := range c.Generations { + known[g.GenerationID] = true + described[g.GenerationID] = true + } + for _, a := range c.Evidence.Attempts { + if a.GenerationID != "" { + known[a.GenerationID] = true + } + } + var missing uint64 + for id := range known { + if !described[id] { + missing++ + } + } + if uint64(len(known)) > c.GenerationsObserved { + c.GenerationsObserved = uint64(len(known)) + } + if missing > 0 { + costGap(c, llm.CostGapMissing) + if missing > c.OmittedGenerations { + c.OmittedGenerations = missing + } + } +} + +// The same bounded metadata-plus-sidecar reconciliation is used for execution, +// cache load, and reset, so reset cannot erase the last known A/B observations. +func reconcileCostCheckpoint(paths Paths, taskID, runID string, meta *Metadata, anyRun bool) (*costCheckpoint, error) { + // Reset has no explicit RunID argument. Valid same-root/task metadata is + // authoritative for its known observations; a foreign-run sidecar must not + // win and cause that sole metadata prefix to be discarded before deletion. + if anyRun && meta != nil && meta.requestCostScopeValid && meta.RequestCostEvidence != nil { + e := meta.RequestCostEvidence + scope, err := artifactCostScope(paths) + if err == nil && e.Version == 1 && e.Source == "openai_responses" && e.Scope == "task_artifact_history" && e.ArtifactScope == scope && e.TaskID == strings.TrimSpace(taskID) { + runID = e.RunID + anyRun = false + } + } + // With no valid raw scope, retain an independently valid same-root/task + // sidecar first. A binding is only a reference to a checkpoint: syntactic + // validity alone cannot authorize replacing an existing checkpoint under + // another run identity, including with an invented empty/no-run scope. + c, gap := readCostCheckpoint(paths, taskID, runID, anyRun) + if c == nil { + if anyRun && meta != nil && meta.RequestCostCheckpoint != nil { + b := meta.RequestCostCheckpoint + scope, err := artifactCostScope(paths) + if err == nil && b.Version == 1 && b.ArtifactScope == scope && b.TaskID == strings.TrimSpace(taskID) && (b.RunID == "" || llm.SafeCostString(b.RunID, 256)) { + runID = b.RunID + } + } + var err error + c, err = newCostCheckpoint(paths, taskID, runID) + if err != nil { + return nil, err + } + costGap(c, gap) + } + + if meta != nil { + if strings.TrimSpace(meta.TaskID) != c.TaskID { + costGap(c, llm.CostGapScope) + } else { + b := meta.RequestCostCheckpoint + if meta.requestCostScopeValid && b != nil && b.Digest == c.Digest && b.Sequence == c.Sequence && meta.RequestCostEvidence != nil { + left, _ := llm.RequestCostEvidenceDigest(c.Evidence) + right, _ := llm.RequestCostEvidenceDigest(meta.RequestCostEvidence) + if left != right { + costGap(c, llm.CostGapConflict) + } + } + if meta.requestCostScopeValid { + mergeCostObservations(c, meta.RequestCostEvidence) + } else if meta.RequestCostEvidence != nil { + costGap(c, llm.CostGapScope) + for _, gap := range meta.RequestCostEvidence.Gaps { + costGap(c, gap) + } + } + if meta.RequestCostEvidence != nil && (b == nil || b.Version != 1 || b.ArtifactScope != c.ArtifactScope || b.RunID != c.RunID || b.TaskID != c.TaskID || b.Sequence != c.Sequence || b.Digest != c.Digest) { + costGap(c, llm.CostGapConflict) + } + } + } + if c.Pending != nil { + costGap(c, llm.CostGapInterrupted) + } + c.Evidence = llm.NormalizeRequestCostEvidence(c.Evidence) + return c, nil +} + +func boundCostCheckpoint(c *costCheckpoint) { + c.FreshOrigin = false + costGap(c, llm.CostGapLegacy) + c.Evidence = llm.NormalizeRequestCostEvidence(c.Evidence) + if c.Generations == nil { + c.Generations = []costGeneration{} + } + if len(c.Generations) > maxCostGenerations { + c.OmittedGenerations = llm.SaturatingCostAdd(c.OmittedGenerations, uint64(len(c.Generations[maxCostGenerations:]))) + c.Generations = c.Generations[:maxCostGenerations] + costGap(c, llm.CostGapOverflow) + } + for { + b, _ := costCheckpointBytes(c) + if len(b) <= llm.MaxRequestCostEvidenceBytes-128 { + break + } + costGap(c, llm.CostGapOverflow) + if len(c.Evidence.Attempts) > 0 { + c.Evidence.Attempts = c.Evidence.Attempts[:len(c.Evidence.Attempts)-1] + c.Evidence.DroppedAttempts = llm.SaturatingCostAdd(c.Evidence.DroppedAttempts, 1) + c.Evidence.Truncated = true + } else if len(c.Generations) > 0 { + c.Generations = c.Generations[:len(c.Generations)-1] + c.OmittedGenerations = llm.SaturatingCostAdd(c.OmittedGenerations, 1) + } else { + break + } + } +} + +func writeCostCheckpoint(paths Paths, c *costCheckpoint) error { + boundCostCheckpoint(c) + if c.Sequence == ^uint64(0) { + costGap(c, llm.CostGapOverflow) + } else { + c.Sequence++ + } + digest, err := checkpointDigest(c) + if err != nil { + return err + } + c.Digest = digest + b, err := json.Marshal(c) + if err != nil { + return err + } + if len(b)+1 > llm.MaxRequestCostEvidenceBytes { + return errors.New("llmlifecycle: request-cost checkpoint exceeds limit") + } + p, err := paths.RequestCostCheckpoint(c.TaskID) + if err != nil { + return err + } + // fsatomic is temp-file/rename without fsync. No power-loss claim is made. + return fsatomic.WriteFileAtomic(p, append(b, '\n'), 0o600) +} + +func costFingerprint(req Request) string { + if s := strings.TrimSpace(req.InputFingerprint); s != "" { + return s + } + return Fingerprint(adapterName(req.Adapter), req.TaskID, req.Phase, req.Model, req.Effort, req.Prompt, req.DependencyTaskIDs) +} + +func beginCostGeneration(req Request, id string) (*costCheckpoint, error) { + meta, ok, err := ReadMetadata(req.Paths, req.TaskID) + if err != nil { + return nil, err + } + var observed *Metadata + if ok { + observed = &meta + } + c, err := reconcileCostCheckpoint(req.Paths, req.TaskID, req.RunID, observed, false) + if err != nil { + return nil, err + } + fingerprint := costFingerprint(req) + if !llm.SafeCostString(id, 256) || !llm.SafeCostString(fingerprint, 256) { + return nil, errors.New("llmlifecycle: invalid cost generation identity") + } + for _, g := range c.Generations { + if g.GenerationID == id { + return nil, errors.New("llmlifecycle: duplicate cost generation identity") + } + } + for _, a := range c.Evidence.Attempts { + if a.GenerationID == id { + return nil, errors.New("llmlifecycle: duplicate cost generation identity") + } + } + if c.Pending != nil { + if c.Pending.GenerationID == id { + return nil, errors.New("llmlifecycle: duplicate pending cost generation identity") + } + c.OmittedGenerations = llm.SaturatingCostAdd(c.OmittedGenerations, 1) + costGap(c, llm.CostGapInterrupted) + } + c.Pending = &pendingCostGeneration{GenerationID: id, InputFingerprint: fingerprint} + if err := writeCostCheckpoint(req.Paths, c); err != nil { + return nil, fmt.Errorf("llmlifecycle: checkpoint before dispatch: %w", err) + } + return c, nil +} + +func finishCostGeneration(paths Paths, c *costCheckpoint, invocation *llm.RequestCostEvidence) error { + if c == nil || c.Pending == nil { + return errors.New("llmlifecycle: missing pending cost generation") + } + e := llm.NormalizeRequestCostEvidence(invocation) + if e == nil || e.Scope != "invocation" { + e = llm.UnknownRequestCostEvidence(llm.CostGapMissing) + } + for _, gap := range e.Gaps { + costGap(c, gap) + } + c.Evidence.AttemptsObserved = llm.SaturatingCostAdd(c.Evidence.AttemptsObserved, e.AttemptsObserved) + c.Evidence.DroppedAttempts = llm.SaturatingCostAdd(c.Evidence.DroppedAttempts, e.DroppedAttempts) + // Preserve the invocation's retained prefix even when that invocation was + // truncated; later attempts must not replace an already full task prefix. + wasTruncated := c.Evidence.Truncated + for _, a := range e.Attempts { + a.GenerationID = c.Pending.GenerationID + if wasTruncated || !appendCostAttempt(c, a) { + wasTruncated = true + c.Evidence.Truncated = true + c.Evidence.DroppedAttempts = llm.SaturatingCostAdd(c.Evidence.DroppedAttempts, 1) + costGap(c, llm.CostGapOverflow) + } + } + c.Evidence.Truncated = c.Evidence.Truncated || e.Truncated + digest, err := llm.RequestCostEvidenceDigest(e) + if err != nil { + return err + } + c.GenerationsObserved = llm.SaturatingCostAdd(c.GenerationsObserved, 1) + if !appendCostGeneration(c, costGeneration{GenerationID: c.Pending.GenerationID, InputFingerprint: c.Pending.InputFingerprint, EvidenceDigest: digest}) { + c.OmittedGenerations = llm.SaturatingCostAdd(c.OmittedGenerations, 1) + costGap(c, llm.CostGapOverflow) + } + c.Pending = nil + validateCostAttemptIdentities(c) + return writeCostCheckpoint(paths, c) +} + +func checkpointBinding(c *costCheckpoint) *RequestCostCheckpointBinding { + if c == nil { + return nil + } + return &RequestCostCheckpointBinding{Version: 1, ArtifactScope: c.ArtifactScope, RunID: c.RunID, TaskID: c.TaskID, Sequence: c.Sequence, Digest: c.Digest} +} + +func overlayCostMetadata(paths Paths, taskID, runID string, meta *Metadata) { + c, err := reconcileCostCheckpoint(paths, taskID, runID, meta, false) + if err != nil { + meta.RequestCostEvidence = llm.UnknownRequestCostEvidence(llm.CostGapCheckpoint) + return + } + meta.RequestCostEvidence = llm.CloneRequestCostEvidence(c.Evidence) +} + +// Both schema-4 readers share this decoder. Optional cost corruption must not +// invalidate valid output/tool fields. The existing whole metadata read remains +// unbounded; the nested cost envelope is bounded before typed nested decoding. +func decodeMetadata(data []byte, meta *Metadata) error { + type ordinary Metadata + var raw struct { + *ordinary + Cost json.RawMessage `json:"request_cost_evidence"` + Binding json.RawMessage `json:"request_cost_checkpoint"` + } + raw.ordinary = (*ordinary)(meta) + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + meta.requestCostScopeValid = validRawMetadataCostScope(raw.Cost) + meta.RequestCostEvidence = llm.DecodeRequestCostEvidence(raw.Cost) + meta.RequestCostCheckpoint = nil + if len(raw.Binding) > 0 && string(raw.Binding) != "null" { + var b RequestCostCheckpointBinding + if len(raw.Binding) > llm.RequestCostReservedBytes || !llm.RequestCostJSONValid(raw.Binding) || json.Unmarshal(raw.Binding, &b) != nil || b.Version != 1 || !isCostDigest(b.Digest) || !llm.SafeCostString(b.TaskID, 256) || !isCostDigest(b.ArtifactScope) || (b.RunID != "" && !llm.SafeCostString(b.RunID, 256)) { + if meta.RequestCostEvidence == nil { + meta.RequestCostEvidence = llm.UnknownRequestCostEvidence(llm.CostGapInvalid) + } else { + meta.RequestCostEvidence.Gaps = llm.AddCostGap(meta.RequestCostEvidence.Gaps, llm.CostGapInvalid) + meta.RequestCostEvidence.AttemptCountExact = false + } + } else { + meta.RequestCostCheckpoint = &b + } + } + if duplicateMetadataCostFields(data) { + meta.RequestCostEvidence = llm.UnknownRequestCostEvidence(llm.CostGapDuplicate) + meta.RequestCostCheckpoint = nil + meta.requestCostScopeValid = false + } + return nil +} + +func duplicateMetadataCostFields(data []byte) bool { + d := json.NewDecoder(bytes.NewReader(data)) + token, err := d.Token() + if err != nil || token != json.Delim('{') { + return false + } + seen := map[string]bool{} + for d.More() { + token, err = d.Token() + if err != nil { + return false + } + key, ok := token.(string) + if !ok { + return false + } + canonical := "" + if strings.EqualFold(key, "request_cost_evidence") { + canonical = "request_cost_evidence" + } + if strings.EqualFold(key, "request_cost_checkpoint") { + canonical = "request_cost_checkpoint" + } + if canonical != "" { + if seen[canonical] { + return true + } + seen[canonical] = true + } + var ignored json.RawMessage + if d.Decode(&ignored) != nil { + return false + } + } + return false +} + +// Scope validity is measured on the raw envelope before normalization can omit +// invalid strings. In particular, an omitted invalid RunID is not proof of an +// actual empty/no-run scope. Only this raw fact (or a valid binding) can guide +// reset's scope selection; it never establishes complete accounting history. +func validRawMetadataCostScope(raw json.RawMessage) bool { + if len(raw) == 0 || len(raw) > llm.MaxRequestCostEvidenceBytes || !llm.RequestCostJSONValid(raw) { + return false + } + var scope struct { + Version int `json:"version"` + Source string `json:"source"` + Scope string `json:"scope"` + ArtifactScope string `json:"artifact_scope"` + RunID *string `json:"run_id"` + TaskID string `json:"task_id"` + } + if json.Unmarshal(raw, &scope) != nil { + return false + } + return scope.Version == 1 && scope.Source == "openai_responses" && scope.Scope == "task_artifact_history" && isCostDigest(scope.ArtifactScope) && llm.SafeCostString(scope.TaskID, 256) && scope.RunID != nil && (*scope.RunID == "" || llm.SafeCostString(*scope.RunID, 256)) +} diff --git a/internal/llmlifecycle/request_cost_evidence_test.go b/internal/llmlifecycle/request_cost_evidence_test.go new file mode 100644 index 00000000..71da3fbc --- /dev/null +++ b/internal/llmlifecycle/request_cost_evidence_test.go @@ -0,0 +1,814 @@ +package llmlifecycle + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "slices" + "strings" + "testing" + + "github.com/open-cli-collective/codereview-cli/internal/llm" + "github.com/open-cli-collective/codereview-cli/internal/runlock" +) + +type lifecycleCostAdapter struct{ *llm.FakeAdapter } + +func (*lifecycleCostAdapter) RequestCostEvidenceSource() string { return "openai_responses" } +func lifecycleCostPtr[T any](v T) *T { return &v } +func lifecycleCostResponse(id, text string) llm.Response { + return llm.Response{StructuredOutput: []byte(text), ReviewerToolEvidence: &llm.ReviewerToolEvidence{DiffStatus: llm.DiffToolStatusSucceeded}, RequestCostEvidence: &llm.RequestCostEvidence{ + Version: 1, Source: "openai_responses", Scope: "invocation", AttemptsObserved: 1, AttemptCountExact: true, + Attempts: []llm.RequestCostAttempt{{Ordinal: 1, AdapterAttempt: 1, ValidationPhase: "initial", RuntimeID: "openai-api-key", EndpointKind: "official_global", TransportMode: "single_send_v1", Dispatch: "possibly_dispatched", HTTPStatus: lifecycleCostPtr(200), BodyState: "complete", ProviderResponseID: lifecycleCostPtr(id), ObservedModel: lifecycleCostPtr("observed"), ObservedServiceTier: lifecycleCostPtr("default"), ResponseStatus: lifecycleCostPtr("completed"), RequestedModel: "model-1", Usage: &llm.OpenAIRequestUsage{InputTokens: lifecycleCostPtr(int64(10)), OutputTokens: lifecycleCostPtr(int64(3)), CacheReadTokens: lifecycleCostPtr(int64(2)), CacheWriteTokens: lifecycleCostPtr(int64(0))}}}, + }} +} +func newLifecycleCostAdapter() *lifecycleCostAdapter { + return &lifecycleCostAdapter{&llm.FakeAdapter{NameValue: "openai_api"}} +} +func requireCostAttempts(t *testing.T, e *llm.RequestCostEvidence, ids ...string) { + t.Helper() + if e == nil || len(e.Attempts) != len(ids) { + t.Fatalf("cost history=%#v, want %v", e, ids) + } + for i, id := range ids { + if e.Attempts[i].ProviderResponseID == nil || *e.Attempts[i].ProviderResponseID != id { + t.Fatalf("receipt %d=%#v, want %s", i, e.Attempts[i], id) + } + } + if e.AttemptCountExact || !slices.Contains(e.Gaps, llm.CostGapLegacy) { + t.Fatal("task history acquired an unestablished fresh origin") + } +} + +func TestRequestCostLifecycleSuccessCacheAndValidation(t *testing.T) { + for _, noRun := range []bool{false, true} { + t.Run(fmt.Sprint(noRun), func(t *testing.T) { + a := newLifecycleCostAdapter() + a.Queue(llm.FakeResult{SessionID: "provider-A", Response: lifecycleCostResponse("A", `invalid`)}) + a.Queue(llm.FakeResult{SessionID: "provider-B", Response: lifecycleCostResponse("B", `{"ok":true}`)}) + store := newLifecycleStore() + req := lifecycleRequest(t, store, a) + if noRun { + req.RunID = "" + req.AllowNoRunCache = true + req.Store = nil + } + first, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err != nil { + t.Fatal(err) + } + requireCostAttempts(t, first.Draft.Response.RequestCostEvidence, "A", "B") + cached, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err != nil || !cached.Cached { + t.Fatalf("cache err=%v", err) + } + requireCostAttempts(t, cached.Draft.Response.RequestCostEvidence, "A", "B") + if !reflect.DeepEqual(first.Draft.Response.RequestCostEvidence, cached.Draft.Response.RequestCostEvidence) || len(a.Requests()) != 2 { + t.Fatal("reload changed/counted history or called provider") + } + if !noRun && len(store.inserted) != 1 { + t.Fatal("cache inserted another session") + } + *cached.Draft.Response.RequestCostEvidence.Attempts[0].Usage.InputTokens = 900 + if *first.Draft.Response.RequestCostEvidence.Attempts[0].Usage.InputTokens != 10 { + t.Fatal("cache aliases response") + } + }) + } +} + +func TestRequestCostIsolatedFailureWithAndWithoutLedgerSession(t *testing.T) { + for _, sessionID := range []string{"", "provider-A"} { + a := newLifecycleCostAdapter() + a.Queue(llm.FakeResult{SessionID: sessionID, Response: lifecycleCostResponse("A", `invalid`)}) + a.Queue(llm.FakeResult{SessionID: sessionID, Response: lifecycleCostResponse("B", `invalid`)}) + req := lifecycleRequest(t, newLifecycleStore(), a) + req.FailureStatus = StatusFailedIsolated + first, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err == nil { + t.Fatal("invalid output succeeded") + } + requireCostAttempts(t, first.Draft.Response.RequestCostEvidence, "A", "B") + cached, ok, err := LoadStructured(context.Background(), req, decodeLifecyclePayload) + if !ok || err == nil || !cached.Cached { + t.Fatalf("isolated cache ok=%t err=%v", ok, err) + } + requireCostAttempts(t, cached.Draft.Response.RequestCostEvidence, "A", "B") + if len(a.Requests()) != 2 { + t.Fatal("isolated cache dispatched again") + } + } +} + +func TestRequestCostResetRetainsMetadataPrefixAfterMissingSidecar(t *testing.T) { + a := newLifecycleCostAdapter() + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("A", `invalid`)}) + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("B", `{"ok":true}`)}) + req := lifecycleRequest(t, newLifecycleStore(), a) + first, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err != nil { + t.Fatal(err) + } + requireCostAttempts(t, first.Draft.Response.RequestCostEvidence, "A", "B") + path, _ := req.Paths.RequestCostCheckpoint(req.TaskID) + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if err := ResetIfInputFingerprintChanged(req.Paths, req.TaskID, "changed"); err != nil { + t.Fatal(err) + } + preserved, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" { + t.Fatalf("preserved checkpoint gap=%s", gap) + } + requireCostAttempts(t, preserved.Evidence, "A", "B") + if preserved.GenerationsObserved < 1 || preserved.OmittedGenerations < 1 { + t.Fatal("metadata-only known generation reported as zero") + } + req.InputFingerprint = "changed" + req.Model = "changed-model" + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("C", `{"ok":true}`)}) + last, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err != nil { + t.Fatal(err) + } + requireCostAttempts(t, last.Draft.Response.RequestCostEvidence, "A", "B", "C") + if !slices.Contains(last.Draft.Response.RequestCostEvidence.Gaps, llm.CostGapCheckpoint) { + t.Fatal("missing sidecar gap healed") + } + final, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" || final.GenerationsObserved < 2 || final.OmittedGenerations < 1 { + t.Fatal("retained A/B/C generation lower bound lost") + } +} + +func TestRequestCostInterruptionAndFinalBeforeLedger(t *testing.T) { + a := newLifecycleCostAdapter() + req := lifecycleRequest(t, newLifecycleStore(), a) + if _, err := beginCostGeneration(req, "interrupted"); err != nil { + t.Fatal(err) + } + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("C", `{"ok":true}`)}) + got, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(got.Draft.Response.RequestCostEvidence.Gaps, llm.CostGapInterrupted) { + t.Fatal("pending generation erased") + } + b := newLifecycleCostAdapter() + b.Queue(llm.FakeResult{SessionID: "reported", Response: lifecycleCostResponse("D", `{"ok":true}`)}) + store := newLifecycleStore() + store.insertErr = errors.New("ledger failure") + req = lifecycleRequest(t, store, b) + got, err = RunStructured(context.Background(), req, decodeLifecyclePayload) + if err == nil { + t.Fatal("ledger failure hidden") + } + requireCostAttempts(t, got.Draft.Response.RequestCostEvidence, "D") + checkpoint, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" || checkpoint.Pending != nil { + t.Fatalf("final checkpoint missing before ledger: %s", gap) + } + requireCostAttempts(t, checkpoint.Evidence, "D") +} + +func TestRequestCostCheckpointFailurePreventsDispatchAndReset(t *testing.T) { + a := newLifecycleCostAdapter() + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("A", `{"ok":true}`)}) + req := lifecycleRequest(t, newLifecycleStore(), a) + path, _ := req.Paths.RequestCostCheckpoint(req.TaskID) + if err := os.MkdirAll(path, 0o700); err != nil { + t.Fatal(err) + } + if _, err := RunStructured(context.Background(), req, decodeLifecyclePayload); err == nil || len(a.Requests()) != 0 { + t.Fatal("dispatched without pending checkpoint") + } + meta := Metadata{SchemaVersion: SchemaVersion, TaskID: req.TaskID, InputFingerprint: "old"} + if err := WriteMetadata(req.Paths, meta); err != nil { + t.Fatal(err) + } + if err := ResetIfInputFingerprintChanged(req.Paths, req.TaskID, "new"); err == nil { + t.Fatal("reset without preservation") + } + if _, ok, _ := ReadMetadata(req.Paths, req.TaskID); !ok { + t.Fatal("reset erased unpreserved metadata") + } +} + +func TestRequestCostBothSchema4ReadersIsolateOptionalCorruption(t *testing.T) { + for _, raw := range []string{`"bad"`, `{"version":2}`, `{"version":1,"attempts":"bad"}`, `{"version":1,"version":1}`, `{"huge":"` + strings.Repeat("x", llm.MaxRequestCostEvidenceBytes) + `"}`} { + paths := Paths{LLMTasksDir: t.TempDir()} + path, _ := paths.Metadata("task") + data := []byte(`{"schema_version":4,"task_id":"task","status":"succeeded","reviewer_tool_evidence":{"diff_status":"succeeded","trace":{"version":1,"source":"pi_rpc","calls":[]}},"request_cost_evidence":` + raw + `}`) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + meta, ok, err := ReadMetadata(paths, "task") + if err != nil || !ok { + t.Fatalf("ReadMetadata rejected optional cost: %v", err) + } + all, err := ListMetadata(paths) + if err != nil || len(all) != 1 { + t.Fatalf("ListMetadata rejected optional cost: %v", err) + } + for _, m := range []Metadata{meta, all[0]} { + if m.ReviewerToolEvidence == nil || m.ReviewerToolEvidence.Trace == nil || m.ReviewerToolEvidence.DiffStatus != llm.DiffToolStatusSucceeded || m.RequestCostEvidence == nil || m.RequestCostEvidence.AttemptCountExact { + t.Fatal("cost corruption damaged valid tool metadata or became complete") + } + } + } + var meta Metadata + if decodeMetadata([]byte(`{"schema_version":4`), &meta) == nil { + t.Fatal("malformed whole metadata accepted") + } +} + +type blockingCostAdapter struct { + *lifecycleCostAdapter + entered chan struct{} + release chan struct{} +} + +func (a *blockingCostAdapter) Start(ctx context.Context, req llm.Request) (llm.Stream, error) { + close(a.entered) + <-a.release + return a.lifecycleCostAdapter.Start(ctx, req) +} +func TestRequestCostConcurrentRunNoRunAndResetOwnership(t *testing.T) { + for _, noRun := range []bool{false, true} { + t.Run(fmt.Sprint(noRun), func(t *testing.T) { + a := &blockingCostAdapter{newLifecycleCostAdapter(), make(chan struct{}), make(chan struct{})} + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("A", `{"ok":true}`)}) + req := lifecycleRequest(t, newLifecycleStore(), a) + if noRun { + req.RunID = "" + req.AllowNoRunCache = true + req.Store = nil + } + finished := make(chan error, 1) + go func() { _, err := RunStructured(context.Background(), req, decodeLifecyclePayload); finished <- err }() + <-a.entered + if _, err := RunStructured(context.Background(), req, decodeLifecyclePayload); !errors.Is(err, runlock.ErrHeld) { + t.Errorf("concurrent run=%v", err) + } + if _, _, err := LoadStructured(context.Background(), req, decodeLifecyclePayload); !errors.Is(err, runlock.ErrHeld) { + t.Errorf("concurrent load=%v", err) + } + if err := ResetIfInputFingerprintChanged(req.Paths, req.TaskID, "changed"); !errors.Is(err, runlock.ErrHeld) { + t.Errorf("active reset=%v", err) + } + close(a.release) + if err := <-finished; err != nil { + t.Fatal(err) + } + if len(a.Requests()) != 1 { + t.Fatal("multiple dispatch owners") + } + }) + } +} + +func TestRequestCostGenerationAndByteBoundsKeepPendingSpace(t *testing.T) { + paths := Paths{LLMTasksDir: t.TempDir()} + c, err := newCostCheckpoint(paths, "task", "run") + if err != nil { + t.Fatal(err) + } + for i := 0; i < 65; i++ { + c.Pending = &pendingCostGeneration{GenerationID: fmt.Sprint(i), InputFingerprint: "fingerprint"} + if err := writeCostCheckpoint(paths, c); err != nil { + t.Fatal(err) + } + if err := finishCostGeneration(paths, c, lifecycleCostResponse(fmt.Sprint(i), "").RequestCostEvidence); err != nil { + t.Fatal(err) + } + } + if len(c.Generations) != 64 || c.OmittedGenerations != 1 || len(c.Evidence.Attempts) != 64 || !c.Evidence.Truncated { + t.Fatalf("bounds=%#v", c) + } + c.Pending = &pendingCostGeneration{GenerationID: "still-pending", InputFingerprint: "fingerprint"} + if err := writeCostCheckpoint(paths, c); err != nil { + t.Fatal(err) + } + b, _ := json.Marshal(c) + if len(b) > llm.MaxRequestCostEvidenceBytes || c.Pending == nil { + t.Fatal("full prefix prevented pending marker") + } + got, gap := readCostCheckpoint(paths, "task", "other-run", false) + if got != nil || gap != llm.CostGapScope { + t.Fatal("cross-run checkpoint imported") + } +} + +func TestRequestCostCheckpointDigestIncludesScopePendingAndFingerprint(t *testing.T) { + c, _ := newCostCheckpoint(Paths{LLMTasksDir: t.TempDir()}, "task", "run") + c.Pending = &pendingCostGeneration{GenerationID: "generation", InputFingerprint: "first"} + a, _ := checkpointDigest(c) + c.Pending.InputFingerprint = "second" + b, _ := checkpointDigest(c) + c.Pending = nil + d, _ := checkpointDigest(c) + c.RunID = "other" + e, _ := checkpointDigest(c) + if a == b || b == d || d == e { + t.Fatal("digest omitted scoped pending or fingerprint state") + } +} + +func TestRequestCostFinalizationFailureKeepsPendingCheckpoint(t *testing.T) { + req := lifecycleRequest(t, nil, newLifecycleCostAdapter()) + c, err := beginCostGeneration(req, "generation") + if err != nil { + t.Fatal(err) + } + path, _ := req.Paths.RequestCostCheckpoint(req.TaskID) + // Blocking the temporary destination models a failure before atomic rename. + if err := os.Mkdir(path+".tmp", 0o700); err != nil { + t.Fatal(err) + } + e := lifecycleCostResponse("A", "").RequestCostEvidence + e.Attempts[0].Ordinal = 1 + if err := finishCostGeneration(req.Paths, c, e); err == nil { + t.Fatal("finalization failure hidden") + } + persisted, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" || persisted.Pending == nil || persisted.Pending.GenerationID != "generation" { + t.Fatal("failed finalization overwrote pending marker") + } +} + +func TestRequestCostMetadataAndOutputFailuresKeepFinalCheckpoint(t *testing.T) { + for _, target := range []string{"metadata.json", "validated-output.json"} { + a := newLifecycleCostAdapter() + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("A", `{"ok":true}`)}) + req := lifecycleRequest(t, newLifecycleStore(), a) + dir, _ := req.Paths.TaskDir(req.TaskID) + // metadata itself must remain absent for pre-dispatch reads. A directory + // at its .tmp blocks only the later atomic commit. + if err := os.MkdirAll(filepath.Join(dir, target+".tmp"), 0o700); err != nil { + t.Fatal(err) + } + got, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err == nil { + t.Fatal("artifact failure hidden") + } + requireCostAttempts(t, got.Draft.Response.RequestCostEvidence, "A") + c, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" || c.Pending != nil { + t.Fatal("artifact error erased final cost checkpoint") + } + requireCostAttempts(t, c.Evidence, "A") + } +} + +func TestRequestCostReconciliationCorruptMissingConflictingAndLegacy(t *testing.T) { + for _, damage := range []string{"missing", "corrupt", "oversized", "wrong-scope"} { + a := newLifecycleCostAdapter() + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("A", `{"ok":true}`)}) + req := lifecycleRequest(t, newLifecycleStore(), a) + if _, err := RunStructured(context.Background(), req, decodeLifecyclePayload); err != nil { + t.Fatal(err) + } + path, _ := req.Paths.RequestCostCheckpoint(req.TaskID) + switch damage { + case "missing": + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + case "corrupt": + if err := os.WriteFile(path, []byte(`{`), 0o600); err != nil { + t.Fatal(err) + } + case "oversized": + if err := os.WriteFile(path, []byte(strings.Repeat("x", llm.MaxRequestCostEvidenceBytes+1)), 0o600); err != nil { + t.Fatal(err) + } + case "wrong-scope": + c, _ := newCostCheckpoint(req.Paths, "other", req.RunID) + c.Digest, _ = checkpointDigest(c) + b, _ := json.Marshal(c) + if err := os.WriteFile(path, b, 0o600); err != nil { + t.Fatal(err) + } + } + got, ok, err := LoadStructured(context.Background(), req, decodeLifecyclePayload) + if err != nil || !ok || !got.Value.OK { + t.Fatalf("cost damage broke output reuse: %s %v", damage, err) + } + requireCostAttempts(t, got.Draft.Response.RequestCostEvidence, "A") + if len(got.Draft.Response.RequestCostEvidence.Gaps) < 2 || len(a.Requests()) != 1 { + t.Fatal("gap repaired by assumption or provider call") + } + } + paths := Paths{LLMTasksDir: t.TempDir()} + c, _ := newCostCheckpoint(paths, "task", "run") + e := lifecycleCostResponse("A", "").RequestCostEvidence + e.Scope = "task_artifact_history" + e.ArtifactScope = c.ArtifactScope + e.TaskID = c.TaskID + e.RunID = c.RunID + e.Attempts[0].GenerationID = "g" + e.Attempts[0].Ordinal = 1 + mergeCostObservations(c, e) + mergeCostObservations(c, e) + if len(c.Evidence.Attempts) != 1 { + t.Fatal("duplicate cache reference counted twice") + } + conflict := llm.CloneRequestCostEvidence(e) + *conflict.Attempts[0].Usage.InputTokens = 99 + mergeCostObservations(c, conflict) + if len(c.Evidence.Attempts) != 1 || !slices.Contains(c.Evidence.Gaps, llm.CostGapConflict) { + t.Fatal("same identity conflict silently selected") + } + e.Attempts[0].GenerationID = "other-generation" + mergeCostObservations(c, e) + if !slices.Contains(c.Evidence.Gaps, llm.CostGapDuplicate) { + t.Fatal("repeated provider response ID silently deduplicated") + } + freshLooking, _ := reconcileCostCheckpoint(Paths{LLMTasksDir: filepath.Join(t.TempDir(), "absent")}, "task", "run", nil, false) + if freshLooking.Evidence.AttemptCountExact || !slices.Contains(freshLooking.Evidence.Gaps, llm.CostGapLegacy) { + t.Fatal("directory absence became origin proof") + } +} + +func TestRequestCostCheckpointCanonicalGolden(t *testing.T) { + want, err := os.ReadFile("testdata/request-cost-checkpoint-v1.golden.json") + if err != nil { + t.Fatal(err) + } + var c costCheckpointProjection + if json.Unmarshal(want, &c) != nil { + t.Fatal("golden fixture") + } + checkpoint := &costCheckpoint{costCheckpointProjection: c} + got, err := costCheckpointBytes(checkpoint) + if err != nil || string(got) != string(want) { + t.Fatalf("canonical bytes changed: %s %v", got, err) + } + digest, err := checkpointDigest(checkpoint) + if err != nil || digest != "3b909dbf6714f12b12b8a5a508f7fd63d4af3d640907d729c37a57a5abb74cc8" { + t.Fatalf("digest=%s err=%v", digest, err) + } +} + +func TestRequestCostResetForeignRunSidecarPreservesMetadataPrefix(t *testing.T) { + a := newLifecycleCostAdapter() + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("A", `invalid`)}) + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("B", `{"ok":true}`)}) + req := lifecycleRequest(t, newLifecycleStore(), a) + if _, err := RunStructured(context.Background(), req, decodeLifecyclePayload); err != nil { + t.Fatal(err) + } + foreign, _ := newCostCheckpoint(req.Paths, req.TaskID, "foreign-run") + if err := writeCostCheckpoint(req.Paths, foreign); err != nil { + t.Fatal(err) + } + if err := ResetIfInputFingerprintChanged(req.Paths, req.TaskID, "changed"); err != nil { + t.Fatal(err) + } + preserved, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" { + t.Fatal(gap) + } + requireCostAttempts(t, preserved.Evidence, "A", "B") + if !slices.Contains(preserved.Evidence.Gaps, llm.CostGapScope) { + t.Fatal("foreign-run mismatch gap missing") + } + req.InputFingerprint = "changed" + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("C", `{"ok":true}`)}) + got, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err != nil { + t.Fatal(err) + } + requireCostAttempts(t, got.Draft.Response.RequestCostEvidence, "A", "B", "C") +} + +func TestRequestCostNearByteLimitMetadataReadersAndReset(t *testing.T) { + paths := Paths{LLMTasksDir: t.TempDir()} + c, _ := newCostCheckpoint(paths, "task", "run") + for i := 0; i < 64; i++ { + a := lifecycleCostResponse(fmt.Sprint(i), "").RequestCostEvidence.Attempts[0] + a.GenerationID = fmt.Sprintf("%02d", i) + strings.Repeat("<", 254) + a.RequestedModel = strings.Repeat("<", 256) + a.ProviderResponseID = lifecycleCostPtr(fmt.Sprintf("%02d", i) + strings.Repeat("<", 254)) + a.ObservedModel = lifecycleCostPtr(strings.Repeat("<", 256)) + a.RequestedServiceTier = lifecycleCostPtr(strings.Repeat("<", 64)) + a.ObservedServiceTier = lifecycleCostPtr(strings.Repeat("<", 64)) + if !appendCostAttempt(c, a) { + break + } + c.Evidence.AttemptsObserved++ + } + if len(c.Evidence.Attempts) < 2 { + t.Fatal("near-cap fixture did not retain observations") + } + if err := writeCostCheckpoint(paths, c); err != nil { + t.Fatal(err) + } + meta := Metadata{SchemaVersion: SchemaVersion, TaskID: "task", InputFingerprint: "before", ReviewerToolEvidence: &llm.ReviewerToolEvidence{DiffStatus: llm.DiffToolStatusSucceeded}, RequestCostEvidence: llm.CloneRequestCostEvidence(c.Evidence), RequestCostCheckpoint: checkpointBinding(c)} + pretty, err := json.MarshalIndent(meta, "", " ") + if err != nil { + t.Fatal(err) + } + var prettyFields map[string]json.RawMessage + if err := json.Unmarshal(pretty, &prettyFields); err != nil { + t.Fatal(err) + } + if len(prettyFields["request_cost_evidence"]) <= llm.MaxRequestCostEvidenceBytes { + t.Fatal("near-cap fixture does not expose the historical indented-envelope overflow") + } + if err := WriteMetadata(paths, meta); err != nil { + t.Fatal(err) + } + read, ok, err := ReadMetadata(paths, "task") + if err != nil || !ok { + t.Fatal(err) + } + listed, err := ListMetadata(paths) + if err != nil || len(listed) != 1 { + t.Fatal(err) + } + for _, m := range []Metadata{read, listed[0]} { + if m.RequestCostEvidence == nil || len(m.RequestCostEvidence.Attempts) != len(c.Evidence.Attempts) || m.ReviewerToolEvidence == nil { + t.Fatal("durable formatting inflated cost envelope beyond reader bound") + } + } + path, _ := paths.RequestCostCheckpoint("task") + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if err := ResetIfInputFingerprintChanged(paths, "task", "after"); err != nil { + t.Fatal(err) + } + preserved, gap := readCostCheckpoint(paths, "task", "run", false) + if gap != "" || len(preserved.Evidence.Attempts) != len(c.Evidence.Attempts) { + t.Fatal("near-cap metadata prefix lost at reset") + } +} + +func TestRequestCostReadersRejectCaseFoldedDuplicateCostFields(t *testing.T) { + for _, tc := range []struct { + fields string + gap llm.CostEvidenceGap + }{ + {`"request_cost_evidence":{"version":1},"REQUEST_COST_EVIDENCE":{"version":2}`, llm.CostGapDuplicate}, + {`"request_cost_evidence":{"version":1,"VERSION":1,"source":"openai_responses","scope":"invocation","attempts_observed":0,"attempt_count_exact":true,"gaps":[],"attempts":[]}`, llm.CostGapInvalid}, + {`"request_cost_checkpoint":null,"REQUEST_COST_CHECKPOINT":null`, llm.CostGapDuplicate}, + } { + paths := Paths{LLMTasksDir: t.TempDir()} + path, _ := paths.Metadata("task") + body := []byte(`{"schema_version":4,"task_id":"task","reviewer_tool_evidence":{"diff_status":"succeeded"},` + tc.fields + `}`) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, body, 0o600); err != nil { + t.Fatal(err) + } + meta, ok, err := ReadMetadata(paths, "task") + if err != nil || !ok { + t.Fatal(err) + } + listed, err := ListMetadata(paths) + if err != nil || len(listed) != 1 { + t.Fatal(err) + } + for _, m := range []Metadata{meta, listed[0]} { + if m.ReviewerToolEvidence == nil || m.RequestCostEvidence == nil || !slices.Contains(m.RequestCostEvidence.Gaps, tc.gap) || m.RequestCostEvidence.AttemptCountExact { + t.Fatalf("case-folded duplicate did not produce expected %s gap", tc.gap) + } + } + } +} + +func TestRequestCostSidecarNamesPreserveExistingLongTaskIDs(t *testing.T) { + for _, length := range []int{234, 238, 242} { + for _, direct := range []bool{false, true} { + fake := &llm.FakeAdapter{} + var adapter llm.Adapter = fake + if direct { + adapter = &lifecycleCostAdapter{fake} + } + fake.Queue(llm.FakeResult{Response: lifecycleCostResponse("A", `{"ok":true}`)}) + req := lifecycleRequest(t, newLifecycleStore(), adapter) + req.TaskID = strings.Repeat("a", length) + if _, err := RunStructured(context.Background(), req, decodeLifecyclePayload); err != nil { + t.Fatalf("length=%d direct=%t error=%v", length, direct, err) + } + path, err := req.Paths.RequestCostCheckpoint(req.TaskID) + if err != nil || len(filepath.Base(path)+".tmp") > 255 { + t.Fatal("sidecar suffix exceeds filename budget") + } + if err := ResetIfInputFingerprintChanged(req.Paths, req.TaskID, "changed"); err != nil { + t.Fatal(err) + } + } + } +} + +func TestRequestCostResetInvalidMetadataRunIDKeepsNewerValidSidecar(t *testing.T) { + for _, badRunID := range []string{"bad\nrun", strings.Repeat("x", 257)} { + a := newLifecycleCostAdapter() + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("A", `{"ok":true}`)}) + req := lifecycleRequest(t, newLifecycleStore(), a) + if _, err := RunStructured(context.Background(), req, decodeLifecyclePayload); err != nil { + t.Fatal(err) + } + metadataPath, _ := req.Paths.Metadata(req.TaskID) + metadataRoot, err := os.OpenRoot(filepath.Dir(metadataPath)) + if err != nil { + t.Fatal(err) + } + data, err := metadataRoot.ReadFile(filepath.Base(metadataPath)) + _ = metadataRoot.Close() + if err != nil { + t.Fatal(err) + } + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + t.Fatal(err) + } + var evidence map[string]json.RawMessage + if err := json.Unmarshal(raw["request_cost_evidence"], &evidence); err != nil { + t.Fatal(err) + } + evidence["run_id"], _ = json.Marshal(badRunID) + raw["request_cost_evidence"], _ = json.Marshal(evidence) + corrupt, _ := json.Marshal(raw) + if err := os.WriteFile(metadataPath, corrupt, 0o600); err != nil { + t.Fatal(err) + } + checkpoint, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" { + t.Fatal(gap) + } + checkpoint.Pending = &pendingCostGeneration{GenerationID: "newer-generation", InputFingerprint: req.InputFingerprint} + if err := finishCostGeneration(req.Paths, checkpoint, lifecycleCostResponse("B", "").RequestCostEvidence); err != nil { + t.Fatal(err) + } + if err := ResetIfInputFingerprintChanged(req.Paths, req.TaskID, "changed"); err != nil { + t.Fatal(err) + } + preserved, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" { + t.Fatal(gap) + } + requireCostAttempts(t, preserved.Evidence, "A", "B") + if preserved.RunID != req.RunID || !slices.Contains(preserved.Evidence.Gaps, llm.CostGapScope) { + t.Fatal("invalid metadata RunID became authoritative no-run scope") + } + } +} + +func TestRequestCostResetUntrustedBindingCannotDisplaceNewerSidecar(t *testing.T) { + for _, rawScope := range []string{"invalid", "overlong", "null", "missing"} { + for _, actualRunID := range []string{"run-1", ""} { + for _, bindingRun := range []string{"foreign-run", ""} { + runLabel := actualRunID + if runLabel == "" { + runLabel = "no-run" + } + t.Run(rawScope+"/"+runLabel+"/"+bindingRun, func(t *testing.T) { + adapter := newLifecycleCostAdapter() + adapter.Queue(llm.FakeResult{Response: lifecycleCostResponse("A", `{"ok":true}`)}) + req := lifecycleRequest(t, newLifecycleStore(), adapter) + req.RunID = actualRunID + if actualRunID == "" { + req.AllowNoRunCache = true + req.Store = nil + } + if _, err := RunStructured(context.Background(), req, decodeLifecyclePayload); err != nil { + t.Fatal(err) + } + metadataPath, _ := req.Paths.Metadata(req.TaskID) + metadataRoot, err := os.OpenRoot(filepath.Dir(metadataPath)) + if err != nil { + t.Fatal(err) + } + data, err := metadataRoot.ReadFile(filepath.Base(metadataPath)) + _ = metadataRoot.Close() + if err != nil { + t.Fatal(err) + } + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + t.Fatal(err) + } + var evidence map[string]json.RawMessage + if err := json.Unmarshal(raw["request_cost_evidence"], &evidence); err != nil { + t.Fatal(err) + } + switch rawScope { + case "invalid": + evidence["run_id"], _ = json.Marshal("bad\nrun") + case "overlong": + evidence["run_id"], _ = json.Marshal(strings.Repeat("x", 257)) + case "null": + evidence["run_id"] = json.RawMessage(`null`) + case "missing": + delete(evidence, "run_id") + } + var binding map[string]json.RawMessage + if err := json.Unmarshal(raw["request_cost_checkpoint"], &binding); err != nil { + t.Fatal(err) + } + binding["run_id"], _ = json.Marshal(bindingRun) + raw["request_cost_evidence"], _ = json.Marshal(evidence) + raw["request_cost_checkpoint"], _ = json.Marshal(binding) + corrupt, _ := json.Marshal(raw) + if err := os.WriteFile(metadataPath, corrupt, 0o600); err != nil { + t.Fatal(err) + } + checkpoint, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" { + t.Fatal(gap) + } + checkpoint.Pending = &pendingCostGeneration{GenerationID: "newer-generation", InputFingerprint: req.InputFingerprint} + if err := finishCostGeneration(req.Paths, checkpoint, lifecycleCostResponse("B", "").RequestCostEvidence); err != nil { + t.Fatal(err) + } + if err := ResetIfInputFingerprintChanged(req.Paths, req.TaskID, "changed"); err != nil { + t.Fatal(err) + } + preserved, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" { + t.Fatal(gap) + } + requireCostAttempts(t, preserved.Evidence, "A", "B") + if preserved.RunID != req.RunID || !slices.Contains(preserved.Evidence.Gaps, llm.CostGapScope) || !slices.Contains(preserved.Evidence.Gaps, llm.CostGapConflict) { + t.Fatal("untrusted binding displaced the newer sidecar or hid ambiguity") + } + req.InputFingerprint = "changed" + adapter.Queue(llm.FakeResult{Response: lifecycleCostResponse("C", `{"ok":true}`)}) + result, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err != nil { + t.Fatal(err) + } + requireCostAttempts(t, result.Draft.Response.RequestCostEvidence, "A", "B", "C") + }) + } + } + } +} + +func TestRequestCostSidecarOwnRawRunScopeMustBeValid(t *testing.T) { + for _, mutation := range []string{"invalid", "overlong", "missing", "null", "actual-empty"} { + paths := Paths{LLMTasksDir: t.TempDir()} + checkpoint, err := newCostCheckpoint(paths, "task", "") + if err != nil { + t.Fatal(err) + } + if mutation == "invalid" { + checkpoint.RunID = "bad\nrun" + checkpoint.Evidence.RunID = checkpoint.RunID + } + if mutation == "overlong" { + checkpoint.RunID = strings.Repeat("x", 257) + checkpoint.Evidence.RunID = checkpoint.RunID + } + checkpoint.Digest, err = checkpointDigest(checkpoint) + if err != nil { + t.Fatal(err) + } + encoded, err := json.Marshal(checkpoint) + if err != nil { + t.Fatal(err) + } + var raw map[string]json.RawMessage + if err := json.Unmarshal(encoded, &raw); err != nil { + t.Fatal(err) + } + if mutation == "missing" { + delete(raw, "run_id") + } + if mutation == "null" { + raw["run_id"] = json.RawMessage(`null`) + } + encoded, err = json.Marshal(raw) + if err != nil { + t.Fatal(err) + } + path, _ := paths.RequestCostCheckpoint("task") + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, encoded, 0o600); err != nil { + t.Fatal(err) + } + got, gap := readCostCheckpoint(paths, "task", "", true) + if mutation == "actual-empty" { + if got == nil || gap != "" { + t.Fatal("genuine empty no-run scope rejected") + } + } else if got != nil || gap != llm.CostGapScope { + t.Fatal("invalid sidecar run scope became authoritative") + } + } +} diff --git a/internal/llmlifecycle/testdata/request-cost-checkpoint-v1.golden.json b/internal/llmlifecycle/testdata/request-cost-checkpoint-v1.golden.json new file mode 100644 index 00000000..f56452a2 --- /dev/null +++ b/internal/llmlifecycle/testdata/request-cost-checkpoint-v1.golden.json @@ -0,0 +1 @@ +{"version":1,"artifact_scope":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","run_id":"run","task_id":"task","sequence":7,"fresh_origin":false,"generations_observed":1,"omitted_generations":0,"evidence":{"version":1,"source":"openai_responses","scope":"task_artifact_history","artifact_scope":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","run_id":"run","task_id":"task","attempts_observed":1,"attempt_count_exact":false,"truncated":false,"dropped_attempts":0,"gaps":["legacy_history_unknown"],"attempts":[{"generation_id":"g","ordinal":1,"validation_phase":"initial","adapter_attempt":1,"runtime_id":"openai-api-key","endpoint_kind":"official_global","transport_mode":"single_send_v1","dispatch":"not_dispatched","http_status":null,"body_state":"missing","provider_response_id":null,"observed_model":null,"observed_service_tier":null,"response_status":null,"requested_model":"requested\u003cmodel\u003e","requested_service_tier":null,"usage":null,"issues":[]}]},"generations":[{"generation_id":"g","input_fingerprint":"fingerprint","evidence_digest":"40a0d89467879b363d192692bb0df2599c92fb476b4e1f7db8ad88368ad842d0"}],"pending":null} \ No newline at end of file diff --git a/internal/llmlifecycle/tool_evidence_test.go b/internal/llmlifecycle/tool_evidence_test.go new file mode 100644 index 00000000..53d5212d --- /dev/null +++ b/internal/llmlifecycle/tool_evidence_test.go @@ -0,0 +1,160 @@ +package llmlifecycle + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/open-cli-collective/codereview-cli/internal/llm" +) + +func TestToolTraceSurvivesDurableReload(t *testing.T) { + for _, callerOwned := range []bool{false, true} { + for _, failed := range []bool{false, true} { + t.Run(fmt.Sprintf("caller_owned=%t/failed=%t", callerOwned, failed), func(t *testing.T) { + want := lifecycleToolEvidence() + adapter := &llm.FakeAdapter{NameValue: "fake-llm"} + result := llm.FakeResult{SessionID: "session", Response: llm.Response{ + StructuredOutput: []byte(`{"ok":true}`), ReviewerToolEvidence: want, + }} + adapter.Queue(result) + req := lifecycleRequest(t, newLifecycleStore(), adapter) + if failed { + // FakeAdapter's WaitErr discards Response. A real Pi failure can + // return collected telemetry alongside the terminal error. + req.Adapter = toolEvidenceWaitErrorAdapter{Adapter: adapter} + } + req.FailureStatus = StatusFailedIsolated + if callerOwned { + req.RunID = "" + req.Store = nil + req.AllowNoRunCache = true + } + _, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if (err != nil) != failed { + t.Fatalf("fresh error = %v, failure expected %t", err, failed) + } + meta, exists, err := ReadMetadata(req.Paths, req.TaskID) + if err != nil || !exists || !reflect.DeepEqual(meta.ReviewerToolEvidence, want) { + t.Fatalf("persisted evidence = %#v, exists %t err %v", meta.ReviewerToolEvidence, exists, err) + } + cachedAdapter := &llm.FakeAdapter{NameValue: "fake-llm"} + req.Adapter = cachedAdapter + got, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if (err != nil) != failed || !got.Cached || !reflect.DeepEqual(got.Draft.Response.ReviewerToolEvidence, want) { + t.Fatalf("cached = %t, evidence %#v, err %v", got.Cached, got.Draft.Response.ReviewerToolEvidence, err) + } + if len(cachedAdapter.Requests()) != 0 || len(cachedAdapter.Resumes()) != 0 { + t.Fatal("durable reuse called the provider") + } + }) + } + } +} + +type toolEvidenceWaitErrorAdapter struct{ llm.Adapter } + +func (adapter toolEvidenceWaitErrorAdapter) Start(ctx context.Context, req llm.Request) (llm.Stream, error) { + stream, err := adapter.Adapter.Start(ctx, req) + if err != nil { + return nil, err + } + return toolEvidenceWaitErrorStream{Stream: stream}, nil +} + +type toolEvidenceWaitErrorStream struct{ llm.Stream } + +func (stream toolEvidenceWaitErrorStream) Wait(ctx context.Context) (llm.Response, error) { + response, err := stream.Stream.Wait(ctx) + if err != nil { + return response, err + } + return response, errors.New("provider stopped after tool execution") +} + +func TestToolTraceSchemaRejectsPreEvidenceCacheBeforeProviderCall(t *testing.T) { + for _, oldSchema := range []int{1, 2, 3} { + t.Run(fmt.Sprint(oldSchema), func(t *testing.T) { + adapter := &llm.FakeAdapter{NameValue: "fake-llm", SupportsResumeValue: true} + req := lifecycleRequest(t, newLifecycleStore(), adapter) + meta := BaseMetadata(req, SessionDraft{ProviderSessionID: "old-session"}) + meta.SchemaVersion = oldSchema + meta.Status = StatusFailedBlocking + if err := WriteMetadata(req.Paths, meta); err != nil { + t.Fatal(err) + } + _, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err == nil || !strings.Contains(err.Error(), "schema version") || !strings.Contains(err.Error(), "want 4") { + t.Fatalf("error = %v, want schema rejection", err) + } + if len(adapter.Requests()) != 0 || len(adapter.Resumes()) != 0 { + t.Fatal("old schema started or resumed a provider session") + } + }) + } +} + +func TestToolTraceOnResumeBelongsOnlyToCurrentInvocation(t *testing.T) { + adapter := &llm.FakeAdapter{NameValue: "fake-llm", SupportsResumeValue: true} + req := lifecycleRequest(t, newLifecycleStore(), adapter) + prior := lifecycleToolEvidence() + prior.Trace.Calls[0].Path = "prior.go" + meta := BaseMetadata(req, SessionDraft{ProviderSessionID: "prior-session", Response: llm.Response{ReviewerToolEvidence: prior}}) + meta.Status = StatusFailedBlocking + if err := WriteMetadata(req.Paths, meta); err != nil { + t.Fatal(err) + } + want := lifecycleToolEvidence() + adapter.Queue(llm.FakeResult{SessionID: "resumed-session", Response: llm.Response{StructuredOutput: []byte(`{"ok":true}`), ReviewerToolEvidence: want}}) + got, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err != nil || !reflect.DeepEqual(got.Draft.Response.ReviewerToolEvidence, want) { + t.Fatalf("resumed evidence = %#v, err %v", got.Draft.Response.ReviewerToolEvidence, err) + } + if resumes := adapter.Resumes(); len(resumes) != 1 || resumes[0].SessionID != "prior-session" { + t.Fatalf("resumes = %#v, want prior session", resumes) + } + restored, exists, err := ReadMetadata(req.Paths, req.TaskID) + if err != nil || !exists || !reflect.DeepEqual(restored.ReviewerToolEvidence, want) { + t.Fatalf("resumed metadata = %#v, exists %t err %v", restored.ReviewerToolEvidence, exists, err) + } +} + +func TestToolTraceMetadataSnapshotsDoNotShareMutableState(t *testing.T) { + evidence := lifecycleToolEvidence() + meta := BaseMetadata(lifecycleRequest(t, newLifecycleStore(), &llm.FakeAdapter{}), SessionDraft{Response: llm.Response{ReviewerToolEvidence: evidence}}) + draft := SessionDraftFromMetadata(meta) + draft.Response.ReviewerToolEvidence.Trace.Calls[0].Range.Offset = 99 + evidence.Trace.Calls[0].Output.Bytes = 99 + if !reflect.DeepEqual(meta.ReviewerToolEvidence, lifecycleToolEvidence()) { + t.Fatal("metadata shares mutable trace state with a response or restored draft") + } +} + +func TestUnsupportedToolTraceRemainsAbsentInMetadata(t *testing.T) { + req := lifecycleRequest(t, newLifecycleStore(), &llm.FakeAdapter{}) + meta := BaseMetadata(req, SessionDraft{}) + if err := WriteMetadata(req.Paths, meta); err != nil { + t.Fatal(err) + } + got, exists, err := ReadMetadata(req.Paths, req.TaskID) + if err != nil || !exists || got.ReviewerToolEvidence != nil || SessionDraftFromMetadata(got).Response.ReviewerToolEvidence != nil { + t.Fatalf("unsupported evidence = %#v, exists %t err %v", got.ReviewerToolEvidence, exists, err) + } +} + +func lifecycleToolEvidence() *llm.ReviewerToolEvidence { + return &llm.ReviewerToolEvidence{DiffStatus: llm.DiffToolStatusSucceeded, Trace: &llm.ReviewerToolTrace{ + Version: 1, Source: "pi_rpc", StreamComplete: false, Truncated: true, DroppedEvents: 7, + Calls: []llm.ReviewerToolCallEvidence{{ + CallID: "read-1", Tool: "cr_read", Path: "src/file.go", ReadView: llm.ReviewerToolReadFile, Status: llm.ReviewerToolCallSucceeded, + StartObserved: true, EndObserved: true, + Range: &llm.ReviewerToolRange{Offset: 12, Limit: 30}, Output: &llm.ReviewerToolOutput{Bytes: 30, Truncated: true}, + }, { + CallID: "read-2", Tool: "cr_read", Path: "context.go", ReadView: llm.ReviewerToolReadSymlink, Status: llm.ReviewerToolCallIncomplete, + StartObserved: true, ProvenanceIssue: "missing_end", Range: &llm.ReviewerToolRange{}, + }}, + }} +} diff --git a/internal/modelcatalog/catalog.go b/internal/modelcatalog/catalog.go index f530837d..07b61588 100644 --- a/internal/modelcatalog/catalog.go +++ b/internal/modelcatalog/catalog.go @@ -28,8 +28,9 @@ import ( ) const ( - // SchemaVersion is the on-disk catalog schema understood by this binary. - SchemaVersion = 1 + // SchemaVersion is the latest schema understood by this binary. + // Schema 1 remains readable with its original exact pricing header. + SchemaVersion = 2 // CatalogDirName is the per-user directory holding installed snapshots. CatalogDirName = "catalog" currentPointer = "catalog.current" @@ -101,23 +102,25 @@ type Default struct { // Price is a verified rate per one million tokens. Nil fields mean unknown, // never zero. type Price struct { - ModelID string `json:"model_id"` - Speed string `json:"speed"` - ContextBand string `json:"context_band"` - Input *float64 `json:"input_usd_per_million,omitempty"` - Output *float64 `json:"output_usd_per_million,omitempty"` - CacheRead *float64 `json:"cache_read_usd_per_million,omitempty"` - CacheWrite *float64 `json:"cache_write_usd_per_million,omitempty"` - CacheWrite5m *float64 `json:"cache_write_5m_usd_per_million,omitempty"` - CacheWrite1h *float64 `json:"cache_write_1h_usd_per_million,omitempty"` - SourceURL string `json:"source_url"` - VerifiedDate string `json:"verified_date"` + ModelID string `json:"model_id"` + Speed string `json:"speed"` + ContextBand string `json:"context_band"` + Input *float64 `json:"input_usd_per_million,omitempty"` + Output *float64 `json:"output_usd_per_million,omitempty"` + CacheRead *float64 `json:"cache_read_usd_per_million,omitempty"` + CacheWrite *float64 `json:"cache_write_usd_per_million,omitempty"` + CacheWrite5m *float64 `json:"cache_write_5m_usd_per_million,omitempty"` + CacheWrite1h *float64 `json:"cache_write_1h_usd_per_million,omitempty"` + SourceURL string `json:"source_url"` + VerifiedDate string `json:"verified_date"` + RequestPricing *RequestPricingBinding `json:"request_pricing,omitempty"` } // Catalog is an immutable validated snapshot. Its slices are private so a // command cannot accidentally mutate the snapshot used by another stage. type Catalog struct { manifest Manifest + digest string source Source runtimes []Runtime models []Model @@ -152,6 +155,15 @@ func (c *Catalog) Source() Source { // Revision returns the snapshot revision. func (c *Catalog) Revision() string { return c.Manifest().Revision } +// Digest returns the full SHA-256 of the exact five snapshot files, including +// the manifest, independent of the revision and declared checksums. +func (c *Catalog) Digest() string { + if c == nil { + return "" + } + return c.digest +} + // Runtimes returns a copy in stable order. func (c *Catalog) Runtimes() []Runtime { return append([]Runtime(nil), c.runtimes...) } @@ -218,10 +230,11 @@ func (c *Catalog) DefaultFor(runtimeID, tier string) (Default, bool) { return Default{}, false } -// PriceFor returns a price row for a concrete model and observed speed. +// PriceFor returns a legacy context-independent row for a model and speed. +// Request-bound rows require PriceForRequest, even for the all context band. func (c *Catalog) PriceFor(modelID, speed string) (Price, bool) { for _, price := range c.pricing { - if price.ModelID == modelID && price.Speed == speed && price.ContextBand == "all" { + if price.ModelID == modelID && price.Speed == speed && price.ContextBand == "all" && price.RequestPricing == nil { return clonePrice(price), true } } @@ -549,7 +562,7 @@ func loadFiles(files map[string][]byte, source Source) (*Catalog, error) { if err := json.Unmarshal(manifestBytes, &manifest); err != nil { return nil, fmt.Errorf("model catalog: parse manifest: %w", err) } - if manifest.SchemaVersion != SchemaVersion { + if manifest.SchemaVersion != 1 && manifest.SchemaVersion != SchemaVersion { return nil, fmt.Errorf("model catalog: schema version %d is incompatible with %d", manifest.SchemaVersion, SchemaVersion) } manifest.Revision = strings.TrimSpace(manifest.Revision) @@ -586,7 +599,7 @@ func loadFiles(files map[string][]byte, source Source) (*Catalog, error) { if err != nil { return nil, err } - pricing, err := parsePricing(fsys, ".") + pricing, err := parsePricing(fsys, ".", manifest.SchemaVersion) if err != nil { return nil, err } @@ -595,7 +608,7 @@ func loadFiles(files map[string][]byte, source Source) (*Catalog, error) { } source.Revision = manifest.Revision manifest.Sources = append([]Source(nil), manifest.Sources...) - return &Catalog{manifest: manifest, source: source, runtimes: runtimes, models: models, defaults: defaults, pricing: pricing}, nil + return &Catalog{manifest: manifest, digest: fmt.Sprintf("%x", snapshotDigest(files)), source: source, runtimes: runtimes, models: models, defaults: defaults, pricing: pricing}, nil } type fsLike interface{ ReadFile(string) ([]byte, error) } @@ -618,12 +631,19 @@ func (f osFS) ReadFile(name string) ([]byte, error) { } func readCSV(fsys fsLike, root, name string, want []string) ([]map[string]string, error) { + return readCSVWithHeaderPolicy(fsys, root, name, want, false) +} + +// Schema-1 catalogs retain their historical header handling. Schema-2 pricing +// headers must match exactly: accepting a padded name while indexing rows by +// that name could erase a populated request binding or its optional maximum. +func readCSVWithHeaderPolicy(fsys fsLike, root, name string, want []string, exactHeaders bool) ([]map[string]string, error) { body, err := fsys.ReadFile(path.Join(root, name)) if err != nil { return nil, fmt.Errorf("model catalog: read %s: %w", name, err) } reader := csv.NewReader(strings.NewReader(string(body))) - reader.TrimLeadingSpace = true + reader.TrimLeadingSpace = !exactHeaders headers, err := reader.Read() if err != nil { return nil, fmt.Errorf("model catalog: read %s header: %w", name, err) @@ -632,10 +652,16 @@ func readCSV(fsys fsLike, root, name string, want []string) ([]map[string]string return nil, fmt.Errorf("model catalog: %s header has %d fields, want %d", name, len(headers), len(want)) } for i := range headers { - if strings.TrimSpace(headers[i]) != want[i] { + header := headers[i] + if !exactHeaders { + header = strings.TrimSpace(header) + } + if header != want[i] { return nil, fmt.Errorf("model catalog: %s header field %d is %q, want %q", name, i, headers[i], want[i]) } } + // Header strictness does not change established whitespace handling of cells. + reader.TrimLeadingSpace = true rows := make([]map[string]string, 0) for line := 2; ; line++ { values, err := reader.Read() @@ -798,8 +824,12 @@ func parseDefaults(fsys fsLike, root string) ([]Default, error) { return out, nil } -func parsePricing(fsys fsLike, root string) ([]Price, error) { - rows, err := readCSV(fsys, root, "pricing.csv", []string{"model_id", "speed", "context_band", "input_usd_per_million", "output_usd_per_million", "cache_read_usd_per_million", "cache_write_usd_per_million", "cache_write_5m_usd_per_million", "cache_write_1h_usd_per_million", "source_url", "verified_date"}) +func parsePricing(fsys fsLike, root string, schemaVersion int) ([]Price, error) { + headers := []string{"model_id", "speed", "context_band", "input_usd_per_million", "output_usd_per_million", "cache_read_usd_per_million", "cache_write_usd_per_million", "cache_write_5m_usd_per_million", "cache_write_1h_usd_per_million", "source_url", "verified_date"} + if schemaVersion == 2 { + headers = append(headers, requestPricingColumns...) + } + rows, err := readCSVWithHeaderPolicy(fsys, root, "pricing.csv", headers, schemaVersion == 2) if err != nil { return nil, err } @@ -836,6 +866,12 @@ func parsePricing(fsys fsLike, root string) ([]Price, error) { if value.Input == nil && value.Output == nil && value.CacheRead == nil && value.CacheWrite == nil && value.CacheWrite5m == nil && value.CacheWrite1h == nil { return nil, fmt.Errorf("model catalog: price %q has no known rates", key) } + if schemaVersion == 2 { + value.RequestPricing, err = parseRequestPricing(row, value.Speed) + if err != nil { + return nil, fmt.Errorf("model catalog: price %q: %w", key, err) + } + } out = append(out, value) } sort.Slice(out, func(i, j int) bool { @@ -906,7 +942,7 @@ func validateRelations(runtimes []Runtime, models []Model, defaults []Default, p return fmt.Errorf("model catalog: price references unknown model %q", value.ModelID) } } - return nil + return validateRequestPricing(runtimesByID, modelsByRuntime, pricing) } func contains(values []string, want string) bool { @@ -926,6 +962,7 @@ func clonePrice(value Price) Price { clone.CacheWrite = cloneRate(value.CacheWrite) clone.CacheWrite5m = cloneRate(value.CacheWrite5m) clone.CacheWrite1h = cloneRate(value.CacheWrite1h) + clone.RequestPricing = cloneRequestPricing(value.RequestPricing) return clone } diff --git a/internal/modelcatalog/data/manifest.json b/internal/modelcatalog/data/manifest.json index 05a29cea..5090950a 100644 --- a/internal/modelcatalog/data/manifest.json +++ b/internal/modelcatalog/data/manifest.json @@ -1,19 +1,14 @@ { - "schema_version": 1, - "revision": "baseline-2026-10-02", - "generated_at": "2026-10-02", + "schema_version": 2, + "revision": "baseline-request-bindings-2026-10-08", + "generated_at": "2026-10-08", "files": { "runtimes.csv": "c8259b7dd4ea3a9ab149656c74a3e37fa58874ecb57d75a59ff4b8642a32ff0d", "models.csv": "cc80619402a47d7338b8c70ffa190001a2247aedc3da00e76aeb9d2c31e6c2d8", "defaults.csv": "7d7ee13a060a70130597b35708d13baddd3f6576cf7aa7f986d7532918734531", - "pricing.csv": "236305213c9876472c4efad072e9c3bb07cbfe20276dd70b65d6fd9729131447" + "pricing.csv": "5a606456ff2449c79b97d8787893fa517de712e8aa2f221f159fa133378a1776" }, "sources": [ - { - "kind": "official", - "path": "https://developers.openai.com/api/docs/models/gpt-6.1-sol", - "revision": "gpt-6.1-sol-2026-10-02" - }, { "kind": "official", "path": "https://platform.claude.com/docs/en/models/sonnet-5-5/overview", @@ -23,6 +18,61 @@ "kind": "harness", "path": "https://learn.chatgpt.com/docs/agent-configuration/speed", "revision": "codex-speed-2026-10-02" + }, + { + "kind": "official", + "path": "https://developers.openai.com/api/docs/models/gpt-5.6-luna", + "revision": "gpt-5.6-luna-whole-request-2026-10-08" + }, + { + "kind": "official", + "path": "https://developers.openai.com/api/docs/models/gpt-5.6-sol", + "revision": "gpt-5.6-sol-whole-request-2026-10-08" + }, + { + "kind": "official", + "path": "https://developers.openai.com/api/docs/models/gpt-5.6-terra", + "revision": "gpt-5.6-terra-whole-request-2026-10-08" + }, + { + "kind": "official", + "path": "https://developers.openai.com/api/docs/models/gpt-6-astra", + "revision": "gpt-6-astra-whole-request-2026-10-08" + }, + { + "kind": "official", + "path": "https://developers.openai.com/api/docs/models/gpt-6-luna", + "revision": "gpt-6-luna-whole-request-2026-10-08" + }, + { + "kind": "official", + "path": "https://developers.openai.com/api/docs/models/gpt-6-sol", + "revision": "gpt-6-sol-whole-request-2026-10-08" + }, + { + "kind": "official", + "path": "https://developers.openai.com/api/docs/models/gpt-6.1-sol", + "revision": "gpt-6.1-sol-whole-request-2026-10-08" + }, + { + "kind": "official", + "path": "https://developers.openai.com/api/docs/pricing", + "revision": "absolute-rates-2026-10-08" + }, + { + "kind": "official", + "path": "https://developers.openai.com/api/docs/guides/fast-mode", + "revision": "observed-fast-priority-tiers-2026-10-08" + }, + { + "kind": "official", + "path": "https://developers.openai.com/api/docs/guides/ultrafast-mode", + "revision": "astra-ultrafast-tier-2026-10-08" + }, + { + "kind": "official", + "path": "https://developers.openai.com/api/docs/guides/prompt-caching", + "revision": "inclusive-input-cache-categories-2026-10-08" } ] } diff --git a/internal/modelcatalog/data/pricing.csv b/internal/modelcatalog/data/pricing.csv index d472cc5a..50098ca3 100644 --- a/internal/modelcatalog/data/pricing.csv +++ b/internal/modelcatalog/data/pricing.csv @@ -1,41 +1,41 @@ -model_id,speed,context_band,input_usd_per_million,output_usd_per_million,cache_read_usd_per_million,cache_write_usd_per_million,cache_write_5m_usd_per_million,cache_write_1h_usd_per_million,source_url,verified_date -claude-fable-5-1,standard,all,10,50,0.25,,12.5,20,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 -claude-opus-4-8,standard,all,5,25,0.5,,6.25,10,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 -claude-opus-4-8,fast,all,10,50,1,,12.5,20,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 -claude-opus-5,standard,all,5,25,0.5,,6.25,10,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 -claude-opus-5,fast,all,10,50,1,,12.5,20,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 -claude-opus-5-5,standard,all,4,20,0.2,,5,8,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 -claude-opus-5-5,fast,all,8,40,0.4,,10,16,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 -claude-sonnet-4-6,standard,all,3,15,0.3,,3.75,6,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 -claude-sonnet-5,standard,all,2,10,0.2,,2.5,4,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 -claude-sonnet-5-5,standard,all,2,10,0.2,,2.5,4,https://platform.claude.com/docs/en/models/sonnet-5-5/overview,2026-10-02 -claude-haiku-4-5,standard,all,1,5,0.1,,1.25,2,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 -claude-haiku-4-5-20251001,standard,all,1,5,0.1,,1.25,2,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 -gpt-6-astra,standard,short,10,50,1,12.5,,,https://developers.openai.com/api/docs/pricing,2026-10-02 -gpt-6-astra,standard,long,20,75,2,25,,,https://developers.openai.com/api/docs/pricing,2026-10-02 -gpt-6-astra,fast,short,20,100,2,25,,,https://developers.openai.com/api/docs/pricing,2026-10-02 -gpt-6-astra,fast,long,40,150,4,50,,,https://developers.openai.com/api/docs/pricing,2026-10-02 -gpt-6-astra,ultrafast,short,60,300,6,75,,,https://developers.openai.com/api/docs/pricing,2026-10-02 -gpt-6-astra,ultrafast,long,120,450,12,150,,,https://developers.openai.com/api/docs/pricing,2026-10-02 -gpt-6-luna,standard,short,0.1,0.5,0.01,0.125,,,https://developers.openai.com/api/docs/pricing,2026-10-02 -gpt-6-luna,standard,long,0.2,0.75,0.02,0.25,,,https://developers.openai.com/api/docs/pricing,2026-10-02 -gpt-6-luna,fast,short,0.2,1,0.02,0.25,,,https://developers.openai.com/api/docs/pricing,2026-10-02 -gpt-6-luna,fast,long,0.4,1.5,0.04,0.5,,,https://developers.openai.com/api/docs/pricing,2026-10-02 -gpt-6.1-sol,standard,short,2,10,0.1,2.5,,,https://developers.openai.com/api/docs/pricing,2026-10-02 -gpt-6.1-sol,standard,long,4,15,0.2,5,,,https://developers.openai.com/api/docs/pricing,2026-10-02 -gpt-6.1-sol,fast,short,4,20,0.2,5,,,https://developers.openai.com/api/docs/pricing,2026-10-02 -gpt-6.1-sol,fast,long,8,30,0.4,10,,,https://developers.openai.com/api/docs/pricing,2026-10-02 -gpt-6-sol,standard,short,2,10,0.2,2.5,,,https://developers.openai.com/api/docs/models/gpt-6-sol,2026-10-02 -gpt-6-sol,standard,long,4,15,0.4,5,,,https://developers.openai.com/api/docs/models/gpt-6-sol,2026-10-02 -gpt-6-sol,fast,short,4,20,0.4,5,,,https://developers.openai.com/api/docs/pricing,2026-10-02 -gpt-6-sol,fast,long,8,30,0.8,10,,,https://developers.openai.com/api/docs/pricing,2026-10-02 -gpt-5.6-sol,standard,short,4,20,0.4,5,,,https://developers.openai.com/api/docs/models/gpt-5.6-sol,2026-10-02 -gpt-5.6-sol,standard,long,8,30,0.8,10,,,https://developers.openai.com/api/docs/models/gpt-5.6-sol,2026-10-02 -gpt-5.6-terra,standard,short,2,12,0.2,2.5,,,https://developers.openai.com/api/docs/models/gpt-5.6-terra,2026-10-02 -gpt-5.6-terra,standard,long,4,18,0.4,5,,,https://developers.openai.com/api/docs/models/gpt-5.6-terra,2026-10-02 -gpt-5.6-luna,standard,short,0.2,1.2,0.02,0.25,,,https://developers.openai.com/api/docs/models/gpt-5.6-luna,2026-10-02 -gpt-5.6-luna,standard,long,0.4,1.8,0.04,0.5,,,https://developers.openai.com/api/docs/models/gpt-5.6-luna,2026-10-02 -gpt-5.4,standard,short,2.5,15,0.25,,,,https://developers.openai.com/api/docs/models/gpt-5.4,2026-10-02 -gpt-5.4,standard,long,5,22.5,0.5,,,,https://developers.openai.com/api/docs/models/gpt-5.4,2026-10-02 -gpt-5.5,standard,short,5,30,0.5,,,,https://developers.openai.com/api/docs/models/gpt-5.5,2026-10-02 -gpt-5.5,standard,long,10,45,1,,,,https://developers.openai.com/api/docs/models/gpt-5.5,2026-10-02 +model_id,speed,context_band,input_usd_per_million,output_usd_per_million,cache_read_usd_per_million,cache_write_usd_per_million,cache_write_5m_usd_per_million,cache_write_1h_usd_per_million,source_url,verified_date,runtime_id,observed_service_tiers,input_tokens_min_inclusive,input_tokens_max_exclusive,rate_application +claude-fable-5-1,standard,all,10,50,0.25,,12.5,20,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02,,,,, +claude-opus-4-8,standard,all,5,25,0.5,,6.25,10,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02,,,,, +claude-opus-4-8,fast,all,10,50,1,,12.5,20,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02,,,,, +claude-opus-5,standard,all,5,25,0.5,,6.25,10,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02,,,,, +claude-opus-5,fast,all,10,50,1,,12.5,20,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02,,,,, +claude-opus-5-5,standard,all,4,20,0.2,,5,8,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02,,,,, +claude-opus-5-5,fast,all,8,40,0.4,,10,16,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02,,,,, +claude-sonnet-4-6,standard,all,3,15,0.3,,3.75,6,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02,,,,, +claude-sonnet-5,standard,all,2,10,0.2,,2.5,4,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02,,,,, +claude-sonnet-5-5,standard,all,2,10,0.2,,2.5,4,https://platform.claude.com/docs/en/models/sonnet-5-5/overview,2026-10-02,,,,, +claude-haiku-4-5,standard,all,1,5,0.1,,1.25,2,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02,,,,, +claude-haiku-4-5-20251001,standard,all,1,5,0.1,,1.25,2,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02,,,,, +gpt-6-astra,standard,short,10,50,1,12.5,,,https://developers.openai.com/api/docs/models/gpt-6-astra,2026-10-08,openai-api-key,default,0,272001,whole_request +gpt-6-astra,standard,long,20,75,2,25,,,https://developers.openai.com/api/docs/models/gpt-6-astra,2026-10-08,openai-api-key,default,272001,,whole_request +gpt-6-astra,fast,short,20,100,2,25,,,https://developers.openai.com/api/docs/models/gpt-6-astra,2026-10-08,openai-api-key,fast|priority,0,272001,whole_request +gpt-6-astra,fast,long,40,150,4,50,,,https://developers.openai.com/api/docs/models/gpt-6-astra,2026-10-08,openai-api-key,fast|priority,272001,,whole_request +gpt-6-astra,ultrafast,short,60,300,6,75,,,https://developers.openai.com/api/docs/models/gpt-6-astra,2026-10-08,openai-api-key,ultrafast,0,272001,whole_request +gpt-6-astra,ultrafast,long,120,450,12,150,,,https://developers.openai.com/api/docs/models/gpt-6-astra,2026-10-08,openai-api-key,ultrafast,272001,,whole_request +gpt-6-luna,standard,short,0.1,0.5,0.01,0.125,,,https://developers.openai.com/api/docs/models/gpt-6-luna,2026-10-08,openai-api-key,default,0,272001,whole_request +gpt-6-luna,standard,long,0.2,0.75,0.02,0.25,,,https://developers.openai.com/api/docs/models/gpt-6-luna,2026-10-08,openai-api-key,default,272001,,whole_request +gpt-6-luna,fast,short,0.2,1,0.02,0.25,,,https://developers.openai.com/api/docs/models/gpt-6-luna,2026-10-08,openai-api-key,fast|priority,0,272001,whole_request +gpt-6-luna,fast,long,0.4,1.5,0.04,0.5,,,https://developers.openai.com/api/docs/models/gpt-6-luna,2026-10-08,openai-api-key,fast|priority,272001,,whole_request +gpt-6.1-sol,standard,short,2,10,0.1,2.5,,,https://developers.openai.com/api/docs/models/gpt-6.1-sol,2026-10-08,openai-api-key,default,0,272001,whole_request +gpt-6.1-sol,standard,long,4,15,0.2,5,,,https://developers.openai.com/api/docs/models/gpt-6.1-sol,2026-10-08,openai-api-key,default,272001,,whole_request +gpt-6.1-sol,fast,short,4,20,0.2,5,,,https://developers.openai.com/api/docs/models/gpt-6.1-sol,2026-10-08,openai-api-key,fast|priority,0,272001,whole_request +gpt-6.1-sol,fast,long,8,30,0.4,10,,,https://developers.openai.com/api/docs/models/gpt-6.1-sol,2026-10-08,openai-api-key,fast|priority,272001,,whole_request +gpt-6-sol,standard,short,2,10,0.2,2.5,,,https://developers.openai.com/api/docs/models/gpt-6-sol,2026-10-08,openai-api-key,default,0,272001,whole_request +gpt-6-sol,standard,long,4,15,0.4,5,,,https://developers.openai.com/api/docs/models/gpt-6-sol,2026-10-08,openai-api-key,default,272001,,whole_request +gpt-6-sol,fast,short,4,20,0.4,5,,,https://developers.openai.com/api/docs/models/gpt-6-sol,2026-10-08,openai-api-key,fast|priority,0,272001,whole_request +gpt-6-sol,fast,long,8,30,0.8,10,,,https://developers.openai.com/api/docs/models/gpt-6-sol,2026-10-08,openai-api-key,fast|priority,272001,,whole_request +gpt-5.6-sol,standard,short,4,20,0.4,5,,,https://developers.openai.com/api/docs/models/gpt-5.6-sol,2026-10-08,openai-api-key,default,0,272001,whole_request +gpt-5.6-sol,standard,long,8,30,0.8,10,,,https://developers.openai.com/api/docs/models/gpt-5.6-sol,2026-10-08,openai-api-key,default,272001,,whole_request +gpt-5.6-terra,standard,short,2,12,0.2,2.5,,,https://developers.openai.com/api/docs/models/gpt-5.6-terra,2026-10-08,openai-api-key,default,0,272001,whole_request +gpt-5.6-terra,standard,long,4,18,0.4,5,,,https://developers.openai.com/api/docs/models/gpt-5.6-terra,2026-10-08,openai-api-key,default,272001,,whole_request +gpt-5.6-luna,standard,short,0.2,1.2,0.02,0.25,,,https://developers.openai.com/api/docs/models/gpt-5.6-luna,2026-10-08,openai-api-key,default,0,272001,whole_request +gpt-5.6-luna,standard,long,0.4,1.8,0.04,0.5,,,https://developers.openai.com/api/docs/models/gpt-5.6-luna,2026-10-08,openai-api-key,default,272001,,whole_request +gpt-5.4,standard,short,2.5,15,0.25,,,,https://developers.openai.com/api/docs/models/gpt-5.4,2026-10-02,,,,, +gpt-5.4,standard,long,5,22.5,0.5,,,,https://developers.openai.com/api/docs/models/gpt-5.4,2026-10-02,,,,, +gpt-5.5,standard,short,5,30,0.5,,,,https://developers.openai.com/api/docs/models/gpt-5.5,2026-10-02,,,,, +gpt-5.5,standard,long,10,45,1,,,,https://developers.openai.com/api/docs/models/gpt-5.5,2026-10-02,,,,, diff --git a/internal/modelcatalog/request_pricing.go b/internal/modelcatalog/request_pricing.go new file mode 100644 index 00000000..394d1eeb --- /dev/null +++ b/internal/modelcatalog/request_pricing.go @@ -0,0 +1,271 @@ +package modelcatalog + +import ( + "fmt" + "sort" + "strconv" + "strings" +) + +var requestPricingColumns = []string{"runtime_id", "observed_service_tiers", "input_tokens_min_inclusive", "input_tokens_max_exclusive", "rate_application"} + +// RequestPricingBinding describes verified whole-request list-price semantics. +// Nil bindings are legacy/display-only. Within a complete binding, minimum is +// required and a nil maximum means unbounded, not unknown. +type RequestPricingBinding struct { + RuntimeID string `json:"runtime_id"` + ObservedServiceTiers []string `json:"observed_service_tiers"` + InputTokensMinInclusive *int64 `json:"input_tokens_min_inclusive"` + InputTokensMaxExclusive *int64 `json:"input_tokens_max_exclusive"` + RateApplication string `json:"rate_application"` +} + +// UnavailableReason explains why no request price could be selected. +type UnavailableReason string + +// Request pricing unavailability reasons returned by PriceForRequest. +const ( + UnavailableCatalog UnavailableReason = "catalog_unavailable" + UnavailableInput UnavailableReason = "invalid_input_tokens" + UnavailableIdentity UnavailableReason = "missing_observed_identity" + UnavailableRuntime UnavailableReason = "unsupported_runtime" + UnavailablePrice UnavailableReason = "no_matching_request_price" + UnavailableAmbiguousPrice UnavailableReason = "ambiguous_request_price" +) + +// PriceSelection is a detached row and its exact immutable catalog identity. +// It is not an estimate receipt: endpoint scope, request evidence and complete +// dispatch history must be established independently before any cost is used. +type PriceSelection struct { + Price Price `json:"price"` + CatalogSchemaVersion int `json:"catalog_schema_version"` + CatalogRevision string `json:"catalog_revision"` + CatalogDigest string `json:"catalog_digest"` + ObservedServiceTier string `json:"observed_service_tier"` +} + +// PriceForRequest selects one absolute-rate row for inclusive input tokens on +// one observed request. Caller must independently establish official global +// endpoint scope and complete request/history evidence. This function does not +// activate an estimator, validate context-window capacity, or normalize aliases, +// service tiers, token categories, requested speed, or aggregate usage. +func (c *Catalog) PriceForRequest(runtimeID, observedModel, observedServiceTier string, inputTokensTotal int64) (PriceSelection, UnavailableReason) { + if c == nil { + return PriceSelection{}, UnavailableCatalog + } + if inputTokensTotal < 0 { + return PriceSelection{}, UnavailableInput + } + if runtimeID == "" || observedModel == "" || observedServiceTier == "" { + return PriceSelection{}, UnavailableIdentity + } + runtime, ok := c.RuntimeByID(runtimeID) + if !ok || !supportsRequestPricing(runtime) { + return PriceSelection{}, UnavailableRuntime + } + var selected Price + matches := 0 + for _, price := range c.pricing { + binding := price.RequestPricing + if price.ModelID != observedModel || binding == nil || binding.RuntimeID != runtimeID || binding.RateApplication != "whole_request" || !contains(binding.ObservedServiceTiers, observedServiceTier) { + continue + } + if binding.InputTokensMinInclusive == nil || inputTokensTotal < *binding.InputTokensMinInclusive || (binding.InputTokensMaxExclusive != nil && inputTokensTotal >= *binding.InputTokensMaxExclusive) { + continue + } + selected = price + matches++ + } + if matches == 0 { + return PriceSelection{}, UnavailablePrice + } + if matches != 1 { + return PriceSelection{}, UnavailableAmbiguousPrice + } + return PriceSelection{ + Price: clonePrice(selected), CatalogSchemaVersion: c.manifest.SchemaVersion, + CatalogRevision: c.Revision(), CatalogDigest: c.Digest(), ObservedServiceTier: observedServiceTier, + }, "" +} + +func supportsRequestPricing(runtime Runtime) bool { + return runtime.ID == "openai-api-key" && runtime.Provider == "openai" && runtime.Auth == "api_key" && runtime.Harness == "openai_api" && runtime.Adapter == "openai_api" +} + +func absentRequestCell(value string) bool { return value == "" || strings.EqualFold(value, "null") } + +func parseRequestPricing(row map[string]string, speed string) (*RequestPricingBinding, error) { + absent := 0 + for _, column := range requestPricingColumns { + if absentRequestCell(row[column]) { + absent++ + } + } + if absent == len(requestPricingColumns) { + return nil, nil + } + for _, column := range []string{"runtime_id", "observed_service_tiers", "input_tokens_min_inclusive", "rate_application"} { + if absentRequestCell(row[column]) { + return nil, fmt.Errorf("incomplete request pricing: %s is required", column) + } + } + if row["rate_application"] != "whole_request" { + return nil, fmt.Errorf("request rate_application %q is unsupported", row["rate_application"]) + } + minimum, err := parseInputBound(row["input_tokens_min_inclusive"]) + if err != nil { + return nil, fmt.Errorf("input_tokens_min_inclusive: %w", err) + } + binding := &RequestPricingBinding{RuntimeID: row["runtime_id"], InputTokensMinInclusive: &minimum, RateApplication: "whole_request"} + if !absentRequestCell(row["input_tokens_max_exclusive"]) { + maximum, err := parseInputBound(row["input_tokens_max_exclusive"]) + if err != nil { + return nil, fmt.Errorf("input_tokens_max_exclusive: %w", err) + } + if maximum <= minimum { + return nil, fmt.Errorf("input_tokens_max_exclusive must exceed minimum") + } + binding.InputTokensMaxExclusive = &maximum + } + seen := map[string]bool{} + for _, tier := range strings.Split(row["observed_service_tiers"], "|") { + if seen[tier] || !tierMatchesSpeed(tier, speed) { + return nil, fmt.Errorf("observed service tier %q is duplicate or incompatible with speed %q", tier, speed) + } + seen[tier] = true + binding.ObservedServiceTiers = append(binding.ObservedServiceTiers, tier) + } + return binding, nil +} + +func parseInputBound(raw string) (int64, error) { + if raw == "" { + return 0, fmt.Errorf("empty input bound") + } + for _, digit := range raw { + if digit < '0' || digit > '9' { + return 0, fmt.Errorf("input bound %q must contain only decimal digits", raw) + } + } + value, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return 0, fmt.Errorf("input bound %q exceeds int64", raw) + } + return value, nil +} + +func tierMatchesSpeed(tier, speed string) bool { + switch speed { + case "standard": + return tier == "default" + case "fast": + return tier == "fast" || tier == "priority" + case "ultrafast": + return tier == "ultrafast" + default: + return false + } +} + +func validateRequestPricing(runtimes map[string]Runtime, models map[string]map[string]Model, prices []Price) error { + groups := map[string][]Price{} + identities := map[string][]Price{} + for _, price := range prices { + key := price.ModelID + "\x00" + price.Speed + groups[key] = append(groups[key], price) + binding := price.RequestPricing + if binding == nil { + continue + } + runtime, exists := runtimes[binding.RuntimeID] + if !exists || !supportsRequestPricing(runtime) { + return fmt.Errorf("model catalog: price %q has unsupported request runtime %q", key, binding.RuntimeID) + } + if _, exists := models[binding.RuntimeID][price.ModelID]; !exists { + return fmt.Errorf("model catalog: request price %q model is not offered by runtime %q", key, binding.RuntimeID) + } + for _, tier := range binding.ObservedServiceTiers { + identity := binding.RuntimeID + "\x00" + price.ModelID + "\x00" + tier + identities[identity] = append(identities[identity], price) + } + } + for key, group := range groups { + enabled := 0 + for _, price := range group { + if price.RequestPricing != nil { + enabled++ + } + } + if enabled == 0 { + continue + } + if enabled != len(group) { + return fmt.Errorf("model catalog: request price group %q is partially enabled", key) + } + if len(group) == 1 && group[0].ContextBand == "all" { + binding := group[0].RequestPricing + if *binding.InputTokensMinInclusive != 0 || binding.InputTokensMaxExclusive != nil { + return fmt.Errorf("model catalog: request price group %q all must span [0,unbounded)", key) + } + continue + } + if len(group) != 2 { + return fmt.Errorf("model catalog: request price group %q needs one all row or a short/long pair", key) + } + var short, long *RequestPricingBinding + for _, price := range group { + switch price.ContextBand { + case "short": + short = price.RequestPricing + case "long": + long = price.RequestPricing + } + } + if short == nil || long == nil || short.RuntimeID != long.RuntimeID || !sameTierSet(short.ObservedServiceTiers, long.ObservedServiceTiers) || *short.InputTokensMinInclusive != 0 || short.InputTokensMaxExclusive == nil || *short.InputTokensMaxExclusive <= 0 || long.InputTokensMaxExclusive != nil || *long.InputTokensMinInclusive != *short.InputTokensMaxExclusive { + return fmt.Errorf("model catalog: request price group %q needs matching tiers and contiguous [0,B),[B,unbounded) bounds", key) + } + } + for identity, group := range identities { + sort.Slice(group, func(i, j int) bool { + return *group[i].RequestPricing.InputTokensMinInclusive < *group[j].RequestPricing.InputTokensMinInclusive + }) + for i := 1; i < len(group); i++ { + previous := group[i-1].RequestPricing + if previous.InputTokensMaxExclusive == nil || *group[i].RequestPricing.InputTokensMinInclusive < *previous.InputTokensMaxExclusive { + return fmt.Errorf("model catalog: overlapping request price identity %q", identity) + } + } + } + return nil +} + +func sameTierSet(a, b []string) bool { + if len(a) != len(b) { + return false + } + for _, value := range a { + if !contains(b, value) { + return false + } + } + return true +} + +func cloneRequestPricing(value *RequestPricingBinding) *RequestPricingBinding { + if value == nil { + return nil + } + clone := *value + clone.ObservedServiceTiers = append([]string(nil), value.ObservedServiceTiers...) + clone.InputTokensMinInclusive = cloneInputBound(value.InputTokensMinInclusive) + clone.InputTokensMaxExclusive = cloneInputBound(value.InputTokensMaxExclusive) + return &clone +} + +func cloneInputBound(value *int64) *int64 { + if value == nil { + return nil + } + clone := *value + return &clone +} diff --git a/internal/modelcatalog/request_pricing_test.go b/internal/modelcatalog/request_pricing_test.go new file mode 100644 index 00000000..3be5124c --- /dev/null +++ b/internal/modelcatalog/request_pricing_test.go @@ -0,0 +1,636 @@ +package modelcatalog + +import ( + "bytes" + "context" + "encoding/csv" + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func requestCatalogFiles(t *testing.T) map[string][]byte { + t.Helper() + files := map[string][]byte{} + for _, name := range requiredFiles { + body, err := bundledFS.ReadFile("data/" + name) + if err != nil { + t.Fatal(err) + } + files[name] = body + } + return files +} + +func pricingRecords(t *testing.T, files map[string][]byte) [][]string { + t.Helper() + rows, err := csv.NewReader(bytes.NewReader(files["pricing.csv"])).ReadAll() + if err != nil { + t.Fatal(err) + } + return rows +} + +func replacePricingRecords(t *testing.T, files map[string][]byte, rows [][]string) { + t.Helper() + var body bytes.Buffer + writer := csv.NewWriter(&body) + if err := writer.WriteAll(rows); err != nil { + t.Fatal(err) + } + files["pricing.csv"] = body.Bytes() + manifest := readManifestBytes(t, files["manifest.json"]) + if manifest.Files != nil { + manifest.Files["pricing.csv"] = digest(files["pricing.csv"]) + } + files["manifest.json"] = marshalManifest(t, manifest) +} + +func setPricingCell(t *testing.T, rows [][]string, model, speed, band, column, value string) { + t.Helper() + index := -1 + for i, name := range rows[0] { + if name == column { + index = i + } + } + if index < 0 { + t.Fatalf("missing column %q", column) + } + for _, row := range rows[1:] { + if row[0] == model && row[1] == speed && row[2] == band { + row[index] = value + return + } + } + t.Fatalf("missing row %s/%s/%s", model, speed, band) +} + +func requireRequestCatalog(t *testing.T, files map[string][]byte) *Catalog { + t.Helper() + catalog, err := loadFiles(files, Source{Kind: "local"}) + if err != nil { + t.Fatal(err) + } + return catalog +} + +func TestRequestPricingSchemaCompatibility(t *testing.T) { + v1 := catalogFiles(t, filepath.Join("testdata", "v1")) + legacy := requireRequestCatalog(t, v1) + if legacy.Digest() != "7e4512bcc46d396e800e8f71d4ef3ba5ca97108411b3ad36d6b920ef8af3f162" { + t.Fatalf("unchanged v1 digest = %s", legacy.Digest()) + } + for _, price := range legacy.Pricing() { + if price.RequestPricing != nil { + t.Fatal("unchanged v1 snapshot produced a request binding") + } + } + if _, reason := legacy.PriceForRequest("openai-api-key", "gpt-6.1-sol", "default", 1); reason != UnavailablePrice { + t.Fatalf("v1 request selection = %q", reason) + } + if price, ok := legacy.PriceFor("claude-sonnet-4-6", "standard"); !ok || *price.Input != 3 { + t.Fatal("legacy Claude row changed") + } + for _, tc := range []struct { + name string + files map[string][]byte + version int + }{ + {"v1 header with v2", v1, 2}, {"v2 header with v1", requestCatalogFiles(t), 1}, + {"future", requestCatalogFiles(t), 3}, {"zero", requestCatalogFiles(t), 0}, + } { + t.Run(tc.name, func(t *testing.T) { + files := copyBytes(tc.files) + manifest := readManifestBytes(t, files["manifest.json"]) + manifest.SchemaVersion = tc.version + files["manifest.json"] = marshalManifest(t, manifest) + if _, err := loadFiles(files, Source{}); err == nil { + t.Fatal("accepted mismatched schema/header") + } + }) + } +} + +func TestBundledRequestPricingScopeAndThresholdEdges(t *testing.T) { + catalog := requireRequestCatalog(t, requestCatalogFiles(t)) + wantRows := map[string]int{"gpt-6.1-sol": 4, "gpt-6-astra": 6, "gpt-6-sol": 4, "gpt-6-luna": 4, "gpt-5.6-sol": 2, "gpt-5.6-terra": 2, "gpt-5.6-luna": 2} + gotRows := map[string]int{} + for _, price := range catalog.Pricing() { + if price.RequestPricing == nil { + if wantRows[price.ModelID] != 0 { + t.Fatalf("missing reviewed binding for %#v", price) + } + continue + } + gotRows[price.ModelID]++ + if price.ContextBand != "short" { + continue + } + for _, tier := range price.RequestPricing.ObservedServiceTiers { + for _, input := range []int64{0, 271999, 272000, 272001, math.MaxInt64} { + selection, reason := catalog.PriceForRequest("openai-api-key", price.ModelID, tier, input) + band := "short" + if input >= 272001 { + band = "long" + } + var expected Price + for _, row := range catalog.PricesFor(price.ModelID, price.Speed) { + if row.ContextBand == band { + expected = row + } + } + if reason != "" || !reflect.DeepEqual(selection.Price, expected) || selection.ObservedServiceTier != tier { + t.Fatalf("%s/%s/%s input=%d got %#v, %q; want literal %s row", price.ModelID, price.Speed, tier, input, selection, reason, band) + } + if selection.CatalogDigest != catalog.Digest() || selection.CatalogRevision != catalog.Revision() || selection.CatalogSchemaVersion != 2 { + t.Fatal("selection lost catalog identity") + } + } + } + } + if !reflect.DeepEqual(gotRows, wantRows) { + t.Fatalf("bound rows = %#v, want %#v", gotRows, wantRows) + } + // The data migration may add provenance but must not change any absolute rates. + legacy := requireRequestCatalog(t, catalogFiles(t, filepath.Join("testdata", "v1"))) + old := legacy.Pricing() + if len(catalog.Pricing()) != len(old) { + t.Fatal("data migration changed row count") + } + for i, current := range catalog.Pricing() { + current.RequestPricing = nil + current.SourceURL, current.VerifiedDate = old[i].SourceURL, old[i].VerifiedDate + if !reflect.DeepEqual(current, old[i]) { + t.Fatalf("migration changed rate identity or rates: %#v vs %#v", current, old[i]) + } + } +} + +func TestPriceForRequestRejectsUnverifiedIdentity(t *testing.T) { + catalog := requireRequestCatalog(t, requestCatalogFiles(t)) + if _, reason := (*Catalog)(nil).PriceForRequest("openai-api-key", "gpt-6.1-sol", "default", 1); reason != UnavailableCatalog { + t.Fatalf("nil catalog reason = %q", reason) + } + for _, tc := range []struct { + runtime, model, tier string + input int64 + }{ + {"openai-api-key", "gpt-6.1-sol", "default", -1}, {"", "gpt-6.1-sol", "default", 1}, + {"openai-subscription-codex-cli", "gpt-6.1-sol", "default", 1}, {"anthropic-api-key", "gpt-6.1-sol", "default", 1}, + {"openai-api-key", "gpt-6.1-sol-20261008", "default", 1}, {"openai-api-key", "gpt-6.1-sol-custom", "default", 1}, + {"openai-api-key", "", "default", 1}, {"openai-api-key", "gpt-5.4", "default", 1}, + {"openai-api-key", "gpt-5.5", "default", 1}, {"openai-api-key", "gpt-5.6-terra", "fast", 1}, + } { + if _, reason := catalog.PriceForRequest(tc.runtime, tc.model, tc.tier, tc.input); reason == "" { + t.Fatalf("accepted unsupported identity %#v", tc) + } + } + for _, tier := range []string{"", "auto", "standard", "Default", "FAST", "priority ", "mixed", "unknown", "flex", "batch", "scale"} { + if _, reason := catalog.PriceForRequest("openai-api-key", "gpt-6.1-sol", tier, 1); reason == "" { + t.Fatalf("accepted raw tier %q", tier) + } + } + for _, price := range catalog.pricing { + if price.ModelID == "gpt-6.1-sol" && price.Speed == "standard" && price.ContextBand == "short" { + catalog.pricing = append(catalog.pricing, price) + break + } + } + if _, reason := catalog.PriceForRequest("openai-api-key", "gpt-6.1-sol", "default", 1); reason != UnavailableAmbiguousPrice { + t.Fatalf("duplicate reason = %q", reason) + } +} + +func TestRequestPricingRejectsMalformedCells(t *testing.T) { + cases := []struct{ column, value string }{ + {"runtime_id", ""}, {"runtime_id", "missing"}, {"runtime_id", "openai-subscription-codex-cli"}, + {"observed_service_tiers", ""}, {"observed_service_tiers", "default|default"}, {"observed_service_tiers", "default|"}, + {"observed_service_tiers", "|default"}, {"observed_service_tiers", "default||priority"}, {"observed_service_tiers", "default| priority"}, + {"observed_service_tiers", "Default"}, {"observed_service_tiers", "auto"}, {"observed_service_tiers", "standard"}, + {"observed_service_tiers", "priority"}, {"observed_service_tiers", "mixed"}, {"observed_service_tiers", "flex"}, + {"input_tokens_min_inclusive", ""}, {"input_tokens_min_inclusive", "null"}, {"input_tokens_min_inclusive", "1"}, + {"input_tokens_max_exclusive", "0"}, {"input_tokens_max_exclusive", ""}, {"input_tokens_max_exclusive", "null"}, + {"rate_application", ""}, {"rate_application", "null"}, {"rate_application", "marginal"}, + } + for _, column := range []string{"input_tokens_min_inclusive", "input_tokens_max_exclusive"} { + for _, raw := range []string{"-1", "+1", "1.0", "1e5", "NaN", "1/2", "9223372036854775808", "1_000", "0x20", "１"} { + cases = append(cases, struct{ column, value string }{column, raw}) + } + } + for _, tc := range cases { + t.Run(tc.column+"="+tc.value, func(t *testing.T) { + files := requestCatalogFiles(t) + rows := pricingRecords(t, files) + setPricingCell(t, rows, "gpt-6.1-sol", "standard", "short", tc.column, tc.value) + replacePricingRecords(t, files, rows) + if _, err := loadFiles(files, Source{}); err == nil { + t.Fatal("accepted malformed binding") + } + }) + } + for _, tc := range []struct{ speed, tier string }{{"fast", "default"}, {"ultrafast", "fast"}, {"standard", "ultrafast"}} { + files := requestCatalogFiles(t) + rows := pricingRecords(t, files) + setPricingCell(t, rows, "gpt-6-astra", tc.speed, "short", "observed_service_tiers", tc.tier) + replacePricingRecords(t, files, rows) + if _, err := loadFiles(files, Source{}); err == nil { + t.Fatalf("accepted cross-speed tier collision %#v", tc) + } + } +} + +func TestRequestPricingRejectsInvalidGroups(t *testing.T) { + for _, kind := range []string{"gap", "overlap", "missing partner", "partial activation", "different tier set", "mixed all", "reversed", "bounded long", "runtime model relation", "runtime semantics", "duplicate row"} { + t.Run(kind, func(t *testing.T) { + files := requestCatalogFiles(t) + rows := pricingRecords(t, files) + switch kind { + case "gap": + setPricingCell(t, rows, "gpt-6.1-sol", "standard", "long", "input_tokens_min_inclusive", "272002") + case "overlap": + setPricingCell(t, rows, "gpt-6.1-sol", "standard", "long", "input_tokens_min_inclusive", "272000") + case "missing partner": + for i, row := range rows { + if row[0] == "gpt-6.1-sol" && row[1] == "standard" && row[2] == "long" { + rows = append(rows[:i], rows[i+1:]...) + break + } + } + case "partial activation": + for _, column := range requestPricingColumns { + setPricingCell(t, rows, "gpt-6.1-sol", "standard", "long", column, "") + } + case "different tier set": + setPricingCell(t, rows, "gpt-6.1-sol", "fast", "long", "observed_service_tiers", "priority") + case "mixed all": + rows = append(rows, []string{"gpt-6.1-sol", "standard", "all", "2", "10", "0.1", "2.5", "", "", "https://example.invalid", "2026-10-08", "openai-api-key", "default", "0", "", "whole_request"}) + case "reversed": + setPricingCell(t, rows, "gpt-6.1-sol", "standard", "short", "input_tokens_min_inclusive", "272002") + case "bounded long": + setPricingCell(t, rows, "gpt-6.1-sol", "standard", "long", "input_tokens_max_exclusive", "999999") + case "runtime model relation": + files["models.csv"] = []byte(strings.Replace(string(files["models.csv"]), "openai-api-key,gpt-5.6-terra,", "openai-api-key,unpriced-custom-model,", 1)) + case "runtime semantics": + files["runtimes.csv"] = []byte(strings.Replace(string(files["runtimes.csv"]), "openai-api-key,openai,api_key,openai_api,openai_api,", "openai-api-key,openai,api_key,gateway,gateway,", 1)) + case "duplicate row": + rows = append(rows, append([]string(nil), rows[len(rows)-1]...)) + } + replacePricingRecords(t, files, rows) + manifest := readManifestBytes(t, files["manifest.json"]) + manifest.Files["models.csv"] = digest(files["models.csv"]) + manifest.Files["runtimes.csv"] = digest(files["runtimes.csv"]) + files["manifest.json"] = marshalManifest(t, manifest) + if _, err := loadFiles(files, Source{}); err == nil { + t.Fatal("accepted invalid request pricing group") + } + }) + } +} + +func TestRequestPricingCustomBoundariesAndLiteralRates(t *testing.T) { + for _, boundary := range []int64{1, 100001, math.MaxInt64} { + files := requestCatalogFiles(t) + // An explicit custom model must use its own data, regardless of its name. + for _, name := range []string{"models.csv", "defaults.csv", "pricing.csv"} { + files[name] = bytes.ReplaceAll(files[name], []byte("gpt-6.1-sol"), []byte("custom-request-model")) + } + manifest := readManifestBytes(t, files["manifest.json"]) + manifest.Files = nil + files["manifest.json"] = marshalManifest(t, manifest) + rows := pricingRecords(t, files) + setPricingCell(t, rows, "custom-request-model", "standard", "short", "input_tokens_max_exclusive", fmt.Sprint(boundary)) + setPricingCell(t, rows, "custom-request-model", "standard", "long", "input_tokens_min_inclusive", fmt.Sprint(boundary)) + for column, value := range map[string]string{"input_usd_per_million": "7", "cache_read_usd_per_million": "0.7", "cache_write_usd_per_million": "9", "output_usd_per_million": "17"} { + setPricingCell(t, rows, "custom-request-model", "standard", "long", column, value) + } + replacePricingRecords(t, files, rows) + catalog := requireRequestCatalog(t, files) + short, r1 := catalog.PriceForRequest("openai-api-key", "custom-request-model", "default", boundary-1) + long, r2 := catalog.PriceForRequest("openai-api-key", "custom-request-model", "default", boundary) + if r1 != "" || short.Price.ContextBand != "short" || r2 != "" || long.Price.ContextBand != "long" { + t.Fatalf("custom boundary %d not honored", boundary) + } + if *long.Price.Input != 7 || *long.Price.CacheRead != 0.7 || *long.Price.CacheWrite != 9 || *long.Price.Output != 17 { + t.Fatalf("custom rates transformed: %#v", long.Price) + } + } +} + +func TestRequestPricingAllAndBlankBindings(t *testing.T) { + for _, binding := range []string{"blank", "null", "bound"} { + files := requestCatalogFiles(t) + rows := pricingRecords(t, files) + filtered := [][]string{rows[0]} + for _, row := range rows[1:] { + if row[0] != "gpt-6.1-sol" { + filtered = append(filtered, row) + } + } + row := []string{"gpt-6.1-sol", "standard", "all", "7", "17", "0.7", "9", "", "", "https://example.invalid", "2026-10-08", "", "", "", "", ""} + if binding == "null" { + for i := 11; i < len(row); i++ { + row[i] = "null" + } + } + if binding == "bound" { + copy(row[11:], []string{"openai-api-key", "default", "0", "null", "whole_request"}) + } + replacePricingRecords(t, files, append(filtered, row)) + catalog := requireRequestCatalog(t, files) + _, reason := catalog.PriceForRequest("openai-api-key", "gpt-6.1-sol", "default", math.MaxInt64) + _, legacy := catalog.PriceFor("gpt-6.1-sol", "standard") + if (reason == "") != (binding == "bound") || legacy != (binding != "bound") { + t.Fatalf("%s binding request=%q legacy=%t", binding, reason, legacy) + } + if binding == "bound" { + for _, change := range []struct{ column, value string }{{"input_tokens_min_inclusive", "1"}, {"input_tokens_max_exclusive", "100"}} { + bad := copyBytes(files) + badRows := pricingRecords(t, bad) + setPricingCell(t, badRows, "gpt-6.1-sol", "standard", "all", change.column, change.value) + replacePricingRecords(t, bad, badRows) + if _, err := loadFiles(bad, Source{}); err == nil { + t.Fatal("accepted partial all interval") + } + } + } + } +} + +func TestRequestPricingDigestAndDetachedAccessors(t *testing.T) { + files := requestCatalogFiles(t) + catalog := requireRequestCatalog(t, files) + if catalog.Digest() != fmt.Sprintf("%x", snapshotDigest(files)) || len(catalog.Digest()) != 64 || (*Catalog)(nil).Digest() != "" { + t.Fatal("digest does not identify exact five-file snapshot") + } + original, reason := catalog.PriceForRequest("openai-api-key", "gpt-6.1-sol", "priority", 1) + if reason != "" { + t.Fatal(reason) + } + for _, accessor := range []string{"selection", "Pricing", "PricesFor"} { + var price Price + switch accessor { + case "selection": + selection, _ := catalog.PriceForRequest("openai-api-key", "gpt-6.1-sol", "priority", 1) + price = selection.Price + case "Pricing": + for _, candidate := range catalog.Pricing() { + if candidate.ModelID == "gpt-6.1-sol" && candidate.Speed == "fast" && candidate.ContextBand == "short" { + price = candidate + } + } + case "PricesFor": + for _, candidate := range catalog.PricesFor("gpt-6.1-sol", "fast") { + if candidate.ContextBand == "short" { + price = candidate + } + } + } + if price.RequestPricing == nil { + t.Fatalf("%s did not return binding", accessor) + } + price.RequestPricing.ObservedServiceTiers[0] = "mutated" + *price.RequestPricing.InputTokensMinInclusive = 100 + *price.RequestPricing.InputTokensMaxExclusive = 101 + price.RequestPricing.RuntimeID = "mutated" + price.RequestPricing.RateApplication = "mutated" + *price.Input = 999 + selection, reason := catalog.PriceForRequest("openai-api-key", "gpt-6.1-sol", "priority", 1) + if reason != "" || !reflect.DeepEqual(selection, original) { + t.Fatalf("%s leaked mutable data", accessor) + } + } + // Byte identity must survive a caller mutating the source byte map. + files["pricing.csv"][0] = '!' + selected, _ := catalog.PriceForRequest("openai-api-key", "gpt-6.1-sol", "priority", 1) + if !reflect.DeepEqual(selected, original) { + t.Fatal("source bytes mutated loaded selection") + } + + baselineFiles := requestCatalogFiles(t) + manifest := readManifestBytes(t, baselineFiles["manifest.json"]) + manifest.Files = nil // Unchecksummed custom snapshots still need exact identity. + baselineFiles["manifest.json"] = marshalManifest(t, manifest) + baseline := requireRequestCatalog(t, baselineFiles) + for _, change := range []string{"bounds", "tiers", "rates", "manifest.json", "runtimes.csv", "models.csv", "defaults.csv"} { + changed := copyBytes(baselineFiles) + rows := pricingRecords(t, changed) + switch change { + case "bounds": + setPricingCell(t, rows, "gpt-6.1-sol", "fast", "short", "input_tokens_max_exclusive", "100001") + setPricingCell(t, rows, "gpt-6.1-sol", "fast", "long", "input_tokens_min_inclusive", "100001") + case "tiers": + for _, band := range []string{"short", "long"} { + setPricingCell(t, rows, "gpt-6.1-sol", "fast", band, "observed_service_tiers", "priority") + } + case "rates": + setPricingCell(t, rows, "gpt-6.1-sol", "fast", "long", "input_usd_per_million", "7") + } + replacePricingRecords(t, changed, rows) + if strings.Contains(change, ".") { + changed[change] = append(changed[change], '\n') + } + other := requireRequestCatalog(t, changed) + if other.Revision() != baseline.Revision() || other.Digest() == baseline.Digest() { + t.Fatalf("%s did not change digest at the same revision", change) + } + } +} + +func TestRequestPricingChecksumAndInvalidUpdateRetention(t *testing.T) { + source := copyBundledCatalog(t) + target := t.TempDir() + before, err := Update(context.Background(), UpdateOptions{SourceDir: source, InstallDir: target}) + if err != nil { + t.Fatal(err) + } + pointer := activeRevision(t, target) + files := catalogFiles(t, source) + for _, column := range []string{"input_tokens_max_exclusive", "observed_service_tiers"} { + bad := copyBytes(files) + rows := pricingRecords(t, bad) + value := "100001" + if column == "observed_service_tiers" { + value = "fast" + } + setPricingCell(t, rows, "gpt-6.1-sol", "fast", "short", column, value) + replacePricingRecords(t, bad, rows) + bad["manifest.json"] = files["manifest.json"] + if _, err := loadFiles(bad, Source{}); err == nil || !strings.Contains(err.Error(), "checksum") { + t.Fatalf("changed %s escaped checksum: %v", column, err) + } + } + rows := pricingRecords(t, files) + setPricingCell(t, rows, "gpt-6.1-sol", "standard", "long", "input_tokens_min_inclusive", "272002") + replacePricingRecords(t, files, rows) + for _, name := range requiredFiles { + if err := os.WriteFile(filepath.Join(source, name), files[name], 0o600); err != nil { + t.Fatal(err) + } // #nosec G703 -- fixed catalog names under t.TempDir. + } + if _, err := LoadPath(source); err == nil { + t.Fatal("invalid explicit catalog fell back") + } + if _, err := Update(context.Background(), UpdateOptions{SourceDir: source, InstallDir: target}); err == nil { + t.Fatal("invalid interval update succeeded") + } + if activeRevision(t, target) != pointer { + t.Fatal("invalid update replaced pointer") + } + after, err := LoadPath(filepath.Join(target, pointer)) + if err != nil || after.Digest() != before.Digest() { + t.Fatalf("previous snapshot not retained: %v", err) + } +} + +func TestRequestPricingJSONPreservesUnboundedAndZero(t *testing.T) { + catalog := requireRequestCatalog(t, requestCatalogFiles(t)) + for _, input := range []int64{0, 272001} { + selection, reason := catalog.PriceForRequest("openai-api-key", "gpt-6.1-sol", "default", input) + if reason != "" { + t.Fatal(reason) + } + body, err := json.Marshal(selection) + if err != nil { + t.Fatal(err) + } + var decoded PriceSelection + if err := json.Unmarshal(body, &decoded); err != nil || !reflect.DeepEqual(decoded, selection) { + t.Fatalf("selection JSON lost binding: %s, %v", body, err) + } + if input == 0 && !bytes.Contains(body, []byte(`"input_tokens_min_inclusive":0`)) { + t.Fatal("JSON omitted known zero") + } + if input == 272001 && !bytes.Contains(body, []byte(`"input_tokens_max_exclusive":null`)) { + t.Fatal("JSON omitted unbounded maximum") + } + } +} + +func TestRequestPricingRequiresCompleteMetadataAndPreservesTierSets(t *testing.T) { + for column, value := range map[string]string{ + "runtime_id": "openai-api-key", "observed_service_tiers": "default", + "input_tokens_min_inclusive": "0", "input_tokens_max_exclusive": "100", + "rate_application": "whole_request", + } { + files := requestCatalogFiles(t) + rows := pricingRecords(t, files) + setPricingCell(t, rows, "claude-sonnet-5", "standard", "all", column, value) + replacePricingRecords(t, files, rows) + if _, err := loadFiles(files, Source{}); err == nil { + t.Fatalf("accepted only %s of a binding", column) + } + } + files := requestCatalogFiles(t) + rows := pricingRecords(t, files) + setPricingCell(t, rows, "gpt-6.1-sol", "fast", "long", "observed_service_tiers", "priority|fast") + setPricingCell(t, rows, "gpt-6.1-sol", "fast", "short", "input_tokens_min_inclusive", " 0 ") + replacePricingRecords(t, files, rows) + catalog := requireRequestCatalog(t, files) + for _, tier := range []string{"fast", "priority"} { + selection, reason := catalog.PriceForRequest("openai-api-key", "gpt-6.1-sol", tier, 272001) + if reason != "" || selection.ObservedServiceTier != tier || selection.Price.ContextBand != "long" { + t.Fatal("tier-set equality lost exact observed tier") + } + } +} + +func TestRequestPricingRetainsLegacyRateValidation(t *testing.T) { + for _, schema := range []int{1, 2} { + for _, invalid := range []string{"-1", "NaN", "+Inf", "1e999", "missing provenance", "all unknown"} { + files := requestCatalogFiles(t) + if schema == 1 { + files = catalogFiles(t, filepath.Join("testdata", "v1")) + } + rows := pricingRecords(t, files) + switch invalid { + case "missing provenance": + setPricingCell(t, rows, "claude-sonnet-5", "standard", "all", "source_url", "") + case "all unknown": + for _, column := range rows[0][3:9] { + setPricingCell(t, rows, "claude-sonnet-5", "standard", "all", column, "null") + } + default: + setPricingCell(t, rows, "claude-sonnet-5", "standard", "all", "input_usd_per_million", invalid) + } + replacePricingRecords(t, files, rows) + if _, err := loadFiles(files, Source{}); err == nil { + t.Fatalf("schema %d accepted %s rate", schema, invalid) + } + } + } +} + +func replacePricingHeader(t *testing.T, files map[string][]byte, header string) { + t.Helper() + parts := bytes.SplitN(files["pricing.csv"], []byte("\n"), 2) + if len(parts) != 2 { + t.Fatal("pricing fixture has no header line") + } + files["pricing.csv"] = append([]byte(header+"\n"), parts[1]...) + manifest := readManifestBytes(t, files["manifest.json"]) + manifest.Files["pricing.csv"] = digest(files["pricing.csv"]) + files["manifest.json"] = marshalManifest(t, manifest) +} + +func TestSchemaTwoPricingRequiresExactHeaderNames(t *testing.T) { + base := requestCatalogFiles(t) + headers := pricingRecords(t, base)[0] + cases := map[string]string{} + for i, header := range headers { + for _, padding := range []struct{ name, prefix, suffix string }{ + {"leading space", " ", ""}, {"trailing space", "", " "}, + {"leading tab", "\t", ""}, {"trailing tab", "", "\t"}, + {"quoted leading", "\" ", "\""}, {"quoted trailing", "\"", " \""}, + } { + changed := append([]string(nil), headers...) + changed[i] = padding.prefix + header + padding.suffix + cases[header+"/"+padding.name] = strings.Join(changed, ",") + } + } + for _, indexes := range [][]int{{11, 12, 13, 14, 15}, {11, 12}, {13, 15}} { + changed := append([]string(nil), headers...) + for _, i := range indexes { + changed[i] += " " + } + cases[fmt.Sprintf("partial-or-all-five-%v", indexes)] = strings.Join(changed, ",") + } + duplicate := append([]string(nil), headers...) + duplicate[12] = duplicate[11] + cases["duplicate"] = strings.Join(duplicate, ",") + duplicate[12] = duplicate[11] + " " + cases["canonical collision"] = strings.Join(duplicate, ",") + reordered := append([]string(nil), headers...) + reordered[13], reordered[14] = reordered[14], reordered[13] + cases["reordered bounds"] = strings.Join(reordered, ",") + cases["missing column"] = strings.Join(headers[:15], ",") + cases["extra duplicate column"] = strings.Join(append(append([]string(nil), headers...), headers[15]), ",") + for name, header := range cases { + t.Run(name, func(t *testing.T) { + files := copyBytes(base) + replacePricingHeader(t, files, header) + if _, err := loadFiles(files, Source{}); err == nil || !strings.Contains(err.Error(), "header") { + t.Fatalf("non-exact v2 header error = %v, want header rejection", err) + } + }) + } +} + +func TestSchemaOnePricingRetainsHistoricalUnquotedLeadingWhitespace(t *testing.T) { + files := catalogFiles(t, filepath.Join("testdata", "v1")) + original := requireRequestCatalog(t, files) + headers := pricingRecords(t, files)[0] + for i := range headers { + headers[i] = " " + headers[i] + } + replacePricingHeader(t, files, strings.Join(headers, ",")) + legacy := requireRequestCatalog(t, files) + if !reflect.DeepEqual(legacy.Pricing(), original.Pricing()) { + t.Fatal("schema-1 historical leading-header whitespace behavior changed") + } +} diff --git a/internal/modelcatalog/testdata/v1/defaults.csv b/internal/modelcatalog/testdata/v1/defaults.csv new file mode 100644 index 00000000..3c447d34 --- /dev/null +++ b/internal/modelcatalog/testdata/v1/defaults.csv @@ -0,0 +1,10 @@ +runtime_id,tier,model_id,effort,max_effort,source_url,verified_date +anthropic-subscription-claude-cli,small,claude-sonnet-5-5,low,,https://platform.claude.com/docs/en/models/sonnet-5-5/overview,2026-10-02 +anthropic-subscription-claude-cli,medium,claude-sonnet-5-5,medium,,https://platform.claude.com/docs/en/models/sonnet-5-5/overview,2026-10-02 +anthropic-subscription-claude-cli,large,claude-opus-5-5,medium,,https://platform.claude.com/docs/en/models,2026-10-02 +openai-api-key,small,gpt-6-luna,max,,https://developers.openai.com/api/docs/models,2026-10-02 +openai-api-key,medium,gpt-6.1-sol,low,,https://developers.openai.com/api/docs/models/gpt-6.1-sol,2026-10-02 +openai-api-key,large,gpt-6.1-sol,medium,,https://developers.openai.com/api/docs/models/gpt-6.1-sol,2026-10-02 +openai-subscription-codex-cli,small,gpt-6-luna,max,,https://learn.chatgpt.com/docs/agent-configuration/speed,2026-10-02 +openai-subscription-codex-cli,medium,gpt-6.1-sol,low,,https://learn.chatgpt.com/docs/agent-configuration/speed,2026-10-02 +openai-subscription-codex-cli,large,gpt-6.1-sol,medium,,https://learn.chatgpt.com/docs/agent-configuration/speed,2026-10-02 diff --git a/internal/modelcatalog/testdata/v1/manifest.json b/internal/modelcatalog/testdata/v1/manifest.json new file mode 100644 index 00000000..05a29cea --- /dev/null +++ b/internal/modelcatalog/testdata/v1/manifest.json @@ -0,0 +1,28 @@ +{ + "schema_version": 1, + "revision": "baseline-2026-10-02", + "generated_at": "2026-10-02", + "files": { + "runtimes.csv": "c8259b7dd4ea3a9ab149656c74a3e37fa58874ecb57d75a59ff4b8642a32ff0d", + "models.csv": "cc80619402a47d7338b8c70ffa190001a2247aedc3da00e76aeb9d2c31e6c2d8", + "defaults.csv": "7d7ee13a060a70130597b35708d13baddd3f6576cf7aa7f986d7532918734531", + "pricing.csv": "236305213c9876472c4efad072e9c3bb07cbfe20276dd70b65d6fd9729131447" + }, + "sources": [ + { + "kind": "official", + "path": "https://developers.openai.com/api/docs/models/gpt-6.1-sol", + "revision": "gpt-6.1-sol-2026-10-02" + }, + { + "kind": "official", + "path": "https://platform.claude.com/docs/en/models/sonnet-5-5/overview", + "revision": "claude-sonnet-5-5-2026-10-02" + }, + { + "kind": "harness", + "path": "https://learn.chatgpt.com/docs/agent-configuration/speed", + "revision": "codex-speed-2026-10-02" + } + ] +} diff --git a/internal/modelcatalog/testdata/v1/models.csv b/internal/modelcatalog/testdata/v1/models.csv new file mode 100644 index 00000000..66e15c19 --- /dev/null +++ b/internal/modelcatalog/testdata/v1/models.csv @@ -0,0 +1,37 @@ +runtime_id,model_id,supported_efforts,fast,ultrafast,min_harness_version,source_url,verified_date +anthropic-api-key,claude-fable-5-1,low|medium|high|xhigh|max,,,,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 +anthropic-api-key,claude-haiku-4-5,,,,,https://platform.claude.com/docs/en/about-claude/models,2026-10-02 +anthropic-api-key,claude-haiku-4-5-20251001,,,,,https://platform.claude.com/docs/en/about-claude/models,2026-10-02 +anthropic-api-key,claude-opus-4-8,low|medium|high|xhigh|max,Y,,,https://platform.claude.com/docs/en/models,2026-10-02 +anthropic-api-key,claude-opus-5,low|medium|high|xhigh|max,Y,,,https://platform.claude.com/docs/en/models,2026-10-02 +anthropic-api-key,claude-opus-5-5,low|medium|high|xhigh|max,Y,,,https://platform.claude.com/docs/en/models,2026-10-02 +anthropic-api-key,claude-sonnet-4-6,low|medium|high|max,,,,https://platform.claude.com/docs/en/about-claude/models,2026-10-02 +anthropic-api-key,claude-sonnet-5,low|medium|high|xhigh|max,,,,https://platform.claude.com/docs/en/about-claude/models,2026-10-02 +anthropic-api-key,claude-sonnet-5-5,low|medium|high|xhigh|max,,,,https://platform.claude.com/docs/en/models/sonnet-5-5/overview,2026-10-02 +anthropic-subscription-claude-cli,claude-fable-5-1,low|medium|high|xhigh|max,,,,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 +anthropic-subscription-claude-cli,claude-haiku-4-5,,,,,https://platform.claude.com/docs/en/about-claude/models,2026-10-02 +anthropic-subscription-claude-cli,claude-haiku-4-5-20251001,,,,,https://platform.claude.com/docs/en/about-claude/models,2026-10-02 +anthropic-subscription-claude-cli,claude-opus-4-8,low|medium|high|xhigh|max,Y,,,https://platform.claude.com/docs/en/models,2026-10-02 +anthropic-subscription-claude-cli,claude-opus-5,low|medium|high|xhigh|max,Y,,,https://platform.claude.com/docs/en/models,2026-10-02 +anthropic-subscription-claude-cli,claude-opus-5-5,low|medium|high|xhigh|max,Y,,,https://platform.claude.com/docs/en/models,2026-10-02 +anthropic-subscription-claude-cli,claude-sonnet-4-6,low|medium|high|max,,,,https://platform.claude.com/docs/en/about-claude/models,2026-10-02 +anthropic-subscription-claude-cli,claude-sonnet-5,low|medium|high|xhigh|max,,,,https://platform.claude.com/docs/en/about-claude/models,2026-10-02 +anthropic-subscription-claude-cli,claude-sonnet-5-5,low|medium|high|xhigh|max,,,,https://platform.claude.com/docs/en/models/sonnet-5-5/overview,2026-10-02 +openai-api-key,gpt-5.4,low|medium|high|xhigh,,,,https://developers.openai.com/api/docs/models/gpt-5.4,2026-10-02 +openai-api-key,gpt-5.5,low|medium|high|xhigh,,,,https://developers.openai.com/api/docs/models/gpt-5.5,2026-10-02 +openai-api-key,gpt-5.6-luna,low|medium|high|xhigh|max,,,,https://developers.openai.com/api/docs/models,2026-10-02 +openai-api-key,gpt-5.6-sol,low|medium|high|xhigh|max,Y,,,https://developers.openai.com/api/docs/models/gpt-5.6-sol,2026-10-02 +openai-api-key,gpt-5.6-terra,low|medium|high|xhigh|max,,,,https://developers.openai.com/api/docs/models,2026-10-02 +openai-api-key,gpt-6-astra,low|medium|high|xhigh|max,Y,Y,,https://developers.openai.com/api/docs/guides/ultrafast-mode,2026-10-02 +openai-api-key,gpt-6-luna,low|medium|high|xhigh|max,Y,,,https://developers.openai.com/api/docs/models,2026-10-02 +openai-api-key,gpt-6-sol,low|medium|high|xhigh|max,Y,,,https://developers.openai.com/api/docs/models,2026-10-02 +openai-api-key,gpt-6.1-sol,low|medium|high|xhigh|max,Y,,,https://developers.openai.com/api/docs/models/gpt-6.1-sol,2026-10-02 +openai-subscription-codex-cli,gpt-5.4,low|medium|high|xhigh|max,Y,,,https://learn.chatgpt.com/docs/agent-configuration/speed,2026-10-02 +openai-subscription-codex-cli,gpt-5.5,low|medium|high|xhigh,Y,,,https://learn.chatgpt.com/docs/agent-configuration/speed,2026-10-02 +openai-subscription-codex-cli,gpt-5.6-luna,low|medium|high|xhigh|max,Y,,,https://learn.chatgpt.com/docs/agent-configuration/speed,2026-10-02 +openai-subscription-codex-cli,gpt-5.6-sol,low|medium|high|xhigh|max,Y,,,https://learn.chatgpt.com/docs/agent-configuration/speed,2026-10-02 +openai-subscription-codex-cli,gpt-5.6-terra,low|medium|high|xhigh|max,Y,,,https://learn.chatgpt.com/docs/agent-configuration/speed,2026-10-02 +openai-subscription-codex-cli,gpt-6-astra,low|medium|high|xhigh|max,Y,Y,,https://learn.chatgpt.com/docs/agent-configuration/speed,2026-10-02 +openai-subscription-codex-cli,gpt-6-luna,low|medium|high|xhigh|max,Y,,,https://learn.chatgpt.com/docs/agent-configuration/speed,2026-10-02 +openai-subscription-codex-cli,gpt-6-sol,low|medium|high|xhigh|max,Y,,,https://learn.chatgpt.com/docs/agent-configuration/speed,2026-10-02 +openai-subscription-codex-cli,gpt-6.1-sol,low|medium|high|xhigh|max,Y,,,https://learn.chatgpt.com/docs/agent-configuration/speed,2026-10-02 diff --git a/internal/modelcatalog/testdata/v1/pricing.csv b/internal/modelcatalog/testdata/v1/pricing.csv new file mode 100644 index 00000000..d472cc5a --- /dev/null +++ b/internal/modelcatalog/testdata/v1/pricing.csv @@ -0,0 +1,41 @@ +model_id,speed,context_band,input_usd_per_million,output_usd_per_million,cache_read_usd_per_million,cache_write_usd_per_million,cache_write_5m_usd_per_million,cache_write_1h_usd_per_million,source_url,verified_date +claude-fable-5-1,standard,all,10,50,0.25,,12.5,20,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 +claude-opus-4-8,standard,all,5,25,0.5,,6.25,10,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 +claude-opus-4-8,fast,all,10,50,1,,12.5,20,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 +claude-opus-5,standard,all,5,25,0.5,,6.25,10,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 +claude-opus-5,fast,all,10,50,1,,12.5,20,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 +claude-opus-5-5,standard,all,4,20,0.2,,5,8,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 +claude-opus-5-5,fast,all,8,40,0.4,,10,16,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 +claude-sonnet-4-6,standard,all,3,15,0.3,,3.75,6,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 +claude-sonnet-5,standard,all,2,10,0.2,,2.5,4,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 +claude-sonnet-5-5,standard,all,2,10,0.2,,2.5,4,https://platform.claude.com/docs/en/models/sonnet-5-5/overview,2026-10-02 +claude-haiku-4-5,standard,all,1,5,0.1,,1.25,2,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 +claude-haiku-4-5-20251001,standard,all,1,5,0.1,,1.25,2,https://platform.claude.com/docs/en/about-claude/pricing,2026-10-02 +gpt-6-astra,standard,short,10,50,1,12.5,,,https://developers.openai.com/api/docs/pricing,2026-10-02 +gpt-6-astra,standard,long,20,75,2,25,,,https://developers.openai.com/api/docs/pricing,2026-10-02 +gpt-6-astra,fast,short,20,100,2,25,,,https://developers.openai.com/api/docs/pricing,2026-10-02 +gpt-6-astra,fast,long,40,150,4,50,,,https://developers.openai.com/api/docs/pricing,2026-10-02 +gpt-6-astra,ultrafast,short,60,300,6,75,,,https://developers.openai.com/api/docs/pricing,2026-10-02 +gpt-6-astra,ultrafast,long,120,450,12,150,,,https://developers.openai.com/api/docs/pricing,2026-10-02 +gpt-6-luna,standard,short,0.1,0.5,0.01,0.125,,,https://developers.openai.com/api/docs/pricing,2026-10-02 +gpt-6-luna,standard,long,0.2,0.75,0.02,0.25,,,https://developers.openai.com/api/docs/pricing,2026-10-02 +gpt-6-luna,fast,short,0.2,1,0.02,0.25,,,https://developers.openai.com/api/docs/pricing,2026-10-02 +gpt-6-luna,fast,long,0.4,1.5,0.04,0.5,,,https://developers.openai.com/api/docs/pricing,2026-10-02 +gpt-6.1-sol,standard,short,2,10,0.1,2.5,,,https://developers.openai.com/api/docs/pricing,2026-10-02 +gpt-6.1-sol,standard,long,4,15,0.2,5,,,https://developers.openai.com/api/docs/pricing,2026-10-02 +gpt-6.1-sol,fast,short,4,20,0.2,5,,,https://developers.openai.com/api/docs/pricing,2026-10-02 +gpt-6.1-sol,fast,long,8,30,0.4,10,,,https://developers.openai.com/api/docs/pricing,2026-10-02 +gpt-6-sol,standard,short,2,10,0.2,2.5,,,https://developers.openai.com/api/docs/models/gpt-6-sol,2026-10-02 +gpt-6-sol,standard,long,4,15,0.4,5,,,https://developers.openai.com/api/docs/models/gpt-6-sol,2026-10-02 +gpt-6-sol,fast,short,4,20,0.4,5,,,https://developers.openai.com/api/docs/pricing,2026-10-02 +gpt-6-sol,fast,long,8,30,0.8,10,,,https://developers.openai.com/api/docs/pricing,2026-10-02 +gpt-5.6-sol,standard,short,4,20,0.4,5,,,https://developers.openai.com/api/docs/models/gpt-5.6-sol,2026-10-02 +gpt-5.6-sol,standard,long,8,30,0.8,10,,,https://developers.openai.com/api/docs/models/gpt-5.6-sol,2026-10-02 +gpt-5.6-terra,standard,short,2,12,0.2,2.5,,,https://developers.openai.com/api/docs/models/gpt-5.6-terra,2026-10-02 +gpt-5.6-terra,standard,long,4,18,0.4,5,,,https://developers.openai.com/api/docs/models/gpt-5.6-terra,2026-10-02 +gpt-5.6-luna,standard,short,0.2,1.2,0.02,0.25,,,https://developers.openai.com/api/docs/models/gpt-5.6-luna,2026-10-02 +gpt-5.6-luna,standard,long,0.4,1.8,0.04,0.5,,,https://developers.openai.com/api/docs/models/gpt-5.6-luna,2026-10-02 +gpt-5.4,standard,short,2.5,15,0.25,,,,https://developers.openai.com/api/docs/models/gpt-5.4,2026-10-02 +gpt-5.4,standard,long,5,22.5,0.5,,,,https://developers.openai.com/api/docs/models/gpt-5.4,2026-10-02 +gpt-5.5,standard,short,5,30,0.5,,,,https://developers.openai.com/api/docs/models/gpt-5.5,2026-10-02 +gpt-5.5,standard,long,10,45,1,,,,https://developers.openai.com/api/docs/models/gpt-5.5,2026-10-02 diff --git a/internal/modelcatalog/testdata/v1/runtimes.csv b/internal/modelcatalog/testdata/v1/runtimes.csv new file mode 100644 index 00000000..033fc952 --- /dev/null +++ b/internal/modelcatalog/testdata/v1/runtimes.csv @@ -0,0 +1,6 @@ +runtime_id,provider,auth,harness,adapter,suggested_name,display_name,maximum_effort,requires_credential_ref,source_url,verified_date +anthropic-subscription-claude-cli,anthropic,subscription,claude_cli,claude_cli,claude-cli,Claude CLI,max,N,https://docs.anthropic.com/en/docs/claude-code,2026-10-02 +anthropic-api-key,anthropic,api_key,anthropic_api,anthropic_api,anthropic-api-key,Anthropic API,high,Y,https://docs.anthropic.com/en/api/overview,2026-10-02 +openai-subscription-codex-cli,openai,subscription,codex_cli,codex_cli,codex-cli,Codex CLI,max,N,https://developers.openai.com/codex/cli,2026-10-02 +openai-api-key,openai,api_key,openai_api,openai_api,openai-api-key,OpenAI API,max,Y,https://developers.openai.com/api/docs/overview,2026-10-02 +pi-subscription-rpc,pi,subscription,pi_rpc,pi_rpc,pi-local,Pi RPC,max,N,https://github.com/badlogic/pi-mono,2026-10-02 diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index 4c27c498..2db9034f 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -201,8 +201,12 @@ type Request struct { ReviewerFast bool ReviewBaseSHA string ReviewHeadSHA string - Rerun bool - FreshSession bool + // WithoutDiscussion replays a pinned dry-run review as a first pass: no + // PR discussion is fetched, and no PR-scoped orchestrator or reviewer + // session (which saw earlier discussion) is resumed or updated. + WithoutDiscussion bool + Rerun bool + FreshSession bool FailOn *review.Severity AllowSelfReview bool @@ -292,6 +296,8 @@ type Result struct { CurrentHeadSHA string ReviewBaseSHA string ReviewHeadSHA string + ThreadAnalysisFailures []threadanalysis.Failure + WithoutDiscussion bool ReviewerFailures []ReviewerFailure ReviewerCoverage []reviewplan.ReviewerCoverageSummary RelocationAssessments []ReviewerRelocationAssessments @@ -380,9 +386,11 @@ type selectionSetupRequest struct { ReviewBaseSHA string ReviewHeadSHA string NoResolveThreads bool - ResolvedPR *reviewPRContext - InvocationRoot *string - ResolveArtifacts func(gitprovider.PR) (ArtifactPaths, error) + // WithoutDiscussion skips every PR discussion read, independent of pinning. + WithoutDiscussion bool + ResolvedPR *reviewPRContext + InvocationRoot *string + ResolveArtifacts func(gitprovider.PR) (ArtifactPaths, error) } type preparedSelectionContext struct { @@ -463,6 +471,9 @@ func Live(ctx context.Context, opts Options, req Request, run ledger.Run) (Resul if strings.TrimSpace(req.ReviewBaseSHA) != "" || strings.TrimSpace(req.ReviewHeadSHA) != "" { return Result{}, Failure(FailureTerminal, fmt.Errorf("pipeline: pinned review SHAs require dry-run review")) } + if req.WithoutDiscussion { + return Result{}, Failure(FailureTerminal, fmt.Errorf("pipeline: review without discussion requires dry-run review")) + } if strings.TrimSpace(run.RunID) == "" { return Result{}, Failure(FailureTerminal, fmt.Errorf("pipeline: live run ID is required")) } @@ -653,15 +664,16 @@ func execute(ctx context.Context, opts Options, req Request, mode executionMode) return Result{}, fmt.Errorf("pipeline: reviewer credentials resolve to PR author %q; pass --allow-self-review to continue", req.PostingIdentity.Login) } prepared, err := prepareSelectionContext(ctx, opts, selectionSetupRequest{ - PRRef: req.PRRef, - Profile: req.Profile, - PostingIdentity: req.PostingIdentity, - AgentDirs: req.AgentDirs, - ReviewBaseSHA: req.ReviewBaseSHA, - ReviewHeadSHA: req.ReviewHeadSHA, - NoResolveThreads: req.NoResolveThreads, - ResolvedPR: &reviewCtx, - InvocationRoot: &invocationRoot, + PRRef: req.PRRef, + Profile: req.Profile, + PostingIdentity: req.PostingIdentity, + AgentDirs: req.AgentDirs, + ReviewBaseSHA: req.ReviewBaseSHA, + ReviewHeadSHA: req.ReviewHeadSHA, + NoResolveThreads: req.NoResolveThreads, + WithoutDiscussion: req.WithoutDiscussion, + ResolvedPR: &reviewCtx, + InvocationRoot: &invocationRoot, ResolveArtifacts: func(reviewPR gitprovider.PR) (ArtifactPaths, error) { if mode.live { return ArtifactPathsFromDir(mode.run.ArtifactPath), nil @@ -678,6 +690,7 @@ func execute(ctx context.Context, opts Options, req Request, mode executionMode) prepared.fastRequested = req.ReviewerFast result := prepared.reviewResult() + result.WithoutDiscussion = req.WithoutDiscussion run := mode.run if !mode.live { if resumedDryRun != nil { @@ -698,7 +711,7 @@ func execute(ctx context.Context, opts Options, req Request, mode executionMode) if err != nil { return Result{}, err } - if err := runartifact.WriteMarker(prepared.artifacts.Dir, runartifact.KindReview, run.RunID); err != nil { + if err := runartifact.WriteMarkerWithOptions(prepared.artifacts.Dir, runartifact.KindReview, run.RunID, runartifact.MarkerOptions{WithoutDiscussion: req.WithoutDiscussion}); err != nil { return Result{}, err } } @@ -716,6 +729,7 @@ func execute(ctx context.Context, opts Options, req Request, mode executionMode) ChangedFiles: prepared.changedFiles, Artifacts: prepared.artifacts, HeadRefNamespace: opts.Provider.Capabilities().HeadRefNamespace, + Offline: req.WithoutDiscussion, }); err != nil { if errors.Is(err, workbench.ErrUnsafeFetchRef) || errors.Is(err, workbench.ErrInvalidRepositoryIdentity) { return Result{}, Failure(FailureTerminal, err) @@ -813,15 +827,25 @@ func executeLLMPhases(ctx context.Context, opts Options, req Request, mode execu if err != nil { return nil, false, Failure(FailureTerminal, err) } - namedSession, err := prepareNamedSession(ctx, opts, req, mode.live, runtimeConfig.model, now) - if err != nil { - return nil, false, err - } - - cohortScope := ledger.ReviewerCohortScope{PRKey: prepared.prKey, Profile: req.ProfileName, PostingIdentity: runlifecycle.PostingKey(req.PostingIdentity)} - selection, reviewerResumeIDs, reusedCohort, err := loadReviewerCohort(ctx, opts, req, cohortScope, prepared.catalog, relocationReviewablePatchPaths(prepared.parsed.Patches, prepared.relocations.Manifest.Moves), maxAgents) - if err != nil { - return executionPhaseFailure(err) + // A review without discussion must not resume the PR's orchestrator or + // reviewer sessions: they carry earlier discussion. It also leaves them + // untouched, so a replay never becomes the PR's next live context. The zero + // named-session state and cohort scope make every checkpoint a no-op. + var namedSession namedSessionState + var cohortScope ledger.ReviewerCohortScope + var selection llm.Selection + var reviewerResumeIDs map[string]string + reusedCohort := false + if !req.WithoutDiscussion { + namedSession, err = prepareNamedSession(ctx, opts, req, mode.live, runtimeConfig.model, now) + if err != nil { + return nil, false, err + } + cohortScope = ledger.ReviewerCohortScope{PRKey: prepared.prKey, Profile: req.ProfileName, PostingIdentity: runlifecycle.PostingKey(req.PostingIdentity)} + selection, reviewerResumeIDs, reusedCohort, err = loadReviewerCohort(ctx, opts, req, cohortScope, prepared.catalog, relocationReviewablePatchPaths(prepared.parsed.Patches, prepared.relocations.Manifest.Moves), maxAgents) + if err != nil { + return executionPhaseFailure(err) + } } if reusedCohort { if err := restoreOrchestratorSessionFromRun(ctx, opts.Store, run.RunID, prepared.artifacts, &namedSession); err != nil { @@ -885,7 +909,7 @@ func executeLLMPhases(ctx context.Context, opts Options, req Request, mode execu prepared.fastIgnored = req.ReviewerFast && !effectiveFast req.ReviewerFast = effectiveFast opts.emitWarning(warning) - if !reusedCohort { + if !reusedCohort && !req.WithoutDiscussion { if err := persistReviewerCohort(ctx, opts, req, cohortScope, prepared.catalog, selection, now); err != nil { return nil, false, err } @@ -895,12 +919,13 @@ func executeLLMPhases(ctx context.Context, opts Options, req Request, mode execu if reusedCohort && !selectionInRun { selectionTaskIDs = nil } - threadResponses, err := analyzeReviewThreads(ctx, opts, req, run, prepared.artifacts, prepared.threadContext, namedSession.resumeID(), func(sessionID string) error { + threadResponses, threadFailures, err := analyzeReviewThreads(ctx, opts, req, run, prepared.artifacts, prepared.threadContext, namedSession.resumeID(), func(sessionID string) error { return namedSession.checkpointProviderSessionID(ctx, sessionID, opts.now()) }) if err != nil { return executionPhaseFailure(err) } + result.ThreadAnalysisFailures = threadFailures checkpointActions, err := checkpointThreadResponses(ctx, opts, req, mode, run, result.EffectiveCaps, threadResponses) if err != nil { return nil, false, err @@ -929,7 +954,7 @@ func executeLLMPhases(ctx context.Context, opts Options, req Request, mode execu return nil, false, Failure(FailureTerminal, err) } rollupModel, rollupEffort := rollupRuntimeConfig.model, rollupRuntimeConfig.effort - rollupPrompt, err := buildRollupPrompt(prepared.reviewPR, reviewerRun.findings, reviewerRun.failures, reviewerCoverage) + rollupPrompt, err := buildRollupPrompt(prepared.reviewPR, reviewerRun.findings, reviewerRun.failures, reviewerCoverage, threadFailureSummaries(threadFailures)) if err != nil { return nil, false, Failure(FailureTerminal, err) } @@ -979,6 +1004,7 @@ func executeLLMPhases(ctx context.Context, opts Options, req Request, mode execu plan, err := opts.buildPlan(req, prepared.reviewPR, mode.planPostMode, result.EffectiveCaps, prepared.parsed.PlanDiff, reviewerRun.findings, rollup, selection.ThreadActions, false, result.AgentDefsChanged, planRunInputs{ threadResponses: threadResponses, + threadFailures: threadFailures, repoSources: repoSources, hasRun: true, selection: selectionSession, @@ -1069,7 +1095,13 @@ func findIncompleteDryRun(ctx context.Context, store Store, req Request, pr gitp } runs = append(runs, legacyRuns...) } - best, found := runlifecycle.NewestCompatibleIncompleteRun(runs, pr.Base.SHA, ledger.PostModeDryRun, runartifact.KindReview, runartifact.MarkerMatches) + // A run resumes only into a run with the same discussion setting, so a + // discussion-free replay never reuses tasks or sessions that saw discussion. + markerMatches := func(artifactPath, kind, runID string) bool { + marker, err := runartifact.ReadMarker(artifactPath, kind) + return err == nil && marker.RunID == runID && marker.WithoutDiscussion == req.WithoutDiscussion + } + best, found := runlifecycle.NewestCompatibleIncompleteRun(runs, pr.Base.SHA, ledger.PostModeDryRun, runartifact.KindReview, markerMatches) return best, found, nil } @@ -1194,7 +1226,9 @@ func prepareSelectionContext(ctx context.Context, opts Options, req selectionSet var threadContext []threadcontext.Thread var reviews []gitprovider.Review var issueComments []gitprovider.IssueComment - if !reviewCtx.pinnedReview { + // Discussion is cut off here, at its only source, so no later consumer + // (dossier, selection, thread analysis, reviewers, planner) can see it. + if !reviewCtx.pinnedReview && !req.WithoutDiscussion { threads, err = opts.Provider.ListInlineThreads(ctx, req.PRRef) if err != nil { return preparedSelectionContext{}, err @@ -1262,6 +1296,7 @@ func prepareSelectionContext(ctx context.Context, opts Options, req selectionSet CurrentPR: pr, ReviewPR: reviewPR, PinnedReview: reviewCtx.pinnedReview, + WithoutDiscussion: req.WithoutDiscussion, ChangedFiles: dossierChangedFiles(parsed.Patches), Threads: threads, ThreadContext: threadContext, @@ -1270,13 +1305,20 @@ func prepareSelectionContext(ctx context.Context, opts Options, req selectionSet Catalog: catalog, CurrentBaseSHA: reviewCtx.currentBaseSHA, CurrentHeadSHA: reviewCtx.currentHeadSHA, - DiscussionOmittedNote: "Current PR discussion omitted because this review is pinned to explicit base/head SHAs.", + DiscussionOmittedNote: discussionOmittedNote(req.WithoutDiscussion), }); err != nil { return preparedSelectionContext{}, err } return out, nil } +func discussionOmittedNote(withoutDiscussion bool) string { + if withoutDiscussion { + return "No PR discussion is provided for this review." + } + return "Current PR discussion omitted because this review is pinned to explicit base/head SHAs." +} + func resolveReviewPRContext(ctx context.Context, provider ReadProvider, ref gitprovider.PRRef, reviewBaseSHA, reviewHeadSHA string) (reviewPRContext, error) { pr, err := provider.GetPR(ctx, ref) if err != nil { @@ -1393,16 +1435,16 @@ func runSelectionPhase(ctx context.Context, opts Options, req selectionPhaseRequ return selection, selectionSession, ledgerSession, nil } -func analyzeReviewThreads(ctx context.Context, opts Options, req Request, run ledger.Run, artifacts ArtifactPaths, threads []threadcontext.Thread, resumeSessionID string, onSessionID func(string) error) ([]review.ThreadResponseAction, error) { +func analyzeReviewThreads(ctx context.Context, opts Options, req Request, run ledger.Run, artifacts ArtifactPaths, threads []threadcontext.Thread, resumeSessionID string, onSessionID func(string) error) ([]review.ThreadResponseAction, []threadanalysis.Failure, error) { eligible := threadcontext.PendingCRAuthoredFindingThreads(threads) if len(eligible) == 0 { - return nil, nil + return nil, nil, nil } runtimeConfig, err := resolveThreadAnalysisRuntimeConfig(req.Profile) if err != nil { - return nil, err + return nil, nil, err } - results, err := threadanalysis.AnalyzeThreads(ctx, threadanalysis.Options{ + results, failures, err := threadanalysis.AnalyzeThreads(ctx, threadanalysis.Options{ Store: opts.Store, RunID: run.RunID, Adapter: opts.Adapter, @@ -1414,13 +1456,17 @@ func analyzeReviewThreads(ctx context.Context, opts Options, req Request, run le NewStepID: opts.newSessionRowID, ResumeSessionID: resumeSessionID, OnSessionID: onSessionID, + IsolateFailures: true, }, eligible, func(thread threadcontext.Thread) (string, error) { return artifacts.ThreadAnalysisLog(string(thread.ID)) }) if err != nil { - return nil, pipelineTaskError(err) + return nil, nil, pipelineTaskError(err) + } + for _, failure := range failures { + opts.emitWarning(fmt.Sprintf("thread %s was not analyzed: %s; isolated failures are cached on resume, use --rerun to retry", failure.ThreadID, failure.Error)) } - return threadanalysis.ResponseActions(results), nil + return threadanalysis.ResponseActions(results), failures, nil } func checkpointThreadResponses(ctx context.Context, opts Options, req Request, mode executionMode, run ledger.Run, caps reviewplan.ProviderCaps, responses []review.ThreadResponseAction) ([]ledger.PlannedAction, error) { @@ -2184,7 +2230,7 @@ func runReviewer(ctx context.Context, opts Options, req Request, runID string, p return reviewerExecution{}, err } agentID := agent.ID - request, cleanupWorkspace, err := workbench.PrepareReviewerRequest(ctx, workbenchDeps(opts), opts.Adapter, artifacts, pr.Head.SHA, agent.ID, selected.AllowedFiles, model, effort, prompt, logPath, resumeState.relocations.Symlinks) + request, cleanupWorkspace, err := workbench.PrepareReviewerRequest(ctx, workbenchDeps(opts), opts.Adapter, artifacts, pr.Head.SHA, agent.ID, selected.AllowedFiles, model, effort, prompt, logPath, workbench.ReviewerOptions{Offline: req.WithoutDiscussion, SymlinkMetadata: resumeState.relocations.Symlinks}) if err != nil { return reviewerExecution{}, err } @@ -2203,6 +2249,9 @@ func runReviewer(ctx context.Context, opts Options, req Request, runID string, p fingerprintDeps = append(fingerprintDeps, "fast=true") } fingerprintDeps = append(fingerprintDeps, "relocation-manifest="+assignment.ManifestDigest, "relocation-assignment="+assignment.AssignmentDigest, "symlink-metadata="+assignment.SymlinkMetadataDigest, "symlink-contract="+symlinkInspectionContractVersion, "context-contract="+reviewerContextContractVersion) + if req.WithoutDiscussion { + fingerprintDeps = append(fingerprintDeps, "without_discussion=true") + } findings, session, ledgerSession, err := runStructuredTask(ctx, opts, llmTaskSpec{ runID: runID, taskID: taskID, @@ -2306,7 +2355,7 @@ func runReviewer(ctx context.Context, opts Options, req Request, runID string, p return repairSetupFailed(err) } // Its own identity: reusing agent.ID would reset the primary pass's workspace and scratch. - repairRequest, cleanupRepairWorkspace, err := workbench.PrepareReviewerRequest(ctx, workbenchDeps(opts), opts.Adapter, artifacts, pr.Head.SHA, repairIdentity, repairSelected.AllowedFiles, model, effort, repairPrompt, repairLogPath, resumeState.relocations.Symlinks) + repairRequest, cleanupRepairWorkspace, err := workbench.PrepareReviewerRequest(ctx, workbenchDeps(opts), opts.Adapter, artifacts, pr.Head.SHA, repairIdentity, repairSelected.AllowedFiles, model, effort, repairPrompt, repairLogPath, workbench.ReviewerOptions{Offline: req.WithoutDiscussion, SymlinkMetadata: resumeState.relocations.Symlinks}) if err != nil { return repairSetupFailed(err) } @@ -2326,6 +2375,9 @@ func runReviewer(ctx context.Context, opts Options, req Request, runID string, p repairFingerprintDeps = append(repairFingerprintDeps, "fast=true") } repairFingerprintDeps = append(repairFingerprintDeps, "relocation-manifest="+repairAssignment.ManifestDigest, "relocation-assignment="+repairAssignment.AssignmentDigest, "symlink-metadata="+repairAssignment.SymlinkMetadataDigest, "symlink-contract="+symlinkInspectionContractVersion, "context-contract="+reviewerContextContractVersion) + if req.WithoutDiscussion { + repairFingerprintDeps = append(repairFingerprintDeps, "without_discussion=true") + } repair, repairSession, repairLedgerSession, repairErr := runStructuredTask(ctx, opts, llmTaskSpec{ runID: runID, taskID: repairTaskID, @@ -2775,6 +2827,7 @@ type planRunInputs struct { rollup sessionDraft selectedAgents []llm.SelectedAgent threadResponses []review.ThreadResponseAction + threadFailures []threadanalysis.Failure repoSources []agents.SourceInfo findingSessions map[review.FindingID]string reviewerFailures []ReviewerFailure @@ -2845,15 +2898,16 @@ func (opts Options) buildRunSummary(req Request, inputs planRunInputs) (reviewpl wallMS := opts.now().Sub(inputs.startedAt).Milliseconds() summary := reviewplan.RunSummary{ - ToolVersion: req.ToolVersion, - Adapter: runAdapter(inputs), - Model: sharedWorkstreamModel(workstreams), - PostingIdentity: runlifecycle.PostingKey(req.PostingIdentity), - SelectedReviewers: selectedIDs, - ReviewerFailures: reviewerFailureSummaries(inputs.reviewerFailures), - ReviewerCoverage: inputs.reviewerCoverage, - WallDurationMS: &wallMS, - Workstreams: workstreams, + ToolVersion: req.ToolVersion, + Adapter: runAdapter(inputs), + Model: sharedWorkstreamModel(workstreams), + PostingIdentity: runlifecycle.PostingKey(req.PostingIdentity), + SelectedReviewers: selectedIDs, + ReviewerFailures: reviewerFailureSummaries(inputs.reviewerFailures), + ThreadAnalysisFailures: threadFailureSummaries(inputs.threadFailures), + ReviewerCoverage: inputs.reviewerCoverage, + WallDurationMS: &wallMS, + Workstreams: workstreams, } findingReviewers := make(map[review.FindingID]string, len(inputs.findingSessions)) @@ -2869,20 +2923,26 @@ func (opts Options) buildRunSummary(req Request, inputs planRunInputs) (reviewpl // several provider sessions, so they are deliberately not a sessionDraft: there // is no single session they could be written back to. type workstreamTotals struct { - model string - usage llm.Usage - durationMS int64 - startedAt time.Time - completedAt time.Time + requestHistories []*llm.RequestCostEvidence + blockLegacyEstimate bool + unknownRequestCost bool + model string + usage llm.Usage + durationMS int64 + startedAt time.Time + completedAt time.Time } func draftWorkstreamTotals(draft sessionDraft) workstreamTotals { return workstreamTotals{ - model: draft.Model, - usage: draft.Response.Usage, - durationMS: draft.Response.DurationMS, - startedAt: draft.StartedAt, - completedAt: draft.CompletedAt, + requestHistories: []*llm.RequestCostEvidence{llm.CloneRequestCostEvidence(draft.Response.RequestCostEvidence)}, + blockLegacyEstimate: draft.Adapter == "openai_api" || draft.Adapter == "codex_cli" || draft.Response.RequestCostEvidence != nil, + unknownRequestCost: draft.Adapter == "openai_api" || draft.Response.RequestCostEvidence != nil, + model: draft.Model, + usage: draft.Response.Usage, + durationMS: draft.Response.DurationMS, + startedAt: draft.StartedAt, + completedAt: draft.CompletedAt, } } @@ -2904,6 +2964,10 @@ func combineReviewerWorkstreamTotals(drafts []sessionDraft) workstreamTotals { allStandard := true anySpeed := false for _, draft := range drafts { + provenance := draftWorkstreamTotals(draft) + combined.requestHistories = append(combined.requestHistories, provenance.requestHistories...) + combined.blockLegacyEstimate = combined.blockLegacyEstimate || provenance.blockLegacyEstimate + combined.unknownRequestCost = combined.unknownRequestCost || provenance.unknownRequestCost combined.durationMS += draft.Response.DurationMS if combined.startedAt.IsZero() || !draft.StartedAt.IsZero() && draft.StartedAt.Before(combined.startedAt) { combined.startedAt = draft.StartedAt @@ -2991,6 +3055,14 @@ func runAdapter(inputs planRunInputs) string { return strings.TrimSpace(inputs.rollup.Adapter) } +func threadFailureSummaries(failures []threadanalysis.Failure) []reviewplan.ThreadAnalysisFailureSummary { + out := make([]reviewplan.ThreadAnalysisFailureSummary, 0, len(failures)) + for _, failure := range failures { + out = append(out, reviewplan.ThreadAnalysisFailureSummary{ThreadID: failure.ThreadID, Error: failure.Error}) + } + return out +} + func reviewerFailureSummaries(failures []ReviewerFailure) []reviewplan.ReviewerFailureSummary { out := make([]reviewplan.ReviewerFailureSummary, 0, len(failures)) for _, failure := range failures { @@ -3444,10 +3516,16 @@ func workstreamUsageFromTotalsForCatalog(name string, totals workstreamTotals, c CacheCreate1h: usage.CacheCreate1h, CostUSD: usage.CostUSD, } + // Collection-only: direct OpenAI history and legacy totals remain unpriced. + // A custom all-band price row cannot bypass request evidence gates. Codex + // retains reported costs but cannot borrow a direct-API estimate. + if totals.unknownRequestCost { + workstream.CostUSD = nil + } // When the adapter reports no cost (e.g. subscription auth), estimate it from // tokens at public list prices — only for models the price table knows, so an // agent's unpriced model leaves cost unavailable rather than wrong. - if workstream.CostUSD == nil { + if workstream.CostUSD == nil && !totals.blockLegacyEstimate { priceUsage := pricing.Usage{ TokensIn: usage.TokensIn, TokensOut: usage.TokensOut, CacheRead: usage.CacheRead, CacheCreate5m: usage.CacheCreate5m, CacheCreate1h: usage.CacheCreate1h, @@ -3598,6 +3676,9 @@ func validate(opts Options, req Request) error { if strings.TrimSpace(req.PRURL) == "" { return fmt.Errorf("pipeline: PR URL is required") } + if req.WithoutDiscussion && (strings.TrimSpace(req.ReviewBaseSHA) == "" || strings.TrimSpace(req.ReviewHeadSHA) == "") { + return fmt.Errorf("pipeline: review without discussion requires pinned review base and head SHAs") + } if strings.TrimSpace(runlifecycle.PostingKey(req.PostingIdentity)) == "" { return fmt.Errorf("pipeline: posting identity is required") } diff --git a/internal/pipeline/prompts.go b/internal/pipeline/prompts.go index f58e6e0b..a6001273 100644 --- a/internal/pipeline/prompts.go +++ b/internal/pipeline/prompts.go @@ -822,7 +822,7 @@ func selectionThreadStatus(thread threadcontext.Thread) string { } } -func buildRollupPrompt(pr gitprovider.PR, findings []review.Finding, reviewerFailures []ReviewerFailure, reviewerCoverage []reviewplan.ReviewerCoverageSummary) (string, error) { +func buildRollupPrompt(pr gitprovider.PR, findings []review.Finding, reviewerFailures []ReviewerFailure, reviewerCoverage []reviewplan.ReviewerCoverageSummary, threadFailures ...[]reviewplan.ThreadAnalysisFailureSummary) (string, error) { payload := map[string]any{ "task": "dedupe findings and return rollup JSON only", "output_contract": rollupOutputContract(findings), @@ -832,6 +832,9 @@ func buildRollupPrompt(pr gitprovider.PR, findings []review.Finding, reviewerFai "reviewer_failures": reviewerFailures, "reviewer_coverage": rollupCoveragePrompt(reviewerCoverage), } + if len(threadFailures) > 0 && len(threadFailures[0]) > 0 { + payload["thread_analysis_failures"] = rollupThreadAnalysisFailurePrompt(threadFailures[0]) + } body, err := json.Marshal(payload) if err != nil { return "", fmt.Errorf("pipeline: build rollup prompt: %w", err) @@ -839,6 +842,38 @@ func buildRollupPrompt(pr gitprovider.PR, findings []review.Finding, reviewerFai return string(body), nil } +type rollupThreadAnalysisFailureSummary struct { + Count int `json:"count"` + Examples []reviewplan.ThreadAnalysisFailureSummary `json:"examples"` + Omitted int `json:"omitted"` + EvidenceDigest string `json:"evidence_digest"` +} + +func rollupThreadAnalysisFailurePrompt(failures []reviewplan.ThreadAnalysisFailureSummary) rollupThreadAnalysisFailureSummary { + // Bind the complete evidence independently of the sampled prompt. A changed + // omitted failure must invalidate cached synthesis just like a visible one. + complete := append([]reviewplan.ThreadAnalysisFailureSummary(nil), failures...) + slices.SortFunc(complete, func(a, b reviewplan.ThreadAnalysisFailureSummary) int { + if order := strings.Compare(a.ThreadID, b.ThreadID); order != 0 { + return order + } + return strings.Compare(a.Error, b.Error) + }) + digest := sha256.New() + for _, failure := range complete { + // Length-prefix raw strings so even invalid UTF-8 repaired for display + // cannot make distinct complete evidence share a reuse fingerprint. + _, _ = fmt.Fprintf(digest, "%d:%s%d:%s", len(failure.ThreadID), failure.ThreadID, len(failure.Error), failure.Error) + } + examples := reviewplan.ThreadAnalysisFailureExamples(complete) + return rollupThreadAnalysisFailureSummary{ + Count: len(complete), + Examples: examples, + Omitted: len(complete) - len(examples), + EvidenceDigest: hex.EncodeToString(digest.Sum(nil)), + } +} + type rollupCoveragePromptEntry struct { AgentID string `json:"agent_id"` Status string `json:"status"` diff --git a/internal/pipeline/request_cost_evidence_test.go b/internal/pipeline/request_cost_evidence_test.go new file mode 100644 index 00000000..422865ba --- /dev/null +++ b/internal/pipeline/request_cost_evidence_test.go @@ -0,0 +1,50 @@ +package pipeline + +import ( + "testing" + + "github.com/open-cli-collective/codereview-cli/internal/llm" +) + +func TestDirectOpenAIAndCodexCannotUseLegacyAggregateEstimator(t *testing.T) { + catalog := pipelineCatalogWithRevision(t, "synthetic-request-cost-catalog") + in, out := 1_000_000, 1_000_000 + // The synthetic choice is deliberately priced by the legacy all-band + // catalog: runtime provenance, rather than a model-prefix guess, must block it. + for _, adapter := range []string{"openai_api", "codex_cli"} { + draft := sessionDraft{Adapter: adapter, Model: "claude-sonnet-5", Response: llm.Response{Usage: llm.Usage{TokensIn: &in, TokensOut: &out, Speed: "standard"}}} + got := workstreamUsageFromTotalsForCatalog("reviewer", draftWorkstreamTotals(draft), catalog) + if got.CostUSD != nil || got.CostEstimated { + t.Fatalf("%s gained legacy pricing: %#v", adapter, got) + } + } +} + +func TestRequestCostPrimaryRepairHistoriesStayDetachedAndUnpriced(t *testing.T) { + known := 0.01 + one := &llm.RequestCostEvidence{Version: 1, Source: "openai_responses", Scope: "task_artifact_history", TaskID: "primary", AttemptsObserved: 1, Attempts: []llm.RequestCostAttempt{{GenerationID: "one", Ordinal: 1}}} + two := llm.CloneRequestCostEvidence(one) + two.TaskID = "repair" + two.Gaps = []llm.CostEvidenceGap{llm.CostGapInterrupted} + drafts := []sessionDraft{ + {Adapter: "openai_api", Model: "first-model", Response: llm.Response{Usage: llm.Usage{CostUSD: &known}, RequestCostEvidence: one}}, + {Adapter: "openai_api", Model: "second-model", Response: llm.Response{RequestCostEvidence: two}}, + } + totals := combineReviewerWorkstreamTotals(drafts) + if len(totals.requestHistories) != 2 || totals.requestHistories[0].TaskID != "primary" || totals.requestHistories[1].TaskID != "repair" { + t.Fatal("task identities merged") + } + one.Attempts[0].GenerationID = "mutated" + if totals.requestHistories[0].Attempts[0].GenerationID != "one" { + t.Fatal("workstream history aliases source") + } + got := workstreamUsageFromTotalsForCatalog("reviewer", totals, nil) + if got.CostUSD != nil || got.CostEstimated { + t.Fatal("known subset masked unknown task") + } + // A duplicate reference is never sent through a price selector in this slice. + duplicate := combineReviewerWorkstreamTotals([]sessionDraft{drafts[0], drafts[0]}) + if workstreamUsageFromTotalsForCatalog("reviewer", duplicate, nil).CostUSD != nil { + t.Fatal("duplicate snapshot charged") + } +} diff --git a/internal/pipeline/thread_analysis_failure_test.go b/internal/pipeline/thread_analysis_failure_test.go new file mode 100644 index 00000000..e9fca2c5 --- /dev/null +++ b/internal/pipeline/thread_analysis_failure_test.go @@ -0,0 +1,159 @@ +package pipeline + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/open-cli-collective/codereview-cli/internal/gitprovider" + "github.com/open-cli-collective/codereview-cli/internal/ledger" + "github.com/open-cli-collective/codereview-cli/internal/llm" + "github.com/open-cli-collective/codereview-cli/internal/llmlifecycle" + "github.com/open-cli-collective/codereview-cli/internal/outbox" + "github.com/open-cli-collective/codereview-cli/internal/review" + "github.com/open-cli-collective/codereview-cli/internal/reviewplan" + "github.com/open-cli-collective/codereview-cli/internal/statepaths" +) + +func TestLiveThreadAnalysisFailureContinuesAndPosts(t *testing.T) { + for _, invalidOutput := range []bool{false, true} { + t.Run(fmt.Sprintf("invalid_output=%t", invalidOutput), func(t *testing.T) { + ctx := context.Background() + store := openPipelineStore(t) + defer closeStore(t, store) + provider, req := dryRunHarness(t) + provider.caps.ThreadResolution = true + for i := 1; i <= 3; i++ { + provider.threads = append(provider.threads, markedReviewThread(t, fmt.Sprintf("thread-%d", i), "main.go", 2, req.PostingIdentity, gitprovider.Identity{Login: "human", ID: "human-id"})) + } + run := allocateLiveRun(t, store, provider, req, "run-partial-thread-analysis") + adapter := &llm.FakeAdapter{NameValue: "fake-llm"} + queuePartialThreadReview(t, adapter, invalidOutput) + var warnings bytes.Buffer + result, err := liveForTest(ctx, Options{ + Provider: provider, Adapter: adapter, Store: store, + Layout: statepaths.NewLayout(t.TempDir(), t.TempDir()), Now: fixedNow, + NewSessionRowID: sequence("session"), NewFindingID: findingSequence("finding"), NewActionID: actionSequence(), + MaxConcurrency: 1, Warnings: &warnings, + }, req, run) + if err != nil { + t.Fatalf("Live: %v", err) + } + if result.Plan.Outcome != reviewplan.OutcomeComment || len(result.ThreadAnalysisFailures) != 1 || result.ThreadAnalysisFailures[0].ThreadID != "thread-2" { + t.Fatalf("plan/failures = %s/%#v, want completed non-approving partial review", result.Plan.Outcome, result.ThreadAnalysisFailures) + } + if len(result.ReviewerCoverage) != 1 || result.ReviewerCoverage[0].Status != reviewerCoverageCompleteBroad { + t.Fatalf("reviewer coverage = %#v, want reviewer phase to run", result.ReviewerCoverage) + } + for _, want := range []string{"thread-2", "not analyzed", "cached on resume", "--rerun"} { + if !strings.Contains(warnings.String(), want) || !strings.Contains(result.Plan.RollupMarkdown, want) { + t.Fatalf("missing diagnostic %q: warnings=%s rollup=%s", want, warnings.String(), result.Plan.RollupMarkdown) + } + } + requests := adapter.Requests() + if !strings.Contains(requests[len(requests)-1].Prompt, `"thread_analysis_failures"`) || !strings.Contains(requests[len(requests)-1].Prompt, `"thread_id":"thread-2"`) { + t.Fatalf("rollup prompt missing failure: %s", requests[len(requests)-1].Prompt) + } + for _, action := range result.PlannedActions { + if action.ThreadID == "thread-2" { + t.Fatalf("failed thread has planned action: %#v", action) + } + } + for i := 1; i <= 3; i++ { + meta, ok, err := llmlifecycle.ReadMetadata(lifecyclePaths(result.Artifacts), fmt.Sprintf("thread-analysis-thread-%d", i)) + want := llmlifecycle.StatusSucceeded + if i == 2 { + want = llmlifecycle.StatusFailedIsolated + } + if err != nil || !ok || meta.Status != want { + t.Fatalf("thread %d metadata = %#v ok=%t err=%v", i, meta, ok, err) + } + } + posting := &gitprovider.Fake{} + posting.SetCapabilities(provider.caps) + if err := posting.SetPR(req.PRRef, provider.pr); err != nil { + t.Fatal(err) + } + if err := posting.SetInlineThreads(req.PRRef, provider.threads); err != nil { + t.Fatal(err) + } + posted, err := outbox.Post(ctx, outbox.Options{Store: store, Provider: posting, Limiter: threadIsolationLimiter{}, Now: fixedNow}, outbox.Request{Run: run, PRRef: req.PRRef, PostingIdentity: req.PostingIdentity, DesiredOutcome: ledger.OutcomeComment}) + if err != nil || posted.ExitCode != 0 || posted.Outcome != ledger.OutcomeComment { + t.Fatalf("Post = %#v err=%v, want completed review", posted, err) + } + if replies := posting.RecordedThreadReplies(req.PRRef); len(replies) != 2 { + t.Fatalf("replies = %#v, want both successful siblings", replies) + } + resolved := posting.RecordedResolvedThreads(req.PRRef) + if len(resolved) != 2 { + t.Fatalf("resolutions = %#v, want successful siblings only", resolved) + } + for _, id := range resolved { + if id == "thread-2" { + t.Fatal("unanalyzed thread was resolved") + } + } + if reviews := posting.RecordedReviews(req.PRRef); len(reviews) != 1 || reviews[0].Event != review.ReviewEventComment || !strings.Contains(reviews[0].Body, "Unanalyzed Threads") { + t.Fatalf("posted reviews = %#v, want reported partial coverage", reviews) + } + }) + } +} + +func TestLiveResumeCachesIsolatedThreadAnalysisFailure(t *testing.T) { + ctx := context.Background() + store := openPipelineStore(t) + defer closeStore(t, store) + provider, req := dryRunHarness(t) + provider.threads = []gitprovider.InlineThread{markedReviewThread(t, "thread-2", "main.go", 2, req.PostingIdentity, gitprovider.Identity{Login: "human", ID: "human-id"})} + run := allocateLiveRun(t, store, provider, req, "run-resume-partial-thread-analysis") + adapter := &llm.FakeAdapter{NameValue: "fake-llm"} + adapter.Queue(fakeLLMResult("dossier", discussionSummaryJSON(nil, nil), 1, 1)) + adapter.Queue(fakeLLMResult("selection", selectionJSON("harness:reviewer", "main.go"), 1, 1)) + adapter.Queue(llm.FakeResult{SessionID: "failed-thread", WaitErr: errors.New("analysis provider failed")}) + adapter.Queue(fakeLLMResult("reviewer", findingsWithCoverageJSON("harness:reviewer", []string{"main.go"}, nil, nil, nil), 1, 1)) + adapter.Queue(llm.FakeResult{SessionID: "failed-rollup", WaitErr: errors.New("rollup provider failed")}) + opts := Options{Provider: provider, Adapter: adapter, Store: store, Layout: statepaths.NewLayout(t.TempDir(), t.TempDir()), Now: fixedNow, NewSessionRowID: sequence("session"), NewFindingID: findingSequence("finding"), NewActionID: actionSequence(), MaxConcurrency: 1} + if _, err := liveForTest(ctx, opts, req, run); err == nil || ClassifyFailure(err) != FailureDurableBlocking { + t.Fatalf("first Live error = %v, want blocking rollup failure", err) + } + resumed := &llm.FakeAdapter{NameValue: "fake-llm"} + resumed.Queue(fakeLLMResult("recovered-rollup", rollupJSON("approve", nil), 1, 1)) + opts.Adapter = resumed + opts.NewSessionRowID = sequence("resumed-session") + result, err := liveForTest(ctx, opts, req, run) + if err != nil { + t.Fatalf("resumed Live: %v", err) + } + if len(resumed.Requests()) != 1 || !strings.Contains(resumed.Requests()[0].Prompt, `"schema":"rollup"`) { + t.Fatalf("resume requests = %#v, want only blocking rollup retry", resumed.Requests()) + } + if len(result.ThreadAnalysisFailures) != 1 || result.Plan.Outcome != reviewplan.OutcomeComment || !strings.Contains(result.Plan.RollupMarkdown, "thread-2") { + t.Fatalf("resumed partial outcome = %#v/%s", result.ThreadAnalysisFailures, result.Plan.Outcome) + } +} + +func queuePartialThreadReview(t *testing.T, adapter *llm.FakeAdapter, invalidOutput bool) { + t.Helper() + adapter.Queue(fakeLLMResult("dossier", discussionSummaryJSON(nil, nil), 1, 1)) + selection := strings.Replace(selectionJSON("harness:reviewer", "main.go"), `"thread_actions": []`, `"thread_actions": [{"thread_id":"thread-2","decision":"summarize_and_resolve","summary":"selector proposed closure","safe_to_resolve_rationale":"settled"}]`, 1) + adapter.Queue(fakeLLMResult("selection", selection, 1, 1)) + adapter.Queue(fakeLLMResult("thread-1", `{"thread_id":"thread-1","decision":"summarize","summary":"First thread settled","resolve":true,"rationale":"settled"}`, 1, 1)) + if invalidOutput { + for i := 0; i < 2; i++ { + adapter.Queue(fakeLLMResult(fmt.Sprintf("thread-2-attempt-%d", i), `{"thread_id":"thread-2","decision":"invalid","resolve":false}`, 1, 1)) + } + } else { + adapter.Queue(llm.FakeResult{SessionID: "thread-2-failed", WaitErr: errors.New("thread provider failed")}) + } + adapter.Queue(fakeLLMResult("thread-3", `{"thread_id":"thread-3","decision":"summarize","summary":"Third thread settled","resolve":true,"rationale":"settled"}`, 1, 1)) + adapter.Queue(fakeLLMResult("reviewer", findingsWithCoverageJSON("harness:reviewer", []string{"main.go"}, nil, nil, nil), 1, 1)) + adapter.Queue(fakeLLMResult("rollup", rollupJSON("approve", nil), 1, 1)) +} + +type threadIsolationLimiter struct{} + +func (threadIsolationLimiter) Wait(context.Context, string) error { return nil } diff --git a/internal/pipeline/thread_analysis_prompt_test.go b/internal/pipeline/thread_analysis_prompt_test.go new file mode 100644 index 00000000..3157eae2 --- /dev/null +++ b/internal/pipeline/thread_analysis_prompt_test.go @@ -0,0 +1,126 @@ +package pipeline + +import ( + "encoding/json" + "fmt" + "reflect" + "unicode/utf8" + + "strings" + "testing" + + "github.com/open-cli-collective/codereview-cli/internal/gitprovider" + "github.com/open-cli-collective/codereview-cli/internal/llmlifecycle" + "github.com/open-cli-collective/codereview-cli/internal/reviewplan" +) + +func TestThreadAnalysisFailuresChangeSynthesisFingerprint(t *testing.T) { + pr := gitprovider.PR{Title: "fixture"} + base, err := buildRollupPrompt(pr, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + empty, err := buildRollupPrompt(pr, nil, nil, nil, nil) + if err != nil || empty != base || strings.Contains(base, "thread_analysis_failures") { + t.Fatalf("no-failure prompt changed: base=%s empty=%s err=%v", base, empty, err) + } + fingerprint := func(prompt string) string { + return llmlifecycle.Fingerprint("fake", orchestratorRollupStage, "rollup", "model", "effort", prompt, nil) + } + seen := map[string]bool{fingerprint(base): true} + for _, failure := range []reviewplan.ThreadAnalysisFailureSummary{ + {ThreadID: "thread-1", Error: "structured output remained invalid after retry"}, + {ThreadID: "thread-2", Error: "structured output remained invalid after retry"}, + {ThreadID: "thread-1", Error: "provider execution failed; see local task artifacts for details"}, + } { + prompt, err := buildRollupPrompt(pr, nil, nil, nil, []reviewplan.ThreadAnalysisFailureSummary{failure}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(prompt, failure.ThreadID) || !strings.Contains(prompt, failure.Error) || !strings.Contains(prompt, "thread_analysis_failures") { + t.Fatalf("prompt missing failure evidence: %s", prompt) + } + fp := fingerprint(prompt) + if seen[fp] { + t.Fatalf("failure evidence did not change fingerprint: %#v", failure) + } + seen[fp] = true + } +} + +func TestThreadFailureSynthesisPromptBoundsExamplesAndBindsOmittedEvidence(t *testing.T) { + failures := make([]reviewplan.ThreadAnalysisFailureSummary, 2500) + for i := range failures { + failures[i] = reviewplan.ThreadAnalysisFailureSummary{ThreadID: fmt.Sprintf("thread-%04d", i), Error: strings.Repeat("界", 800)} + } + original := append([]reviewplan.ThreadAnalysisFailureSummary(nil), failures...) + prompt, err := buildRollupPrompt(gitprovider.PR{}, nil, nil, nil, failures) + if err != nil { + t.Fatal(err) + } + var payload struct { + Failures rollupThreadAnalysisFailureSummary `json:"thread_analysis_failures"` + } + if err := json.Unmarshal([]byte(prompt), &payload); err != nil { + t.Fatal(err) + } + if payload.Failures.Count != 2500 || payload.Failures.Omitted != 2495 || len(payload.Failures.Examples) != 5 || len(payload.Failures.EvidenceDigest) != 64 { + t.Fatalf("failure prompt summary = %#v", payload.Failures) + } + if len(prompt) > 12000 || strings.Contains(prompt, "thread-0005") || strings.Contains(prompt, "thread-2499") || !utf8.ValidString(prompt) { + t.Fatalf("failure prompt bytes=%d, want bounded examples", len(prompt)) + } + if !reflect.DeepEqual(failures, original) { + t.Fatal("prompt sampling mutated complete failure collection") + } + changed := append([]reviewplan.ThreadAnalysisFailureSummary(nil), failures...) + changed[5].ThreadID = "zz-omitted-thread-changed" + changedPrompt, err := buildRollupPrompt(gitprovider.PR{}, nil, nil, nil, changed) + if err != nil { + t.Fatal(err) + } + var changedPayload struct { + Failures rollupThreadAnalysisFailureSummary `json:"thread_analysis_failures"` + } + if err := json.Unmarshal([]byte(changedPrompt), &changedPayload); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(payload.Failures.Examples, changedPayload.Failures.Examples) { + t.Fatal("omitted evidence mutation unexpectedly changed visible examples") + } + fingerprint := func(value string) string { + return llmlifecycle.Fingerprint("fake", orchestratorRollupStage, "rollup", "model", "effort", value, []string{"coverage-artifact=current", "relocation-manifest=current", "context-contract=current"}) + } + if payload.Failures.EvidenceDigest == changedPayload.Failures.EvidenceDigest || fingerprint(prompt) == fingerprint(changedPrompt) { + t.Fatal("changed omitted failure did not invalidate complete-evidence fingerprint") + } + changed = append([]reviewplan.ThreadAnalysisFailureSummary(nil), failures...) + changed[5].Error = "provider execution failed; see local task artifacts for details" + changedPrompt, err = buildRollupPrompt(gitprovider.PR{}, nil, nil, nil, changed) + if err != nil { + t.Fatal(err) + } + if fingerprint(prompt) == fingerprint(changedPrompt) { + t.Fatal("changed omitted diagnostic did not invalidate fingerprint") + } + reordered := append([]reviewplan.ThreadAnalysisFailureSummary(nil), failures...) + for i, j := 0, len(reordered)-1; i < j; i, j = i+1, j-1 { + reordered[i], reordered[j] = reordered[j], reordered[i] + } + reorderedPrompt, err := buildRollupPrompt(gitprovider.PR{}, nil, nil, nil, reordered) + if err != nil || reorderedPrompt != prompt { + t.Fatalf("failure evidence digest is not deterministic across order, err=%v", err) + } +} + +func TestThreadFailureDigestBindsRawEvidenceBeforeDisplayRepair(t *testing.T) { + first := []reviewplan.ThreadAnalysisFailureSummary{{ThreadID: "thread-" + string([]byte{0xff}), Error: "provider execution failed"}} + second := []reviewplan.ThreadAnalysisFailureSummary{{ThreadID: "thread-�", Error: "provider execution failed"}} + firstSummary, secondSummary := rollupThreadAnalysisFailurePrompt(first), rollupThreadAnalysisFailurePrompt(second) + if !reflect.DeepEqual(firstSummary.Examples, secondSummary.Examples) { + t.Fatal("display repair should produce identical safe examples") + } + if firstSummary.EvidenceDigest == secondSummary.EvidenceDigest { + t.Fatal("distinct raw evidence shared a complete-evidence digest after display repair") + } +} diff --git a/internal/pipeline/without_discussion_relocation_test.go b/internal/pipeline/without_discussion_relocation_test.go new file mode 100644 index 00000000..251df2c5 --- /dev/null +++ b/internal/pipeline/without_discussion_relocation_test.go @@ -0,0 +1,368 @@ +package pipeline + +import ( + "context" + "encoding/json" + "fmt" + "os" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/open-cli-collective/codereview-cli/internal/dossier" + "github.com/open-cli-collective/codereview-cli/internal/ledger" + "github.com/open-cli-collective/codereview-cli/internal/llm" + "github.com/open-cli-collective/codereview-cli/internal/llmlifecycle" + "github.com/open-cli-collective/codereview-cli/internal/reviewplan" + "github.com/open-cli-collective/codereview-cli/internal/runartifact" + "github.com/open-cli-collective/codereview-cli/internal/symlinkmetadata" +) + +func TestPinnedWithoutDiscussionRelocationsRepairWithoutPRSessions(t *testing.T) { + ctx := context.Background() + store := openPipelineStore(t) + defer closeStore(t, store) + provider, req, contextPath := newRelocationFixture(t, 2) + req.ReviewBaseSHA, req.ReviewHeadSHA = provider.pr.Base.SHA, provider.pr.Head.SHA + req.WithoutDiscussion = true + provider.diffBetween = provider.diff + provider.pr.Title = "Offline relocation replay" + provider.pr.Body = "Keep the pinned relocation change visible." + addExistingDiscussion(provider, req.PostingIdentity) + sessionName, cohortScope := storePriorSessions(ctx, t, store, provider, req) + cohort, err := store.GetReviewerCohort(ctx, cohortScope) + if err != nil { + t.Fatalf("seeded cohort: %v", err) + } + cohort.Members[0].AgentID = "harness:relocation" + cohort.Members[0].Files = []string{"apps/components/shared/file-0000.go", "apps/components/shared/file-0001.go", "src/router.go", "scripts/build.sh", "workspace.yaml"} + if err := store.ReplaceReviewerCohort(ctx, cohort); err != nil { + t.Fatalf("seed relocation cohort: %v", err) + } + observedStore := &offlinePRSessionStore{Store: store, calls: map[string]int{}} + adapter := &offlineRelocationAdapter{ + relocationWorkspaceAdapter: &relocationWorkspaceAdapter{ + FakeAdapter: &llm.FakeAdapter{NameValue: "fake-llm", SupportsResumeValue: true}, + contextPath: contextPath, omitPrimary: "src/router.go", repairResolve: true, + }, + gitCommand: workbenchGitCommandForTest(req.PRRef, provider.fixtureRepoDir), + } + opts := withoutDiscussionOptions(t, provider, adapter, store, "run-offline-relocation-repair") + opts.Store, opts.NamedSessions = observedStore, observedStore + opts.KeepWorkbench = true + withDiscussion := allocateDryRunForSHAs(t, store, opts.Layout, req, "run-pinned-with-discussion", req.ReviewHeadSHA, req.ReviewBaseSHA, fixedNow().Add(-time.Minute)) + + result, err := dryRunForTest(ctx, opts, req) + if err != nil { + t.Fatalf("DryRun: %v", err) + } + if result.Run.RunID == withDiscussion.RunID || result.Run.RunID != "run-offline-relocation-repair" { + t.Fatalf("run = %q, want fresh discussion-free run", result.Run.RunID) + } + withoutDiscussion := allocateDryRunForSHAs(t, store, opts.Layout, req, "run-pinned-without-discussion", req.ReviewHeadSHA, req.ReviewBaseSHA, fixedNow().Add(-time.Second)) + if err := runartifact.WriteMarkerWithOptions(withoutDiscussion.ArtifactPath, runartifact.KindReview, withoutDiscussion.RunID, runartifact.MarkerOptions{WithoutDiscussion: true}); err != nil { + t.Fatalf("write discussion-free incomplete marker: %v", err) + } + for _, discussionFree := range []bool{false, true} { + resumeReq := req + resumeReq.WithoutDiscussion = discussionFree + wantRun := withDiscussion.RunID + if discussionFree { + wantRun = withoutDiscussion.RunID + } + resumed, found, err := findIncompleteDryRun(ctx, observedStore, resumeReq, provider.pr) + if err != nil || !found || resumed.RunID != wantRun { + t.Fatalf("discussion-free %t resume = %q found %t error %v, want same-mode %q", discussionFree, resumed.RunID, found, err, wantRun) + } + } + if !result.WithoutDiscussion || result.NamedSessionCandidate != nil || result.Plan.Outcome != reviewplan.OutcomeApproved { + t.Fatalf("replay result = flag %t named session %#v outcome %q", result.WithoutDiscussion, result.NamedSessionCandidate, result.Plan.Outcome) + } + if provider.threadCalls != 0 || provider.reviewCalls != 0 || provider.issueCommentCalls != 0 { + t.Fatalf("discussion reads = %d/%d/%d, want none", provider.threadCalls, provider.reviewCalls, provider.issueCommentCalls) + } + if len(result.ReviewerFailures) != 0 || len(result.ReviewerCoverage) != 1 { + t.Fatalf("reviewer failures/coverage = %#v/%#v", result.ReviewerFailures, result.ReviewerCoverage) + } + coverage := result.ReviewerCoverage[0] + if coverage.Status != reviewerCoverageCompleteConstrained || len(coverage.RelocationReviewedFiles) != 2 || len(coverage.MissingFiles) != 0 || len(coverage.SkippedFiles) != 0 { + t.Fatalf("combined relocation/repair coverage = %#v, want both moves and complete residual coverage", coverage) + } + if !slices.Contains(coverage.InspectedFiles, "src/router.go") || !slices.Contains(coverage.ContextFiles, contextPath) { + t.Fatalf("coverage omitted repaired residual or read context: %#v", coverage) + } + requests := adapter.Requests() + counts := map[string]int{} + var reviewerRequests []llm.Request + for _, request := range requests { + var prompt struct { + Schema string `json:"schema"` + Task string `json:"task"` + } + if err := json.Unmarshal([]byte(request.Prompt), &prompt); err != nil { + t.Fatalf("decode captured prompt: %v", err) + } + counts[prompt.Schema]++ + if prompt.Schema == "findings" { + reviewerRequests = append(reviewerRequests, request) + if strings.Contains(prompt.Task, "coverage repair") { + counts["repair"]++ + } + } + for _, marker := range append(append([]string(nil), discussionMarkers...), "discussion_outcomes") { + if strings.Contains(request.Prompt, marker) { + t.Fatalf("%s prompt leaked discussion %q", prompt.Schema, marker) + } + } + } + if len(requests) != 4 || counts["selection"] != 1 || counts["findings"] != 2 || counts["repair"] != 1 || counts["rollup"] != 1 { + t.Fatalf("task calls = %#v (%d total), want selection, primary, one repair, rollup", counts, len(requests)) + } + if !strings.Contains(requests[0].Prompt, provider.pr.Title) || !strings.Contains(requests[0].Prompt, provider.pr.Body) { + t.Fatal("selection lost the pinned PR title or body") + } + adapter.resumeMu.Lock() + resumes := append([]llm.ResumeRequest(nil), adapter.resumes...) + adapter.resumeMu.Unlock() + if len(resumes) != 1 || resumes[0].SessionID != "relocation-findings" { + t.Fatalf("resumes = %#v, want only repair resuming this invocation's primary session", resumes) + } + for _, resume := range resumes { + if resume.SessionID == priorOrchestratorSession || resume.SessionID == priorReviewerSession { + t.Fatalf("resumed seeded PR session %q", resume.SessionID) + } + } + root, err := os.OpenRoot(result.Artifacts.Dir) + if err != nil { + t.Fatalf("open artifact root: %v", err) + } + defer root.Close() + var manifest relocationManifest + manifestBytes, err := root.ReadFile("relocations.json") + if err != nil { + t.Fatalf("read relocation artifact: %v", err) + } + if err := json.Unmarshal(manifestBytes, &manifest); err != nil { + t.Fatalf("decode relocation artifact: %v", err) + } + var artifact coverageArtifact + coverageBytes, err := root.ReadFile("coverage.json") + if err != nil { + t.Fatalf("read coverage artifact: %v", err) + } + if err := json.Unmarshal(coverageBytes, &artifact); err != nil { + t.Fatalf("decode coverage artifact: %v", err) + } + if len(manifest.Moves) != 2 || manifest.Digest == "" || manifest.BaseSHA != req.ReviewBaseSHA || manifest.HeadSHA != req.ReviewHeadSHA { + t.Fatalf("pinned relocation manifest = %#v", manifest) + } + if artifact.ManifestDigest != manifest.Digest || !slices.Equal(artifact.Moves, manifest.Moves) || len(artifact.Reviewers) != 1 || len(artifact.Assessments) != 1 || len(artifact.Failures) != 0 { + t.Fatalf("complete coverage artifact = %#v", artifact) + } + if !slices.Equal(artifact.Reviewers[0].RelocationReviewedFiles, coverage.RelocationReviewedFiles) || !slices.Equal(artifact.Reviewers[0].InspectedFiles, coverage.InspectedFiles) || !slices.Equal(artifact.Reviewers[0].ContextFiles, coverage.ContextFiles) || len(artifact.Reviewers[0].MissingFiles) != 0 { + t.Fatalf("artifact lost combined coverage collections: %#v", artifact.Reviewers[0]) + } + if len(artifact.Assessments[0].Assessments) != 1 || !artifact.Assessments[0].Assessments[0].Valid || len(artifact.Assessments[0].Assessments[0].ReviewedFiles) != 2 { + t.Fatalf("artifact lost valid complete move assessment: %#v", artifact.Assessments) + } + marker, err := runartifact.ReadMarker(result.Artifacts.Dir, runartifact.KindReview) + if err != nil || !marker.WithoutDiscussion { + t.Fatalf("review marker = %#v, %v", marker, err) + } + summary, err := dossier.ReadDiscussionSummary(result.Artifacts) + if err != nil || !summary.WithoutDiscussion || len(summary.TopLevelComments) != 0 || len(summary.InlineThreads) != 0 { + t.Fatalf("discussion summary = %#v, %v", summary, err) + } + for _, marker := range discussionMarkers { + if strings.Contains(dossierText(t, result.Artifacts.DossierDir), marker) { + t.Fatalf("dossier leaked discussion %q", marker) + } + } + var symlinks symlinkmetadata.Artifact + symlinkBytes, err := root.ReadFile("symlink-metadata.json") + if err != nil { + t.Fatalf("read symlink artifact: %v", err) + } + if err := json.Unmarshal(symlinkBytes, &symlinks); err != nil || symlinks.Digest == "" || symlinks.BaseSHA != req.ReviewBaseSHA || symlinks.HeadSHA != req.ReviewHeadSHA { + t.Fatalf("pinned symlink metadata = %#v, %v", symlinks, err) + } + for _, request := range reviewerRequests { + workspace := request.ReviewerWorkspace + if workspace == nil || workspace.SymlinkMetadataPath != result.Artifacts.SymlinkMetadataJSON || workspace.SymlinkMetadataDigest != symlinks.Digest || workspace.BaseSHA != symlinks.BaseSHA || workspace.HeadSHA != symlinks.HeadSHA { + t.Fatalf("offline primary/repair lost pinned symlink identity: %#v", workspace) + } + } + assertOfflineRelocationFingerprints(ctx, t, opts, result, reviewerRequests, manifest.Digest, symlinks.Digest) + observedStore.mu.Lock() + for method, calls := range observedStore.calls { + if calls != 0 { + t.Errorf("PR-scoped %s calls = %d, want zero", method, calls) + } + } + observedStore.mu.Unlock() + storedSession, err := store.GetNamedSession(ctx, sessionName) + if err != nil || storedSession.ProviderSessionID != priorOrchestratorSession { + t.Fatalf("seeded orchestrator session changed: %#v, %v", storedSession, err) + } + storedCohort, err := store.GetReviewerCohort(ctx, cohortScope) + if err != nil || len(storedCohort.Members) != 1 || storedCohort.Members[0].ProviderSessionID != priorReviewerSession || !slices.Equal(storedCohort.Members[0].Files, cohort.Members[0].Files) { + t.Fatalf("seeded reviewer cohort changed: %#v, %v", storedCohort, err) + } +} + +func assertOfflineRelocationFingerprints(ctx context.Context, t *testing.T, opts Options, result Result, requests []llm.Request, manifestDigest, symlinkDigest string) { + t.Helper() + core, err := loadDossierPromptCore(result.Artifacts) + if err != nil { + t.Fatalf("load fingerprint inputs: %v", err) + } + for i, request := range requests { + var prompt struct { + Assignment reviewerPromptAssignment `json:"assignment"` + FileManifest promptFileManifest `json:"file_manifest"` + } + if err := json.Unmarshal([]byte(request.Prompt), &prompt); err != nil { + t.Fatalf("decode reviewer assignment: %v", err) + } + taskID := reviewerTaskID(prompt.Assignment.AgentID) + wantDeps := []string{orchestratorSelectionStage} + wantMoves := 2 + if i == 1 { + taskID = reviewerCoverageRepairTaskID(prompt.Assignment.AgentID) + wantDeps = []string{reviewerTaskID(prompt.Assignment.AgentID)} + wantMoves = 0 + if !slices.Equal(promptManifestPathsAtIndices(t, prompt.FileManifest, prompt.Assignment.ScopeIndices), []string{"src/router.go"}) { + t.Fatalf("repair scope = %#v, want only omitted residual", prompt.Assignment) + } + } + if prompt.Assignment.ManifestDigest != manifestDigest || prompt.Assignment.AssignmentDigest == "" || prompt.Assignment.RelocationCount != wantMoves { + t.Fatalf("task %s relocation assignment = %#v", taskID, prompt.Assignment) + } + meta, ok, err := llmlifecycle.ReadMetadata(lifecyclePaths(result.Artifacts), taskID) + if err != nil || !ok || meta.SchemaVersion != llmlifecycle.SchemaVersion || meta.Status != llmlifecycle.StatusSucceeded || !slices.Equal(meta.DependencyTaskIDs, wantDeps) { + t.Fatalf("task metadata = %#v, found %t, error %v", meta, ok, err) + } + deps := append(append([]string(nil), wantDeps...), core.Dependencies...) + deps = append(deps, "relocation-manifest="+prompt.Assignment.ManifestDigest, "relocation-assignment="+prompt.Assignment.AssignmentDigest, "symlink-metadata="+symlinkDigest, "symlink-contract="+symlinkInspectionContractVersion, "context-contract="+reviewerContextContractVersion, "without_discussion=true") + fingerprint := func(values []string) string { + return llmlifecycle.Fingerprint(opts.Adapter.Name(), taskID, meta.Phase, request.Model, request.Effort, request.Prompt, values) + } + if meta.InputFingerprint != fingerprint(deps) { + t.Fatalf("task %s fingerprint = %q, want all relocation/symlink/context/discussion dependencies %q", taskID, meta.InputFingerprint, fingerprint(deps)) + } + cacheReq := llmlifecycle.Request{Store: opts.Store, Adapter: opts.Adapter, RunID: result.Run.RunID, TaskID: taskID, Phase: meta.Phase, DependencyTaskIDs: wantDeps, InputFingerprint: meta.InputFingerprint, Paths: lifecyclePaths(result.Artifacts), Model: request.Model, Effort: request.Effort, Prompt: request.Prompt} + decode := func(data []byte) (json.RawMessage, error) { return append(json.RawMessage(nil), data...), nil } + if _, loaded, err := llmlifecycle.LoadStructured(ctx, cacheReq, decode); err != nil || !loaded { + t.Fatalf("unchanged %s cache = loaded %t, error %v", taskID, loaded, err) + } + for _, prefix := range []string{"relocation-manifest=", "relocation-assignment=", "symlink-metadata=", "symlink-contract=", "context-contract=", "without_discussion="} { + changed := append([]string(nil), deps...) + for index, dep := range changed { + if strings.HasPrefix(dep, prefix) { + changed[index] = prefix + "changed" + } + } + cacheReq.InputFingerprint = fingerprint(changed) + if _, _, err := llmlifecycle.LoadStructured(ctx, cacheReq, decode); err == nil || !strings.Contains(err.Error(), "input fingerprint changed") { + t.Fatalf("%s cache accepted changed %s dependency: %v", taskID, prefix, err) + } + } + cacheReq.InputFingerprint = meta.InputFingerprint + for _, version := range []int{1, 2, 3} { + legacy := meta + legacy.SchemaVersion = version + if err := llmlifecycle.WriteMetadata(cacheReq.Paths, legacy); err != nil { + t.Fatalf("write schema-v%d cache: %v", version, err) + } + if _, _, err := llmlifecycle.LoadStructured(ctx, cacheReq, decode); err == nil || !strings.Contains(err.Error(), fmt.Sprintf("schema version = %d", version)) { + t.Fatalf("%s cache accepted schema-v%d metadata: %v", taskID, version, err) + } + } + if err := llmlifecycle.WriteMetadata(cacheReq.Paths, meta); err != nil { + t.Fatalf("restore current metadata: %v", err) + } + } +} + +// Resume runs the same dynamic relocation response as Start. Only the focused +// repair may resume the fresh primary session created by this invocation. +type offlineRelocationAdapter struct { + *relocationWorkspaceAdapter + gitCommand func(context.Context, string, ...string) ([]byte, error) + resumeMu sync.Mutex + resumes []llm.ResumeRequest +} + +func (a *offlineRelocationAdapter) Start(ctx context.Context, req llm.Request) (llm.Stream, error) { + if workspace := req.ReviewerWorkspace; workspace != nil { + if !workspace.NoNetwork { + return nil, fmt.Errorf("offline relocation reviewer has network tools enabled") + } + remotes, err := a.gitCommand(ctx, workspace.RepoDir, "remote") + if err != nil { + return nil, fmt.Errorf("list offline relocation reviewer remotes: %w", err) + } + if strings.TrimSpace(string(remotes)) != "" { + return nil, fmt.Errorf("offline relocation reviewer remotes = %q, want none", remotes) + } + } + return a.relocationWorkspaceAdapter.Start(ctx, req) +} + +func (a *offlineRelocationAdapter) Resume(ctx context.Context, sessionID string, req llm.Request) (llm.Stream, error) { + a.resumeMu.Lock() + a.resumes = append(a.resumes, llm.ResumeRequest{SessionID: sessionID, Request: req}) + a.resumeMu.Unlock() + return a.Start(ctx, req) +} + +// Record every PR-scoped operation separately from allowed run-owned ledger +// sessions, so an empty or incompatible saved cohort cannot hide a lookup. +type offlinePRSessionStore struct { + *ledger.Store + mu sync.Mutex + calls map[string]int +} + +func (s *offlinePRSessionStore) record(method string) { + s.mu.Lock() + defer s.mu.Unlock() + s.calls[method]++ +} + +func (s *offlinePRSessionStore) GetNamedSession(ctx context.Context, name string) (ledger.NamedSession, error) { + s.record("GetNamedSession") + return s.Store.GetNamedSession(ctx, name) +} + +func (s *offlinePRSessionStore) UpsertNamedSession(ctx context.Context, session ledger.NamedSession) error { + s.record("UpsertNamedSession") + return s.Store.UpsertNamedSession(ctx, session) +} + +func (s *offlinePRSessionStore) DeleteNamedSession(ctx context.Context, name string) error { + s.record("DeleteNamedSession") + return s.Store.DeleteNamedSession(ctx, name) +} + +func (s *offlinePRSessionStore) GetReviewerCohort(ctx context.Context, scope ledger.ReviewerCohortScope) (ledger.ReviewerCohort, error) { + s.record("GetReviewerCohort") + return s.Store.GetReviewerCohort(ctx, scope) +} + +func (s *offlinePRSessionStore) ReplaceReviewerCohort(ctx context.Context, cohort ledger.ReviewerCohort) error { + s.record("ReplaceReviewerCohort") + return s.Store.ReplaceReviewerCohort(ctx, cohort) +} + +func (s *offlinePRSessionStore) UpdateReviewerCohortSession(ctx context.Context, scope ledger.ReviewerCohortScope, agentID, sessionID string, at time.Time) error { + s.record("UpdateReviewerCohortSession") + return s.Store.UpdateReviewerCohortSession(ctx, scope, agentID, sessionID, at) +} + +func (s *offlinePRSessionStore) DeleteReviewerCohort(ctx context.Context, scope ledger.ReviewerCohortScope) error { + s.record("DeleteReviewerCohort") + return s.Store.DeleteReviewerCohort(ctx, scope) +} diff --git a/internal/pipeline/without_discussion_symlink_test.go b/internal/pipeline/without_discussion_symlink_test.go new file mode 100644 index 00000000..ade299b5 --- /dev/null +++ b/internal/pipeline/without_discussion_symlink_test.go @@ -0,0 +1,112 @@ +package pipeline + +import ( + "context" + "encoding/json" + "os" + "slices" + "testing" + + "github.com/open-cli-collective/codereview-cli/internal/llm" + "github.com/open-cli-collective/codereview-cli/internal/reviewplan" + "github.com/open-cli-collective/codereview-cli/internal/symlinkmetadata" +) + +func TestPinnedWithoutDiscussionPreservesSymlinkInspectionObligations(t *testing.T) { + for _, tc := range []struct { + name string + kind string + readBody bool + wantOutcome reviewplan.Outcome + }{ + {name: "head symlink payload", kind: "symlink", wantOutcome: reviewplan.OutcomeApproved}, + {name: "historical metadata is not head body", kind: "symlink-to-regular", wantOutcome: reviewplan.OutcomeComment}, + {name: "transition head body inspected", kind: "symlink-to-regular", readBody: true, wantOutcome: reviewplan.OutcomeApproved}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + store := openPipelineStore(t) + defer closeStore(t, store) + provider, req, contextPath, linkPath, residualPath := newPatchlessRenameFixture(t, tc.kind) + req.ReviewBaseSHA, req.ReviewHeadSHA = provider.pr.Base.SHA, provider.pr.Head.SHA + req.WithoutDiscussion = true + provider.diffBetween = provider.diff + addExistingDiscussion(provider, req.PostingIdentity) + sessionName, cohortScope := storePriorSessions(ctx, t, store, provider, req) + observedStore := &offlinePRSessionStore{Store: store, calls: map[string]int{}} + adapter := &offlineRelocationAdapter{ + relocationWorkspaceAdapter: &relocationWorkspaceAdapter{ + FakeAdapter: &llm.FakeAdapter{NameValue: "fake-llm", SupportsResumeValue: true}, + contextPath: contextPath, inspectSymlink: true, readBaseOnlyBody: tc.readBody, + }, + gitCommand: workbenchGitCommandForTest(req.PRRef, provider.fixtureRepoDir), + } + opts := withoutDiscussionOptions(t, provider, adapter, store, "run-offline-symlink") + opts.Store, opts.NamedSessions = observedStore, observedStore + result, err := dryRunForTest(ctx, opts, req) + if err != nil { + t.Fatalf("DryRun: %v", err) + } + if !result.WithoutDiscussion || result.NamedSessionCandidate != nil || result.Plan.Outcome != tc.wantOutcome || len(result.ReviewerCoverage) != 1 || len(result.ReviewerFailures) != 0 { + t.Fatalf("offline symlink result = %#v, want outcome %q and one reviewer without failures", result, tc.wantOutcome) + } + coverage := result.ReviewerCoverage[0] + wantInspected := tc.kind == "symlink" || tc.readBody + if slices.Contains(coverage.InspectedFiles, linkPath) != wantInspected || !slices.Contains(coverage.InspectedFiles, residualPath) || len(coverage.RelocationReviewedFiles) != 0 { + t.Fatalf("offline symlink inspection coverage = %#v", coverage) + } + if !wantInspected && (!slices.Contains(coverage.SkippedFiles, linkPath) || !slices.Contains(coverage.MissingFiles, linkPath)) { + t.Fatalf("historical symlink metadata cleared the head-body obligation: %#v", coverage) + } + adapter.mu.Lock() + metadataReads := append([]string(nil), adapter.metadataReads...) + bodyReads := append([]string(nil), adapter.bodyReads...) + adapter.mu.Unlock() + if !slices.Contains(metadataReads, linkPath) || slices.Contains(bodyReads, linkPath) != tc.readBody { + t.Fatalf("offline symlink read modes = metadata %#v/body %#v", metadataReads, bodyReads) + } + root, err := os.OpenRoot(result.Artifacts.Dir) + if err != nil { + t.Fatal(err) + } + defer root.Close() + data, err := root.ReadFile("symlink-metadata.json") + if err != nil { + t.Fatal(err) + } + var metadata symlinkmetadata.Artifact + if err := json.Unmarshal(data, &metadata); err != nil || metadata.Digest == "" || len(metadata.Links) != 1 || metadata.Links[0].Path != linkPath { + t.Fatalf("offline pinned metadata = %#v, error %v", metadata, err) + } + var reviewers int + for _, request := range adapter.Requests() { + workspace := request.ReviewerWorkspace + if workspace == nil { + continue + } + reviewers++ + if !workspace.NoNetwork || workspace.SymlinkMetadataPath != result.Artifacts.SymlinkMetadataJSON || workspace.SymlinkMetadataDigest != metadata.Digest || workspace.BaseSHA != req.ReviewBaseSHA || workspace.HeadSHA != req.ReviewHeadSHA { + t.Fatalf("offline reviewer lost metadata identity: %#v", workspace) + } + } + if reviewers == 0 || provider.threadCalls != 0 || provider.reviewCalls != 0 || provider.issueCommentCalls != 0 { + t.Fatalf("reviewers %d, discussion reads %d/%d/%d", reviewers, provider.threadCalls, provider.reviewCalls, provider.issueCommentCalls) + } + observedStore.mu.Lock() + for method, count := range observedStore.calls { + if count != 0 { + t.Errorf("PR-scoped %s calls = %d, want zero", method, count) + } + } + observedStore.mu.Unlock() + session, err := store.GetNamedSession(ctx, sessionName) + if err != nil || session.ProviderSessionID != priorOrchestratorSession { + t.Fatalf("saved orchestrator session changed: %#v, %v", session, err) + } + cohort, err := store.GetReviewerCohort(ctx, cohortScope) + if err != nil || len(cohort.Members) != 1 || cohort.Members[0].ProviderSessionID != priorReviewerSession { + t.Fatalf("saved reviewer cohort changed: %#v, %v", cohort, err) + } + }) + } +} diff --git a/internal/pipeline/without_discussion_test.go b/internal/pipeline/without_discussion_test.go new file mode 100644 index 00000000..59c14ab4 --- /dev/null +++ b/internal/pipeline/without_discussion_test.go @@ -0,0 +1,368 @@ +package pipeline + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/open-cli-collective/codereview-cli/internal/dossier" + "github.com/open-cli-collective/codereview-cli/internal/gitprovider" + "github.com/open-cli-collective/codereview-cli/internal/ledger" + "github.com/open-cli-collective/codereview-cli/internal/llm" + "github.com/open-cli-collective/codereview-cli/internal/review" + "github.com/open-cli-collective/codereview-cli/internal/runartifact" + "github.com/open-cli-collective/codereview-cli/internal/runlifecycle" + "github.com/open-cli-collective/codereview-cli/internal/statepaths" +) + +// Distinct text for each kind of existing discussion, so any leak into a +// prompt or dossier file is caught by a plain substring search. +const ( + discussionThreadMarker = "DISCUSSION-THREAD-MARKER" + discussionCommentMarker = "DISCUSSION-ISSUE-COMMENT-MARKER" + discussionReviewMarker = "DISCUSSION-PRIOR-REVIEW-MARKER" + discussionCRThreadMarker = "Original finding." + priorOrchestratorSession = "prior-orchestrator-session" + priorReviewerSession = "prior-reviewer-session" +) + +var discussionMarkers = []string{discussionThreadMarker, discussionCommentMarker, discussionReviewMarker} + +func addExistingDiscussion(provider *readOnlyProvider, bot gitprovider.Identity) { + human := gitprovider.Identity{Login: "maintainer", ID: "maintainer-id"} + provider.threads = []gitprovider.InlineThread{{ + ID: "thread-human", + Path: "main.go", + Side: review.DiffSideRight, + Line: 2, + SubjectType: review.AnchorKindLine, + Comments: []gitprovider.ThreadComment{{ + ID: "thread-human-1", + Body: discussionThreadMarker + " this looks wrong", + Author: human, + }}, + }} + provider.issueComments = []gitprovider.IssueComment{{ + ID: "issue-1", + Body: discussionCommentMarker + " please also handle the empty case", + Author: human, + }} + provider.reviews = []gitprovider.Review{{ + ID: "review-1", + Body: discussionReviewMarker + " earlier cr review summary", + Author: bot, + Event: review.ReviewEventComment, + }} +} + +// storePriorSessions stores the PR's default orchestrator session and reviewer +// cohort as an earlier live review would have left them. +func storePriorSessions(ctx context.Context, t *testing.T, store *ledger.Store, provider *readOnlyProvider, req Request) (string, ledger.ReviewerCohortScope) { + t.Helper() + // The earlier live review that left these sessions behind. + allocateLiveRun(t, store, provider, req, "run-prior-live") + name, err := defaultSessionName(req) + if err != nil { + t.Fatalf("defaultSessionName: %v", err) + } + stored := namedSessionForRequest(req, priorOrchestratorSession) + stored.Name = name + if err := store.UpsertNamedSession(ctx, stored); err != nil { + t.Fatalf("UpsertNamedSession: %v", err) + } + prKey, err := statepaths.PRKey(req.PRRef.Host, req.PRRef.Owner, req.PRRef.Repo, req.PRRef.Number) + if err != nil { + t.Fatalf("PRKey: %v", err) + } + scope := ledger.ReviewerCohortScope{PRKey: prKey, Profile: req.ProfileName, PostingIdentity: runlifecycle.PostingKey(req.PostingIdentity)} + if err := store.ReplaceReviewerCohort(ctx, ledger.ReviewerCohort{ + Scope: scope, Adapter: "fake-llm", CreatedAt: fixedNow().Add(-time.Hour), UpdatedAt: fixedNow().Add(-time.Hour), + Members: []ledger.ReviewerCohortMember{{ + AgentID: "harness:reviewer", AssignmentMode: ledger.ReviewerAssignmentScoped, Files: []string{"main.go"}, + Model: "claude-sonnet-5-5", Effort: "medium", ProviderSessionID: priorReviewerSession, + }}, + }); err != nil { + t.Fatalf("ReplaceReviewerCohort: %v", err) + } + return name, scope +} + +func pinnedDiscussionHarness(t *testing.T) (*readOnlyProvider, Request) { + t.Helper() + provider, req := dryRunHarness(t) + fixture, reviewBaseSHA, reviewHeadSHA := newPinnedReviewFixtureForRef(t, req.PRRef) + provider.pr = fixture.pr + provider.pr.Title = "Replay title" + provider.pr.Body = "Replay description stays visible." + addRepoAgentFixture(provider) + provider.fixtureRepoDir = fixture.repoDir + provider.diffBetween = gitprovider.UnifiedDiff{Raw: smallDiff("main.go")} + req.ReviewBaseSHA = reviewBaseSHA + req.ReviewHeadSHA = reviewHeadSHA + addExistingDiscussion(provider, req.PostingIdentity) + provider.threads = append(provider.threads, markedReviewThread(t, "thread-cr", "main.go", 2, req.PostingIdentity, gitprovider.Identity{Login: "maintainer", ID: "maintainer-id"})) + return provider, req +} + +func withoutDiscussionOptions(t *testing.T, provider *readOnlyProvider, adapter llm.Adapter, store *ledger.Store, runID string) Options { + t.Helper() + return Options{ + Provider: provider, + Adapter: adapter, + Store: store, + NamedSessions: store, + Layout: statepaths.NewLayout(t.TempDir(), t.TempDir()), + Now: fixedNow, + NewRunID: func() string { return runID }, + NewSessionRowID: sequence("session"), + NewFindingID: findingSequence("finding"), + NewActionID: actionSequence(), + MaxConcurrency: 1, + } +} + +// allPrompts returns every prompt the adapter received, started or resumed. +func allPrompts(adapter *llm.FakeAdapter) []string { + var prompts []string + for _, request := range adapter.Requests() { + prompts = append(prompts, request.Prompt) + } + for _, resume := range adapter.Resumes() { + prompts = append(prompts, resume.Request.Prompt) + } + return prompts +} + +// dossierText concatenates every file the run wrote under its dossier. +func dossierText(t *testing.T, dir string) string { + t.Helper() + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatalf("open dossier root: %v", err) + } + defer root.Close() + var out strings.Builder + err = fs.WalkDir(root.FS(), ".", func(path string, entry fs.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return err + } + data, err := root.ReadFile(path) + if err != nil { + return err + } + out.WriteString(filepath.Join(dir, path)) + out.WriteString("\n") + out.Write(data) + out.WriteString("\n") + return nil + }) + if err != nil { + t.Fatalf("walk dossier: %v", err) + } + return out.String() +} + +func TestDryRunWithoutDiscussionReplaysAsFirstPass(t *testing.T) { + ctx := context.Background() + store := openPipelineStore(t) + defer closeStore(t, store) + provider, req := pinnedDiscussionHarness(t) + req.WithoutDiscussion = true + sessionName, cohortScope := storePriorSessions(ctx, t, store, provider, req) + adapter := &llm.FakeAdapter{NameValue: "fake-llm", SupportsResumeValue: true} + adapter.Queue(fakeLLMResult("selection-replay", selectionJSON("harness:reviewer", "main.go"), 10, 2)) + adapter.Queue(fakeLLMResult("reviewer-replay", findingsJSON("harness:reviewer", "main.go", "major", 2, "Fix this"), 20, 4)) + adapter.Queue(fakeLLMResult("rollup-replay", rollupJSON("comment", []string{"finding-1"}), 30, 6)) + + result, err := dryRunForTest(ctx, withoutDiscussionOptions(t, provider, adapter, store, "run-replay"), req) + if err != nil { + t.Fatalf("DryRun: %v", err) + } + + if provider.threadCalls != 0 || provider.reviewCalls != 0 || provider.issueCommentCalls != 0 { + t.Fatalf("discussion reads = threads %d reviews %d comments %d, want none", provider.threadCalls, provider.reviewCalls, provider.issueCommentCalls) + } + if resumes := adapter.Resumes(); len(resumes) != 0 { + t.Fatalf("resumes = %#v, want no prior orchestrator or reviewer session resumed", resumes) + } + prompts := allPrompts(adapter) + if len(prompts) != 3 { + t.Fatalf("prompts = %d, want selection, reviewer, rollup", len(prompts)) + } + for i, prompt := range prompts { + for _, leaked := range append(discussionMarkers, discussionCRThreadMarker, "discussion_outcomes") { + if strings.Contains(prompt, leaked) { + t.Fatalf("prompt %d contains discussion %q:\n%s", i, leaked, prompt) + } + } + } + if reviewer := adapter.Requests()[1]; reviewer.ReviewerWorkspace == nil || !reviewer.ReviewerWorkspace.NoNetwork { + t.Fatalf("reviewer workspace = %#v, want network tools denied", reviewer.ReviewerWorkspace) + } + if !strings.Contains(prompts[0], "Replay title") || !strings.Contains(prompts[0], "Replay description stays visible.") { + t.Fatalf("selection prompt lost PR title/description:\n%s", prompts[0]) + } + dossierFiles := dossierText(t, result.Artifacts.DossierDir) + for _, leaked := range append(discussionMarkers, discussionCRThreadMarker) { + if strings.Contains(dossierFiles, leaked) { + t.Fatalf("dossier contains discussion %q:\n%s", leaked, dossierFiles) + } + } + summary, err := dossier.ReadDiscussionSummary(result.Artifacts) + if err != nil { + t.Fatalf("ReadDiscussionSummary: %v", err) + } + if !summary.WithoutDiscussion || len(summary.TopLevelComments) != 0 || len(summary.InlineThreads) != 0 { + t.Fatalf("discussion summary = %#v, want empty and marked without discussion", summary) + } + + marker, err := runartifact.ReadMarker(result.Artifacts.Dir, runartifact.KindReview) + if err != nil { + t.Fatalf("ReadMarker: %v", err) + } + if !marker.WithoutDiscussion || !result.WithoutDiscussion { + t.Fatalf("marker/result without discussion = %v/%v, want recorded", marker.WithoutDiscussion, result.WithoutDiscussion) + } + if result.NamedSessionCandidate != nil { + t.Fatalf("named session candidate = %#v, want none", result.NamedSessionCandidate) + } + storedSession, err := store.GetNamedSession(ctx, sessionName) + if err != nil { + t.Fatalf("GetNamedSession: %v", err) + } + if storedSession.ProviderSessionID != priorOrchestratorSession { + t.Fatalf("orchestrator session = %q, want untouched %q", storedSession.ProviderSessionID, priorOrchestratorSession) + } + cohort, err := store.GetReviewerCohort(ctx, cohortScope) + if err != nil { + t.Fatalf("GetReviewerCohort: %v", err) + } + if len(cohort.Members) != 1 || cohort.Members[0].ProviderSessionID != priorReviewerSession { + t.Fatalf("reviewer cohort = %#v, want untouched %q", cohort, priorReviewerSession) + } +} + +// Without the flag, a pinned replay still resumes the PR's prior sessions, +// which is the leak the flag closes. +func TestDryRunPinnedWithDiscussionStillResumesPriorSessions(t *testing.T) { + ctx := context.Background() + store := openPipelineStore(t) + defer closeStore(t, store) + provider, req := pinnedDiscussionHarness(t) + storePriorSessions(ctx, t, store, provider, req) + adapter := &llm.FakeAdapter{NameValue: "fake-llm", SupportsResumeValue: true} + adapter.Queue(fakeLLMResult("reviewer-resumed", findingsJSON("harness:reviewer", "main.go", "major", 2, "Fix this"), 20, 4)) + adapter.Queue(fakeLLMResult("rollup-resumed", rollupJSON("comment", []string{"finding-1"}), 30, 6)) + + result, err := dryRunForTest(ctx, withoutDiscussionOptions(t, provider, adapter, store, "run-pinned"), req) + if err != nil { + t.Fatalf("DryRun: %v", err) + } + resumed := map[string]bool{} + for _, resume := range adapter.Resumes() { + resumed[resume.SessionID] = true + } + if !resumed[priorReviewerSession] || !resumed[priorOrchestratorSession] { + t.Fatalf("resumes = %#v, want prior reviewer and orchestrator sessions", adapter.Resumes()) + } + for _, resume := range adapter.Resumes() { + if workspace := resume.Request.ReviewerWorkspace; workspace != nil && workspace.NoNetwork { + t.Fatalf("reviewer workspace = %#v, want network tools left alone without the flag", workspace) + } + } + if result.WithoutDiscussion { + t.Fatal("result.WithoutDiscussion = true, want false") + } + marker, err := runartifact.ReadMarker(result.Artifacts.Dir, runartifact.KindReview) + if err != nil { + t.Fatalf("ReadMarker: %v", err) + } + if marker.WithoutDiscussion { + t.Fatal("marker records without_discussion for a run with discussion") + } +} + +// Without the flag and without pinning, the same discussion fixture reaches +// the dossier and the selection prompt, so the assertions above would see a leak. +func TestDryRunWithDiscussionFeedsExistingDiscussion(t *testing.T) { + ctx := context.Background() + store := openPipelineStore(t) + defer closeStore(t, store) + provider, req := dryRunHarness(t) + addExistingDiscussion(provider, req.PostingIdentity) + adapter := &llm.FakeAdapter{NameValue: "fake-llm"} + adapter.Queue(fakeLLMResult("dossier-summary-session", discussionSummaryJSON([]string{discussionCommentMarker}, nil), 8, 2)) + adapter.Queue(fakeLLMResult("selection-session", selectionJSON("harness:reviewer", "main.go"), 10, 2)) + adapter.Queue(fakeLLMResult("reviewer-session", findingsJSON("harness:reviewer", "main.go", "major", 2, "Fix this"), 20, 4)) + adapter.Queue(fakeLLMResult("rollup-session", rollupJSON("comment", []string{"finding-1"}), 30, 6)) + + result, err := dryRunForTest(ctx, withoutDiscussionOptions(t, provider, adapter, store, "run-with-discussion"), req) + if err != nil { + t.Fatalf("DryRun: %v", err) + } + if provider.threadCalls == 0 || provider.reviewCalls == 0 || provider.issueCommentCalls == 0 { + t.Fatalf("discussion reads = threads %d reviews %d comments %d, want all read", provider.threadCalls, provider.reviewCalls, provider.issueCommentCalls) + } + dossierFiles := dossierText(t, result.Artifacts.DossierDir) + for _, want := range discussionMarkers { + if !strings.Contains(dossierFiles, want) { + t.Fatalf("dossier missing discussion %q", want) + } + } + prompts := allPrompts(adapter) + selectionPrompt := prompts[1] + if !strings.Contains(selectionPrompt, discussionThreadMarker) || !strings.Contains(selectionPrompt, discussionCommentMarker) { + t.Fatalf("selection prompt missing existing discussion:\n%s", selectionPrompt) + } +} + +// An incomplete run made with discussion is never resumed by a replay without +// it, because its tasks and sessions may have seen that discussion. +func TestDryRunWithoutDiscussionDoesNotResumeRunWithDiscussion(t *testing.T) { + ctx := context.Background() + store := openPipelineStore(t) + defer closeStore(t, store) + provider, req := pinnedDiscussionHarness(t) + opts := withoutDiscussionOptions(t, provider, &llm.FakeAdapter{NameValue: "fake-llm"}, store, "run-fresh-replay") + provider.diffBetween = gitprovider.UnifiedDiff{} + withDiscussion := allocateDryRunForSHAs(t, store, opts.Layout, req, "run-with-discussion", req.ReviewHeadSHA, req.ReviewBaseSHA, fixedNow().Add(-time.Minute)) + req.WithoutDiscussion = true + + result, err := dryRunForTest(ctx, opts, req) + if err != nil { + t.Fatalf("DryRun: %v", err) + } + if result.Run.RunID == withDiscussion.RunID || result.Run.RunID != "run-fresh-replay" { + t.Fatalf("run = %q, want a fresh run instead of resuming %q", result.Run.RunID, withDiscussion.RunID) + } + marker, err := runartifact.ReadMarker(result.Artifacts.Dir, runartifact.KindReview) + if err != nil { + t.Fatalf("ReadMarker: %v", err) + } + if !marker.WithoutDiscussion { + t.Fatal("fresh replay marker does not record without_discussion") + } +} + +func TestPipelineRejectsWithoutDiscussionOutsidePinnedDryRun(t *testing.T) { + ctx := context.Background() + store := openPipelineStore(t) + defer closeStore(t, store) + provider, req := dryRunHarness(t) + req.WithoutDiscussion = true + opts := withoutDiscussionOptions(t, provider, &llm.FakeAdapter{NameValue: "fake-llm"}, store, "run-unpinned") + if _, err := dryRunForTest(ctx, opts, req); err == nil || !strings.Contains(err.Error(), "requires pinned review base and head SHAs") { + t.Fatalf("DryRun error = %v, want pinned SHA requirement", err) + } + run := allocateLiveRun(t, store, provider, Request{PRRef: req.PRRef, PRURL: req.PRURL, ProfileName: req.ProfileName, Profile: req.Profile, PostingIdentity: req.PostingIdentity}, "run-live") + if _, err := liveForTest(ctx, opts, req, run); err == nil || !strings.Contains(err.Error(), "requires dry-run review") { + t.Fatalf("Live error = %v, want dry-run requirement", err) + } + if provider.threadCalls != 0 || provider.reviewCalls != 0 || provider.issueCommentCalls != 0 { + t.Fatalf("discussion reads = %d/%d/%d, want none before rejection", provider.threadCalls, provider.reviewCalls, provider.issueCommentCalls) + } +} diff --git a/internal/pricing/pricing.go b/internal/pricing/pricing.go index d33616fe..30432ade 100644 --- a/internal/pricing/pricing.go +++ b/internal/pricing/pricing.go @@ -37,12 +37,13 @@ func EstimateUsageUSD(model string, usage Usage) (float64, bool) { // EstimateUsageUSDFor estimates usage against one immutable catalog snapshot. // Context-banded rows are intentionally unavailable because the current usage // record does not identify the context band or provider token normalization. +// Request-bound rows remain unavailable even when their context band is all. func EstimateUsageUSDFor(catalog *modelcatalog.Catalog, model string, usage Usage) (cost float64, ok bool) { if catalog == nil { return 0, false } prices := catalog.PricesFor(model, usage.Speed) - if len(prices) != 1 || prices[0].ContextBand != "all" { + if len(prices) != 1 || prices[0].ContextBand != "all" || prices[0].RequestPricing != nil { return 0, false } price := prices[0] diff --git a/internal/pricing/pricing_test.go b/internal/pricing/pricing_test.go index c065490e..2559db5c 100644 --- a/internal/pricing/pricing_test.go +++ b/internal/pricing/pricing_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "testing" "github.com/open-cli-collective/codereview-cli/internal/modelcatalog" @@ -252,6 +253,20 @@ func TestEstimateUsageUSDForLeavesUnknownRatesAndContextBandsUnavailable(t *test } func customPricingCatalog(t *testing.T) *modelcatalog.Catalog { + t.Helper() + return customPricingCatalogForSchema(t, 1, false) +} + +func customPricingCatalogForSchema(t *testing.T, schema int, requestBound bool) *modelcatalog.Catalog { + t.Helper() + catalog, err := modelcatalog.LoadPath(customPricingCatalogPathForSchema(t, schema, requestBound)) + if err != nil { + t.Fatalf("LoadPath: %v", err) + } + return catalog +} + +func customPricingCatalogPathForSchema(t *testing.T, schema int, requestBound bool) string { t.Helper() _, testFile, _, ok := runtime.Caller(0) if !ok { @@ -259,6 +274,9 @@ func customPricingCatalog(t *testing.T) *modelcatalog.Catalog { } source := t.TempDir() dataDir := filepath.Join(filepath.Dir(testFile), "..", "modelcatalog", "data") + if schema == 1 { + dataDir = filepath.Join(filepath.Dir(testFile), "..", "modelcatalog", "testdata", "v1") + } for _, name := range []string{"manifest.json", "runtimes.csv", "models.csv", "defaults.csv", "pricing.csv"} { body, err := os.ReadFile(filepath.Join(dataDir, name)) // #nosec G304 -- dataDir is the repository's bundled fixture. if err != nil { @@ -269,7 +287,15 @@ func customPricingCatalog(t *testing.T) *modelcatalog.Catalog { } } appendCatalogRow(t, source, "models.csv", "openai-api-key,pricing-test-model,low,,,,https://example.invalid,2026-10-03\n") - appendCatalogRow(t, source, "pricing.csv", "pricing-test-model,standard,all,7,11,0.7,,3,5,https://example.invalid,2026-10-03\n") + priceRow := "pricing-test-model,standard,all,7,11,0.7,,3,5,https://example.invalid,2026-10-03" + if schema == 2 { + if requestBound { + priceRow += ",openai-api-key,default,0,,whole_request" + } else { + priceRow += ",,,,," + } + } + appendCatalogRow(t, source, "pricing.csv", priceRow+"\n") manifestPath := filepath.Join(source, "manifest.json") manifestBody, err := os.ReadFile(manifestPath) // #nosec G304 -- manifestPath is under t.TempDir. if err != nil { @@ -295,11 +321,7 @@ func customPricingCatalog(t *testing.T) *modelcatalog.Catalog { if err := os.WriteFile(manifestPath, append(manifestBody, '\n'), 0o600); err != nil { // #nosec G703 -- manifestPath is under t.TempDir. t.Fatalf("write manifest: %v", err) } - catalog, err := modelcatalog.LoadPath(source) - if err != nil { - t.Fatalf("LoadPath: %v", err) - } - return catalog + return source } func appendCatalogRow(t *testing.T, dir, name, row string) { @@ -314,3 +336,120 @@ func appendCatalogRow(t *testing.T, dir, name, row string) { t.Fatalf("append %s: %v", name, err) } } + +func TestLegacyEstimateRejectsRequestBoundAllAndPreservesV1(t *testing.T) { + for _, tc := range []struct { + schema int + bound bool + }{{1, false}, {2, false}, {2, true}} { + catalog := customPricingCatalogForSchema(t, tc.schema, tc.bound) + if _, ok := catalog.PriceFor("pricing-test-model", "standard"); ok == tc.bound { + t.Fatalf("schema=%d bound=%t PriceFor availability=%t", tc.schema, tc.bound, ok) + } + for _, usage := range []Usage{{Speed: "standard"}, {TokensIn: p(1_000_000), Speed: "standard"}} { + cost, ok := EstimateUsageUSDFor(catalog, "pricing-test-model", usage) + if ok == tc.bound { + t.Fatalf("schema=%d bound=%t legacy availability=%t", tc.schema, tc.bound, ok) + } + if ok && cost != float64(deref(usage.TokensIn))*7/perMillion { + t.Fatal("legacy price changed") + } + } + if tc.bound { + if _, reason := catalog.PriceForRequest("openai-api-key", "pricing-test-model", "default", 1); reason != "" { + t.Fatalf("request-bound all row unavailable to exact selector: %q", reason) + } + } + } +} + +func TestLegacyEstimateStillRejectsEveryOpenAIContextBand(t *testing.T) { + catalog, err := modelcatalog.LoadBundled() + if err != nil { + t.Fatal(err) + } + for _, price := range catalog.Pricing() { + if price.ContextBand == "all" { + continue + } + if _, ok := EstimateUsageUSDFor(catalog, price.ModelID, Usage{TokensIn: p(1), Speed: price.Speed}); ok { + t.Fatalf("legacy estimator activated for %s/%s", price.ModelID, price.Speed) + } + } +} + +func TestPaddedV2HeadersCannotDemoteRequestBoundAllToLegacy(t *testing.T) { + for _, tc := range []struct { + name string + indexes []int + quoted, finiteMaximum bool + }{ + {name: "all five", indexes: []int{11, 12, 13, 14, 15}}, + {name: "all five quoted", indexes: []int{11, 12, 13, 14, 15}, quoted: true}, + {name: "partial", indexes: []int{11, 12}}, + {name: "optional maximum only", indexes: []int{14}, finiteMaximum: true}, + {name: "optional maximum quoted", indexes: []int{14}, quoted: true, finiteMaximum: true}, + } { + t.Run(tc.name, func(t *testing.T) { + source := customPricingCatalogPathForSchema(t, 2, true) + catalog, err := modelcatalog.LoadPath(source) + if err != nil { + t.Fatal(err) + } + usage := Usage{TokensIn: p(1_000_000), Speed: "standard"} + if _, ok := catalog.PriceFor("pricing-test-model", "standard"); ok { + t.Fatal("canonical bound all row entered PriceFor") + } + if _, ok := EstimateUsageUSDFor(catalog, "pricing-test-model", usage); ok { + t.Fatal("canonical bound all row entered legacy estimator") + } + sourceRoot, err := os.OpenRoot(source) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sourceRoot.Close() }) + body, err := sourceRoot.ReadFile("pricing.csv") + if err != nil { + t.Fatal(err) + } + parts := strings.SplitN(string(body), "\n", 2) + headers := strings.Split(parts[0], ",") + for _, index := range tc.indexes { + headers[index] += " " + if tc.quoted { + headers[index] = "\"" + headers[index] + "\"" + } + } + if tc.finiteMaximum { + // A populated invalid finite bound on all must not become unbounded + // merely because its header key was padded. + parts[1] = strings.Replace(parts[1], ",openai-api-key,default,0,,whole_request", ",openai-api-key,default,0,100,whole_request", 1) + } + body = []byte(strings.Join(headers, ",") + "\n" + parts[1]) + if err := sourceRoot.WriteFile("pricing.csv", body, 0o600); err != nil { + t.Fatal(err) + } + manifest := catalog.Manifest() + manifest.Files["pricing.csv"] = fmt.Sprintf("%x", sha256.Sum256(body)) + manifestBody, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + if err := sourceRoot.WriteFile("manifest.json", manifestBody, 0o600); err != nil { + t.Fatal(err) + } + if err := sourceRoot.Close(); err != nil { + t.Fatal(err) + } + malformed, err := modelcatalog.LoadPath(source) + if err == nil { + _, priceOK := malformed.PriceFor("pricing-test-model", "standard") + _, estimateOK := EstimateUsageUSDFor(malformed, "pricing-test-model", usage) + t.Fatalf("padded header accepted: PriceFor=%t legacy estimate=%t", priceOK, estimateOK) + } + if !strings.Contains(err.Error(), "header") { + t.Fatalf("error = %v, want exact header rejection", err) + } + }) + } +} diff --git a/internal/reviewplan/relocation_coverage_test.go b/internal/reviewplan/relocation_coverage_test.go index a440add6..22290ec6 100644 --- a/internal/reviewplan/relocation_coverage_test.go +++ b/internal/reviewplan/relocation_coverage_test.go @@ -32,7 +32,7 @@ func TestApprovalWithheldDoesNotCallValidRelocationUnread(t *testing.T) { RelocationReviewedFiles: []string{"old/module.go"}, }} var out strings.Builder - writeApprovalWithheld(&out, nil, coverage) + writeApprovalWithheld(&out, nil, coverage, nil) text := out.String() if !strings.Contains(text, "Every changed file was either body-inspected or received relocation-impact review") { t.Fatalf("withheld text does not describe relocation coverage accurately: %s", text) diff --git a/internal/reviewplan/reviewplan.go b/internal/reviewplan/reviewplan.go index fa454035..b84d3cb3 100644 --- a/internal/reviewplan/reviewplan.go +++ b/internal/reviewplan/reviewplan.go @@ -250,6 +250,27 @@ type builder struct { // Build turns validated review-domain values into a deterministic action plan. func Build(req Request) (Plan, error) { + // Selection runs before analysis. Its proposed thread actions cannot make + // an unanalyzed discussion safe to reply to or resolve. + if len(req.RunSummary.ThreadAnalysisFailures) > 0 { + failed := make(map[string]bool, len(req.RunSummary.ThreadAnalysisFailures)) + for _, failure := range req.RunSummary.ThreadAnalysisFailures { + failed[strings.TrimSpace(failure.ThreadID)] = true + } + var actions []review.ThreadAction + for _, action := range req.ThreadActions { + if !failed[strings.TrimSpace(action.ThreadID)] { + actions = append(actions, action) + } + } + var responses []review.ThreadResponseAction + for _, response := range req.ThreadResponses { + if !failed[strings.TrimSpace(response.ThreadID)] { + responses = append(responses, response) + } + } + req.ThreadActions, req.ThreadResponses = actions, responses + } b, err := newBuilder(req) if err != nil { return Plan{}, err @@ -463,7 +484,7 @@ func (b *builder) buildReview() (Plan, error) { } event = applySelfApprovalPolicy(event, b.req.EventOptions) approvalWithheld := false - if event == review.ReviewEventApprove && (len(b.req.RunSummary.ReviewerFailures) > 0 || hasIncompleteReviewerCoverage(b.req.RunSummary.ReviewerCoverage)) { + if event == review.ReviewEventApprove && (len(b.req.RunSummary.ReviewerFailures) > 0 || len(b.req.RunSummary.ThreadAnalysisFailures) > 0 || hasIncompleteReviewerCoverage(b.req.RunSummary.ReviewerCoverage)) { event = review.ReviewEventComment approvalWithheld = true } @@ -965,7 +986,7 @@ func (b *builder) renderRollup(ordered []review.Finding, anchored []AnchoredFind // misleads, so the explanation has to be the next thing read. withheld := func() { if approvalWithheld { - writeApprovalWithheld(&out, summary.Run.ReviewerFailures, summary.Run.ReviewerCoverage) + writeApprovalWithheld(&out, summary.Run.ReviewerFailures, summary.Run.ReviewerCoverage, summary.Run.ThreadAnalysisFailures) } } if len(summary.Reviewers) > 0 { @@ -995,6 +1016,7 @@ func (b *builder) renderRollup(ordered []review.Finding, anchored []AnchoredFind writeReviewerCoverageDiagnostics(&out, summary.Run.ReviewerCoverage) writeReviewerFailureDiagnostics(&out, summary.Run.ReviewerFailures) } + writeThreadAnalysisDiagnostics(&out, summary.Run.ThreadAnalysisFailures) writeThreadCountsLine(&out, summary.Threads) if b.req.AgentDefinitionsChanged { out.WriteString("\n---\n\n") diff --git a/internal/reviewplan/summary.go b/internal/reviewplan/summary.go index d69d01ae..18da144a 100644 --- a/internal/reviewplan/summary.go +++ b/internal/reviewplan/summary.go @@ -60,14 +60,15 @@ type ThreadCounts struct { type RunSummary struct { // ToolVersion is the raw version string (e.g. "0.3.63"); the renderer // adds the "cr " prefix. - ToolVersion string - Adapter string - Model string - PostingIdentity string - SelectedReviewers []string - ReviewerFailures []ReviewerFailureSummary - ReviewerCoverage []ReviewerCoverageSummary - WallDurationMS *int64 + ToolVersion string + Adapter string + Model string + PostingIdentity string + SelectedReviewers []string + ReviewerFailures []ReviewerFailureSummary + ThreadAnalysisFailures []ThreadAnalysisFailureSummary + ReviewerCoverage []ReviewerCoverageSummary + WallDurationMS *int64 // Workstreams holds the stages that ran in this round; a stage skipped by // reuse (e.g. selection under a reused reviewer cohort) is absent rather // than present with empty usage, which is what lets AggregateUsage's @@ -81,6 +82,13 @@ type ReviewerFailureSummary struct { Error string } +// ThreadAnalysisFailureSummary identifies an existing discussion that could not +// be analyzed. Its absence from thread responses does not imply it was settled. +type ThreadAnalysisFailureSummary struct { + ThreadID string `json:"thread_id"` + Error string `json:"error"` +} + // WorkstreamUsage is adapter-reported usage for one workstream: the reserved // orchestrator stage names or a reviewer agent ID. type WorkstreamUsage struct { @@ -119,7 +127,7 @@ type AggregateUsage struct { func (r RunSummary) hasData() bool { return r.ToolVersion != "" || r.Adapter != "" || r.Model != "" || r.PostingIdentity != "" || len(r.SelectedReviewers) > 0 || - len(r.ReviewerFailures) > 0 || len(r.ReviewerCoverage) > 0 || r.WallDurationMS != nil || + len(r.ReviewerFailures) > 0 || len(r.ThreadAnalysisFailures) > 0 || len(r.ReviewerCoverage) > 0 || r.WallDurationMS != nil || len(r.Workstreams) > 0 } @@ -311,6 +319,53 @@ func writeReviewerFailureDiagnostics(out *strings.Builder, failures []ReviewerFa out.WriteString("\n") } +func writeThreadAnalysisDiagnostics(out *strings.Builder, failures []ThreadAnalysisFailureSummary) { + if len(failures) == 0 { + return + } + out.WriteString("### Unanalyzed Threads\n\n") + examples := ThreadAnalysisFailureExamples(failures) + out.WriteString(threadAnalysisFailureCount(len(failures))) + if omitted := len(failures) - len(examples); omitted > 0 { + fmt.Fprintf(out, "; showing %d examples; %d omitted", len(examples), omitted) + } + out.WriteString(".\n\n") + for _, failure := range examples { + fmt.Fprintf(out, "- %s — not analyzed: %s\n", codeSpan(failure.ThreadID), escapeCell(failure.Error)) + } + out.WriteString("\nThese threads were left untouched. Complete failure evidence remains in the local task artifacts. Isolated analysis failures are cached on resume; use `--rerun` to retry with a fresh run.\n\n") +} + +const maxPublicThreadExamples = 5 + +// ThreadAnalysisFailureExamples returns deterministic, bounded display samples. +// The complete typed failure collection is left unchanged for local evidence, +// JSON presentation and safe cache fingerprinting. +func ThreadAnalysisFailureExamples(failures []ThreadAnalysisFailureSummary) []ThreadAnalysisFailureSummary { + examples := append([]ThreadAnalysisFailureSummary(nil), failures...) + sort.Slice(examples, func(i, j int) bool { + if examples[i].ThreadID != examples[j].ThreadID { + return examples[i].ThreadID < examples[j].ThreadID + } + return examples[i].Error < examples[j].Error + }) + if len(examples) > maxPublicThreadExamples { + examples = examples[:maxPublicThreadExamples] + } + for i := range examples { + examples[i].ThreadID = samplePublicDiagnostic(examples[i].ThreadID) + examples[i].Error = samplePublicDiagnostic(examples[i].Error) + } + return examples +} + +func threadAnalysisFailureCount(count int) string { + if count == 1 { + return "1 thread was not analyzed" + } + return fmt.Sprintf("%d threads were not analyzed", count) +} + // GitHub tables offer no column-width control, so variable-length values // (paths, constraint prose) wrap mid-word and repeat per row. Coverage is // rendered as a list instead, with bounded path examples and unknown fields @@ -526,9 +581,30 @@ func filesNotIn(files []string, exclude map[string]bool) []string { // already knows that incomplete coverage downgrades the event. Re-running does // not clear it either, since a reviewer that declined a file declines it again, // so a reader given no cause has no next step. -func writeApprovalWithheld(out *strings.Builder, failures []ReviewerFailureSummary, coverage []ReviewerCoverageSummary) { +func writeApprovalWithheld(out *strings.Builder, failures []ReviewerFailureSummary, coverage []ReviewerCoverageSummary, threadFailures []ThreadAnalysisFailureSummary) { out.WriteString("### Approval Withheld\n\n") - out.WriteString("No blocking or major findings were reported. Approval is withheld because at least one reviewer did not complete its assigned review obligations:\n\n") + threadOnly := len(failures) == 0 && !hasIncompleteReviewerCoverage(coverage) && len(threadFailures) > 0 + if threadOnly { + out.WriteString("No blocking or major findings were reported. Approval is withheld because existing discussions could not be analyzed:\n\n") + } else { + out.WriteString("No blocking or major findings were reported. Approval is withheld because at least one reviewer did not complete its assigned review obligations:\n\n") + } + if len(threadFailures) > 0 { + examples := ThreadAnalysisFailureExamples(threadFailures) + ids := make([]string, 0, len(examples)) + for _, failure := range examples { + ids = append(ids, codeSpan(failure.ThreadID)) + } + fmt.Fprintf(out, "- %s; examples: %s", threadAnalysisFailureCount(len(threadFailures)), strings.Join(ids, ", ")) + if omitted := len(threadFailures) - len(examples); omitted > 0 { + fmt.Fprintf(out, "; %d omitted", omitted) + } + out.WriteString(".\n") + } + if threadOnly { + out.WriteString("\nSee Unanalyzed Threads below for diagnostics and retry guidance.\n\n") + return + } for _, failure := range failures { fmt.Fprintf(out, "- %s did not produce a result: %s\n", codeSpan(failure.AgentID), escapeCell(samplePublicDiagnostic(failure.Error))) } diff --git a/internal/reviewplan/thread_analysis_failure_test.go b/internal/reviewplan/thread_analysis_failure_test.go new file mode 100644 index 00000000..97fa669b --- /dev/null +++ b/internal/reviewplan/thread_analysis_failure_test.go @@ -0,0 +1,139 @@ +package reviewplan + +import ( + "fmt" + "reflect" + "unicode/utf8" + + "strings" + "testing" + + "github.com/open-cli-collective/codereview-cli/internal/review" +) + +func TestBuildUnanalyzedThreadsWithholdApprovalAndSuppressActions(t *testing.T) { + for _, event := range []review.ReviewEvent{review.ReviewEventApprove, review.ReviewEventComment, review.ReviewEventRequestChanges} { + t.Run(string(event), func(t *testing.T) { + req := baseRequest() + req.Findings = nil + req.Rollup = review.Rollup{ReviewEvent: event} + if event != review.ReviewEventApprove { + f := finding("blocking", "main.go", review.Anchor{Kind: review.AnchorKindLine, Side: review.DiffSideRight, Line: 12}) + if event == review.ReviewEventRequestChanges { + f.Severity = review.SeverityBlocking + } + req.Findings = []review.Finding{f} + req.Rollup.OrderedFindings = []review.FindingID{f.ID} + } + req.ProviderCaps.ThreadResolution = true + req.RunSummary.ThreadAnalysisFailures = []ThreadAnalysisFailureSummary{{ThreadID: "failed-thread", Error: "invalid output"}} + req.ThreadActions = []review.ThreadAction{{ThreadID: "failed-thread", Decision: review.ThreadDecisionSummarizeAndResolve, Summary: "selector proposed closure"}} + req.ThreadResponses = []review.ThreadResponseAction{ + {ThreadID: "failed-thread", Kind: review.ThreadResponseSummaryReply, Body: "unsafe response", Resolve: true}, + {ThreadID: "successful-thread", Kind: review.ThreadResponseSummaryReply, Body: "settled", Resolve: true}, + } + plan, err := Build(req) + if err != nil { + t.Fatal(err) + } + for _, action := range plan.Actions { + if action.ThreadID == "failed-thread" { + t.Fatalf("action for unanalyzed thread = %#v", action) + } + } + if plan.Summary.Threads.Considered != 1 || plan.Summary.Threads.Resolved != 1 { + t.Fatalf("thread counts = %#v, want successful response only", plan.Summary.Threads) + } + want := OutcomeComment + if event == review.ReviewEventRequestChanges { + want = OutcomeRequestChanges + } + if plan.Outcome != want { + t.Fatalf("outcome = %q, want %q", plan.Outcome, want) + } + for _, want := range []string{"### Unanalyzed Threads", "failed-thread", "invalid output", "cached on resume", "--rerun"} { + if !strings.Contains(plan.RollupMarkdown, want) { + t.Fatalf("rollup missing %q: %s", want, plan.RollupMarkdown) + } + } + if got := strings.Contains(plan.RollupMarkdown, "### Approval Withheld"); got != (event == review.ReviewEventApprove) { + t.Fatalf("withheld section = %t for event %s", got, event) + } + }) + } +} + +func TestUnanalyzedThreadDiagnosticsBoundLargeCollections(t *testing.T) { + failures := make([]ThreadAnalysisFailureSummary, 2500) + for i := range failures { + failures[i] = ThreadAnalysisFailureSummary{ThreadID: fmt.Sprintf("thread-%04d", len(failures)-i-1), Error: strings.Repeat("界", 800)} + } + original := append([]ThreadAnalysisFailureSummary(nil), failures...) + for _, mixed := range []bool{false, true} { + t.Run(fmt.Sprintf("mixed_coverage=%t", mixed), func(t *testing.T) { + req := baseRequest() + req.Findings = nil + req.Rollup = review.Rollup{ReviewEvent: review.ReviewEventApprove} + req.RunSummary.ThreadAnalysisFailures = failures + if mixed { + req.RunSummary.ReviewerCoverage = []ReviewerCoverageSummary{{AgentID: "reviewer", Status: "incomplete_skipped", Scope: []string{"relocated.go", "missing.go"}, MissingFiles: []string{"missing.go"}, RelocationReviewedFiles: []string{"relocated.go"}}} + } + plan, err := Build(req) + if err != nil { + t.Fatal(err) + } + if plan.Outcome != OutcomeComment { + t.Fatalf("outcome = %s, want withheld approval", plan.Outcome) + } + for _, want := range []string{"2500 threads were not analyzed", "2495 omitted", "thread-0000", "thread-0004", "### Approval Withheld", "### Unanalyzed Threads"} { + if !strings.Contains(plan.RollupMarkdown, want) { + t.Errorf("rollup missing %q", want) + } + } + if strings.Contains(plan.RollupMarkdown, "thread-0005") || strings.Contains(plan.RollupMarkdown, "thread-2499") { + t.Fatal("public body rendered omitted thread IDs") + } + if len(plan.RollupMarkdown) > 15000 || !utf8.ValidString(plan.RollupMarkdown) { + t.Fatalf("public body bytes=%d utf8=%t, want bounded valid body", len(plan.RollupMarkdown), utf8.ValidString(plan.RollupMarkdown)) + } + if strings.Contains(plan.RollupMarkdown, strings.Repeat("界", 501)) { + t.Fatal("public diagnostic exceeded 500-rune limit") + } + if !reflect.DeepEqual(failures, original) || !reflect.DeepEqual(plan.Summary.Run.ThreadAnalysisFailures, original) { + t.Fatal("bounded rendering lost or mutated complete typed evidence") + } + if mixed { + for _, want := range []string{"assigned review obligations", "no body inspection or relocation-impact review", "missing.go", "relocation-impact-reviewed (not body-inspected)"} { + if !strings.Contains(plan.RollupMarkdown, want) { + t.Errorf("mixed coverage rollup lost %q", want) + } + } + } + var forward, reversed strings.Builder + writeThreadAnalysisDiagnostics(&forward, failures) + reordered := append([]ThreadAnalysisFailureSummary(nil), failures...) + for i, j := 0, len(reordered)-1; i < j; i, j = i+1, j-1 { + reordered[i], reordered[j] = reordered[j], reordered[i] + } + writeThreadAnalysisDiagnostics(&reversed, reordered) + if forward.String() != reversed.String() { + t.Fatal("thread diagnostic examples depend on source order") + } + }) + } +} + +func TestThreadFailureExamplesBoundLongIDsWithoutMutatingEvidence(t *testing.T) { + failures := []ThreadAnalysisFailureSummary{{ThreadID: strings.Repeat("🙂", 800) + string([]byte{0xff}), Error: strings.Repeat("界", 800) + ""}} + original := append([]ThreadAnalysisFailureSummary(nil), failures...) + examples := ThreadAnalysisFailureExamples(failures) + if len(examples) != 1 || utf8.RuneCountInString(examples[0].ThreadID) != 500 || utf8.RuneCountInString(examples[0].Error) != 500 { + t.Fatalf("example sizes = %#v, want 500-rune samples", examples) + } + if !utf8.ValidString(examples[0].ThreadID+examples[0].Error) || strings.Contains(examples[0].Error, "