From 737b0b0bc08990c8736fe7b27cf7b1e0bee79ab1 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Sat, 5 Sep 2026 22:47:44 +0000 Subject: [PATCH 1/3] Clarify CLI documentation --- README.md | 79 ++---- bench/README.md | 658 ++++-------------------------------------------- 2 files changed, 67 insertions(+), 670 deletions(-) diff --git a/README.md b/README.md index be51536..106383e 100644 --- a/README.md +++ b/README.md @@ -1,88 +1,53 @@ # Postil CLI -Postil is a quiet AI code review gate. It reports merge-relevant bugs, security issues, breaking changes, concurrency hazards, and explicit policy violations. Clean changes produce no review comment. +Postil reviews local changes and pull requests for merge-relevant defects. It reports gate status separately from advisory findings and leaves clean changes without a review comment. -One binary reviews local changes, pull requests, and merge requests. It supports OpenAI-compatible model endpoints and the native Anthropic Messages API. +## Quick start -## Install +Install a release build: ```sh curl -fsSL https://postil.dev/install.sh | sh ``` -The installer verifies the release checksum and, when `cosign` is available, its Sigstore signature. Pass `--require-cosign` to refuse checksum-only installation. You can also build from source: +The installer verifies the release checksum. Add `--require-cosign` to require Sigstore verification. To build from source instead: ```sh cargo install --git https://github.com/postil-dev/postil-cli --locked ``` -Release binaries cover Linux x86_64 and ARM64 with glibc or musl, plus macOS on Intel and Apple Silicon. - -## Review before pushing - -Authenticate once for hosted inference against your organization's entitlement, or bring your own model key: +Authenticate for hosted inference, or provide a key for a supported model endpoint, then review the repository: ```sh -postil login # zero-config: stores a renewable login for hosted inference -# or: export MODEL_API_KEY=... # OpenRouter is the default endpoint -postil doctor # validate the endpoint and repository -postil review # review staged, branch, or tracked working-tree changes -postil review --staged # explicitly review the staged change -postil review --base origin/main # review the branch -postil review --bounded --base origin/main # cap large reviews at five source batches -postil hook install # add a pre-push review +postil login +# Or set MODEL_API_KEY for your model endpoint. +postil doctor +postil review +postil hook install ``` -Bare local review selects staged changes first, then committed changes since the current branch's locally known default branch, then tracked working-tree changes, and finally an empty clean diff. Symbolic remote HEAD plus conventional `main`, `master`, and `trunk` refs are recognized without fetching. If no default branch can be resolved for a repository with commits, Postil fails closed and asks for `--base`, `--staged`, or `--diff-file` instead of reporting a partial review as clean. It sends the selected diff to the configured inference endpoint, but does not modify the working tree, index, or refs and does not write comments or checks to a forge unless `--publish` is supplied. - -`postil models` answers model-selection questions offline without reading credentials or contacting a provider. No model setting is required: it identifies the embedded default, explains the model-ID contracts for OpenAI-compatible and native Anthropic endpoints, separates local protocol compatibility from hosted qualification, and gives exact `postil doctor` and override commands. The Luna reviewer and hosted scorer candidate use `low` reasoning effort. Local scoring stays disabled until `REVIEW_SCORER_MODEL` names a scorer. `postil review` exits `0` when the gate passes, `1` when it fails, and `2` when it cannot produce a review envelope. Interactive terminals show concise progress; `--no-progress` or `POSTIL_NO_PROGRESS=1` replaces animation with concise static milestones, while `--verbose` enables complete provider, retry, and batch telemetry. +`postil review` selects staged changes first, then branch changes, then tracked working-tree changes. It does not change the working tree, index, or refs. Use `--publish` only when a forge review and checks should be written. -## Review a pull request +For a pull request, set `GITHUB_TOKEN` through the environment: ```sh -export GITHUB_TOKEN=... postil review --repo owner/repository --pr 123 --publish ``` -`--publish` is required for any forge write. Without it, the CLI fetches the pull request and reports locally. Published runs create separate `postil/review` and `postil/gate` checks. Findings appear in one batched review by default; GitHub repositories can set `review.findingPresentation: checkAnnotations` to put them on the advisory check instead. Mark only `postil/gate` as required in branch protection. - -For GitHub Actions, use [`postil-action`](https://github.com/postil-dev/postil-action). Hosted GitHub reviews are available at [postil.dev/install](https://postil.dev/install). - -## Configuration - -`postil init` writes `.postil.yaml`. Flags override environment variables, which override trusted repository configuration, stored login routing, and embedded defaults. `postil config` prints both the resolved model and its winning source. - -```yaml -ignore: - - "**/dist/**" -severityThreshold: info -minConfidence: 0.6 -maxFindings: 20 -reviewer: - tone: "direct, specific, no praise, no filler" - focus: [security, concurrency] -gate: - failOn: error - onError: block -model: - reasoningEffort: low - scorerReasoningEffort: low -``` - -Unknown keys are rejected. Repository configuration cannot redirect a deployment credential to another API host unless the operator explicitly permits that behavior. The embedded local preset uses `openai/gpt-5.6-luna`; local scoring is disabled until explicitly configured. Hosted profile selection remains service-controlled. +`--publish` is required for forge writes. Published reviews create `postil/review` and `postil/gate`; require only `postil/gate` in branch protection. ## Documentation -| Guide | Covers | -| --- | --- | -| [Configuration](docs/configuration.md) | Policy, precedence, model selection, and environment variables | -| [Model providers](docs/model-providers.md) | OpenAI-compatible, Anthropic, and local endpoints | -| [Code forges](docs/forges.md) | GitHub, GitLab, Bitbucket, and Azure DevOps | -| [Automation](docs/automation.md) | SARIF, incremental review, envelopes, planning, and usage receipts | -| [Architecture](ARCHITECTURE.md) | Trust boundaries and review pipeline | -| [Benchmarks](bench/README.md) | Model evaluation harness | +- [Configuration](docs/configuration.md): policy, model selection, and configuration precedence. +- [Model providers](docs/model-providers.md): OpenAI-compatible and Anthropic endpoints. +- [Code forges](docs/forges.md): GitHub, GitLab, Bitbucket, and Azure DevOps. +- [Automation](docs/automation.md): SARIF, incremental review, envelopes, and receipts. +- [Benchmarks](bench/README.md): offline regression suite and live qualification. +- [Public benchmark results](https://postil.dev/bench): model-screening results and raw report. +- [Architecture](ARCHITECTURE.md): trust boundaries and review pipeline. +- [Product documentation](https://postil.dev/docs): hosted setup and reference material. -The rendered product documentation is at [postil.dev/docs](https://postil.dev/docs). +Use `postil models` for offline guidance about the embedded model configuration and accepted endpoint model IDs. Use `postil config` to inspect resolved non-secret configuration. ## License diff --git a/bench/README.md b/bench/README.md index af62797..f860f1e 100644 --- a/bench/README.md +++ b/bench/README.md @@ -1,653 +1,85 @@ -# postil bench +# Postil benchmark suite -Hermetic PR-review regression suite for the `postil` CLI. The 70-fixture -admission matrix contains 47 must-block defects, 10 advisory defects, and 13 -clean PRs where the correct review is silence. Each fixture runs the release -binary against a per-case mock GitHub API and a mock OpenAI-compatible model -endpoint in an isolated run directory, then scores the v1 envelope and the -forge interactions against ground truth. +The benchmark lives in `bench/`. It is the CLI's regression and model-qualification harness. Its 70 fixtures include must-block defects, advisory defects, and clean changes that must receive no finding. -## What it measures +## Offline mock run -Pipeline fidelity, not authored-target detection rate: grounding (no -ungrounded findings, `counts.ungrounded == 0`), gating (gate fails exactly on -error-severity findings, exit codes match), statusline correctness -(`postil/review` and `postil/gate` check-runs created and completed with the -right conclusions), silence on clean PRs (no review comment posted), and -prompt-leakage guardrails (fixture metadata such as policy phrasing must never -appear in the prompt or any pipeline output). +Prerequisites: Rust, Bun `1.3.14`, and the repository checkout. This path makes no provider or forge requests. -The mock model returns recorded findings generated from the fixture specs, so -mock mode measures pipeline fidelity rather than authored-target detection -rate. Comparative claims require peer runs on the identical fixture set; site -comparisons stay qualitative and sourced until then. - -The fixture set includes direct defects and adversarial review cases: -off-by-one boundaries, prompt-injection text with rejected-source assertions, -misleading comments, huge low-signal multi-hunk diffs, near-duplicate clean and -defect pairs, Unicode homoglyphs, subtle races, and clean changes where silence -is the expected review. - -## Atomic scorer experiment - -`bineval-scorer.ts` compares scalar, batched binary, and independent binary -scoring contracts across two canonical development banks containing 20 cases. -`runCompleteDevelopmentScorerExperiment` runs every case through each selected -method and returns frozen reports derived from normalized request and response -evidence. The caller supplies the transport, model identity, provider identity, -bounded scorer settings, and repeat count. Each transport call has a bounded -evaluator wait and receives an abort signal. The caller-supplied transport is -trusted executable code and must honor that signal to stop its underlying -work. Provider receipts and telemetry remain bounded, untrusted observations -and cannot change evaluator correctness. - -The experiment is development evidence. Its fixtures do not provide an -independently held-out validation set and cannot qualify a production scorer. -Reports contain full prompts and normalized provider responses. Treat them as -sensitive development evidence and keep them out of public artifacts. - -## Running (mock mode: default, CI) - -```sh -cargo build --quiet --release # from the repo root -cd bench -bun install -bun run bench # add --json or --json-out report.json for machine output -``` - -`POSTIL_BIN` points the suite at a different binary; -`POSTIL_BENCH_KEEP_RUNS=1` keeps per-case run directories under `.runs/` -(failing cases are always kept). - -## Pair qualification (opt-in, live inference) - -Live qualification exercises the exact deployed generator and scorer together. -There are no unlisted or implicit fallbacks: the ordered generator chain and -consensus width are part of the qualified profile. The scorer runs through the production -prompt, filtering, usage accounting, and gate path. Each pair must complete the -entire matrix at least three times. - -The `prompt-injection-comment-clean` fixture runs first for up to three -requested repeats. A qualifying run uses at least three repeats. Admission -stops when any canary repeat emits a final or suppressed finding, records an -invalid generator, repair, or scorer result, changes the gate outcome, or -posts a review comment. Passing canary results are reused in the full report, -so the fixture is not billed twice. - -The manual `Bench (managed OpenRouter admission)` workflow is fixed to the -managed OpenRouter endpoint and its OpenAI-compatible interface. The local -pair-qualification command enforces the same endpoint and interface. The -separate diff-file live benchmark supports operator-owned OpenAI-compatible -and Anthropic BYOK endpoints. - -The managed workflow runs only from the exact `refs/heads/main` ref, includes -that immutable source SHA and a 30-day authority window in every candidate -profile, and attests the exact candidate file with GitHub OIDC through -Sigstore. Every hosted candidate binds the canonical endpoint to -`openrouter:managed-routing`; custom and local evidence cannot produce a -hosted manifest candidate. The workflow uploads the candidate and its -attestation bundle together. To admit a profile, commit the downloaded candidate as -`qualified-models.json` and its bundle as -`qualified-models.attestation.json`. CI verifies both with: +From the repository root: ```sh +cargo build --quiet --release cd bench +bun install --frozen-lockfile +bun test --timeout 60000 bun run verify-admission +bun run bench ``` -Verification uses [`gh attestation verify`](https://cli.github.com/manual/gh_attestation_verify) -to pin `postil-dev/postil-cli`, the exact admission workflow, -the main source ref, source and signer commit, GitHub's OIDC issuer, the SLSA -provenance predicate, public Sigstore trust, and a GitHub-hosted runner. The -source commit must be an ancestor of the candidate commit, and the intervening -diff may contain only the manifest and bundle. A cryptographically verified -Sigstore timestamp must match the signed issue time within 15 minutes and fit -inside the 30-day window. The expiry second is outside the authority window. -CI, release validation, and the runtime reject expired authority. A -missing, mismatched, stale, or invalid bundle rejects a nonempty manifest. The -empty manifest is exempt because it admits no models. Report and profile -checksums detect changes; they do not authenticate who produced a candidate. - -The release binary includes a guarded qualification path that accepts one exact -candidate profile inside the hermetic benchmark. Activation requires CI, -managed privacy enforcement, and a loopback mock forge. Candidate runs execute -the production hosted planner, request preflight, price ceilings, consensus, -and scorer behavior without granting unqualified profiles authority in a -deployed service. - -```sh -export MODEL_API_KEY=... # or POSTIL_API_KEY, OPENROUTER_API_KEY, or LLM_API_KEY -export POSTIL_BENCH_MODE=live -export POSTIL_BENCH_PAIRS=provider/generator::provider/scorer+provider/scorer-fallback -export POSTIL_BENCH_REPEATS=3 -export POSTIL_API_BASE=https://openrouter.ai/api/v1 -export POSTIL_API_FORMAT=openai-compatible -export POSTIL_BENCH_UPSTREAM_PROVIDER='Exact upstream provider name' -export POSTIL_BENCH_UPSTREAM_PROVIDER_ROUTE='exact/provider-route' -bun run bench --json-out report.json --manifest-out ../qualified-models.json -``` - -Live admission emits public report schema version 4. Consumers must call -`parseLiveModelsReport`; unversioned reports and unknown schema versions are -rejected. The parser upgrades retained schema-3 reports by defaulting their -provider route to the recorded provider identity. Public case diagnostics contain counts and SHA-256 digests only. -Finding prose, target contracts, raw evaluator responses, evaluator reasons, -and diagnostic text are absent. - -The same invocation writes a separate private replay bundle. Use -`--private-evidence-out ` or `POSTIL_BENCH_PRIVATE_EVIDENCE_OUT` to select -its location. Without either, the mode-0600 file stays under the gitignored -`bench/.runs/` directory. The runner reads the file back, checks its exact-byte -digest against `privateEvidenceSha256` in the public report, and replays every -attribution record before writing the report or admission candidate. This file -contains model requests, finding prose, target contracts, raw responses, and -evaluator reasons. Store it as sensitive qualification evidence and remove it -when the applicable evidence-retention period ends. The managed workflow keeps -plaintext in runner temporary storage and requires the -`POSTIL_PRIVATE_EVIDENCE_PASSPHRASE` repository or environment secret before -inference. It encrypts the bundle with GnuPG AES-256, decrypts and byte-compares -the result, uploads only the encrypted artifact, and removes both runner files -in an `always()` cleanup step. GitHub's configured artifact retention policy -owns the encrypted bundle lifetime. The workflow publishes no raw-run artifact. - -The release binary must embed the exact profile under test. Set the intended -generator, cascade, consensus width, scorer, API base, and interface in -`config.toml`, leave the -admission manifest empty, then build the binary. The benchmark compares the -binary's embedded metadata with the worktree before inference. Environment -model variables select that same profile for the isolated run; they do not -define a different candidate. - -Managed qualification uses the canonical OpenRouter endpoint and one exact -upstream provider route. An operator-owned pricing file can replace the -OpenRouter endpoint catalog response for that same route: - -```json -{ - "provider/model": { - "providerIdentity": "Exact upstream provider name", - "promptUsdPerToken": "0.000001", - "completionUsdPerToken": "0.000005" - } -} -``` - -Pass it with `--pricing-file prices.json` or -`POSTIL_BENCH_PRICING_FILE=prices.json`. Prices are positive canonical decimal -strings that must be exactly representable as integer micros per million -tokens. Every row names the exact upstream provider passed with -`--upstream-provider`; `--upstream-provider-route` identifies the exact -endpoint slug when it differs from the response provider identity. A mismatch -fails before inference. Each admitted profile carries immutable input and output price bounds -for its exact generator and scorer model set. The catalog request uses the -inference credential when no file is supplied and fails closed when any model -is unpriced. Catalog redirects are rejected so credentials remain bound to the -configured endpoint origin. - -Native Anthropic and authenticated private endpoints remain available to BYOK -runtime configurations. They are outside managed hosted qualification. +`bun run bench` is the CI benchmark command. It runs the release binary against a per-case mock GitHub API and mock OpenAI-compatible model endpoint. Set `POSTIL_BIN` to test a different release binary. Set `POSTIL_BENCH_KEEP_RUNS=1` to retain successful per-case run directories; failures are retained automatically. -Pair syntax is `generators::scorer+fallback` or -`generators::consensus::scorer+fallback`. For example, -`provider/one+provider/two+provider/three::2::provider/scorer` qualifies an -ordered three-model generator chain with two-model consensus. Omitting the -consensus field requires every listed generator. The optional scorer fallback -is tried after the primary scorer. +Add `--json` for machine-readable stdout or `--json-out report.json` to write a report. The report records fixture results, gate and status-check behavior, and fidelity failures. -Admission requires all of these in every repeat: +## Reading mock results -- 100% must-block detection and final blocking by the attributed finding itself -- at least 90% advisory detection and at most 10% advisory overblocking -- no clean false blocks and at most 5% clean cases with any finding -- no execution, structured-output, grounding, statusline, or usage-accounting failure -- mean pair cost at most $0.04 and mean review latency at most 15 seconds -- every review reports at most $1 of actual provider cost -- per-repeat p95 latency at most 30 seconds and maximum latency at most 60 seconds +A passing run means the review pipeline handled the recorded fixture outputs correctly: grounded findings, expected gate verdicts and exit codes, correct `postil/review` and `postil/gate` checks, and silence on clean changes. It does not measure whether a model can detect the authored defects, because the mock model replays recorded findings. -Findings suppressed by the scorer count as detector evidence but cannot satisfy -final blocking. An unrelated error cannot substitute for the attributed finding. -The report stores only attributable finding coordinates and labels, never model -finding titles or bodies. Per-case attribution records retain the verdict and -immutable request, response, usage, and evidence hashes. Evaluator-bank records -retain eligibility, call count, and their aggregate evidence hash. Requests, -raw responses, target contracts, and evaluator reasons stay out of the report. -The report records separate fixture, review-contract source, -configuration, evaluator contract, and CLI binary SHA-256 hashes; the canonical API base and -provider interface; the ordered generator chain and consensus width; the -ordered scorer chain; repeat number; and provider-exact or catalog-estimate -cost provenance. One checked-in source manifest defines the identical Rust and -TypeScript evaluator source list, including the attestation verifier, -`bench/package.json`, and `bench/bun.lock`; `packageManager` pins the Bun -runtime identity. -Source-bundle hashes use the runtime's ordered -`path + NUL + exact bytes + NUL` framing. Each immutable profile and the -complete sanitized evidence payload have their own SHA-256 identifier. -The runtime recomputes the profile identifier from one canonical manifest -material object: model defaults, provider identity, endpoint and interface, -ordered model chains and consensus, sorted price bounds, review and evaluator -contracts, evaluator runtime, report digest, and repeat count. The report -records the evaluated binary hash, but the profile identifier excludes it -because embedding the resulting manifest changes the binary bytes and would -create a self-referential digest. Source-contract and report digests bind the -profile to the evaluated binary without that cycle. -`manifestCandidate` uses the runtime admission-manifest schema directly and is -absent from a failed report. Only the process that performs the live run can -write a candidate, using `--manifest-out` on that invocation. Saved JSON -reports are evidence only and cannot be admitted later. Explicit report and -candidate paths are invalidated before a run and replaced atomically; mock mode -rejects `--manifest-out`. Output aliases are rejected by canonical parent path -and existing file identity, including symlinked parents and hardlinks. +The suite also checks prompt-leakage guardrails, adversarial source text, large low-signal diffs, Unicode homoglyphs, and concurrency cases. It is a regression suite, not a competitor comparison or a production-quality claim. -The managed preflight runs the CLI's exact normalized and compacted request -plan for every fixture before inference. It includes bounded planner, selected -source and synthesis requests, scoring, consensus, fallback, repair, and -bounded post-processing requests. Transport retries reserve exact exposure at -runtime under the same hard limits. Preflight rejects missing prices, more than six models, a review -whose worst-case retry projection exceeds $25, a total above the configured qualification -cap, or a cap outside `(0, $70]`. Runtime independently rejects more than $1 of -reported provider cost or 20 million reported tokens. A single model used for more than one role is -priced for each planned invocation. -Atomic attribution accepts at most three findings anchored in one authored -region. More is a fidelity failure. Each decision is limited to a 4 KiB input, -a 5,000-byte serialized provider request whose size conservatively caps prompt -tokens, and six possibly billed attempts across the initial request and one -schema repair. -The inference key stays in the child environment and is never printed or placed -on an argument list. Every inference request crosses one loopback proxy owned by -the admission run. The proxy admits at most four request starts per second across -all child processes and applies a provider `Retry-After` pause to the complete -bank, capped at 30 seconds. The CLI retains its per-call attempt, deadline, and -spend limits. +## Paid live screen -These fixtures are internal evidence, not a competitor comparison. Inference is -nondeterministic, so one successful matrix is insufficient for admission. -OpenRouter's endpoint identity and the selected ZDR provider route are pinned. -Pricing, requests, responses, attribution evidence, and the emitted candidate -must all identify that exact route. Hosted OpenRouter qualification uses the -same non-collection and ZDR request preferences as production. +Live screening sends fixture content to a model provider and can incur charges. It is opt-in and is not part of the offline command above. Set one of `MODEL_API_KEY`, `POSTIL_API_KEY`, `OPENROUTER_API_KEY`, or `LLM_API_KEY` in the environment before starting. Set `REVIEW_MODEL` to a model ID available to that credential. -## Scorer qualification (opt-in, mocked generator + real scorer) - -The independent scorer has a different job from the primary review generator: -it receives already-generated findings, without the generator's confidence or -kind, and calibrates each finding's confidence and kind against local diff -context. `bun run scorer-eval` screens that role directly by mocking the -primary generator with fixed findings and proxying scorer requests through one -named OpenRouter provider with fallback routing disabled. -Mock generator and planner usage is identified separately and excluded from -live scorer cost evidence. -This diagnostic can reject a scorer but cannot admit a production pair; pair -qualification above is the admission authority. +From the repository root: ```sh cargo build --quiet --release cd bench -export MODEL_API_KEY=... # or LLM_API_KEY / OPENROUTER_API_KEY -POSTIL_SCORER_EVAL_MODELS=provider/candidate-a,provider/candidate-b \ -POSTIL_SCORER_EVAL_REPEATS=5 \ -POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER=provider-name \ -POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE=provider-route \ -POSTIL_SCORER_EVAL_ROOT_DIR=.runs/scorer-eval/unique-run-id \ - bun run scorer-eval --json-out scorer-eval-report.json -``` - -The provider name is the identity echoed in responses. The optional provider -route is the exact OpenRouter endpoint slug; it defaults to the provider name. -Qualification starts only when the evaluator source bundle matches `HEAD`. -The retained report binds that commit to an evaluator SHA-256 digest, the exact -release-binary digest, provider identity and route, ZDR and fallback policy, -and sorted per-model maximum price bounds. - -The default candidates come from `config.toml`; the workflow input may override -them explicitly. Qualification repeats 12 fixtures five times: six unambiguous -authored target risks and six injected false findings. Admission requires a complete -matrix, no malformed, repaired, fallback, or reason-contract failures, all -target risks preserved as published gate failures, at least 80% of false -findings actually suppressed overall and per fixture, scorer-only p50/p95/max -latency at or below 5/10/20 seconds, known live -catalog pricing, and mean scorer cost at or below $0.005 per case. A failed -candidate makes the command exit nonzero after writing its report. Candidate -listing alone never enables the embedded scorer. Before any model call, the -evaluator rejects more than six candidates, more than ten repeats, missing -prices, or a conservative projected total above $10. The projection prices the -runtime retry graph: three transport attempts for the initial request and three -for at most one schema-repair request. A one-finding qualification request uses -a 17,000-byte prompt bound, a 400-token output bound, and at most 1,600 bytes -of repair context. Scorer responses also fail -admission when provider usage is missing or malformed, runtime accounting is -incomplete, or the assessment is not trimmed single-line text ending in -sentence punctuation. The prompt targets at most 180 UTF-8 bytes, and the -parser rejects more than 240 UTF-8 bytes. Each adjudication and scorer request -has its own 20-second admission limit. A 45-second outer safety cutoff leaves -both sequential live phases their full window plus bounded fixture overhead, -and teardown aborts outstanding provider requests. A timeout rejects that -candidate immediately rather than running the -rest of its matrix. Any other admission-fatal structural result, including an -unroutable provider response, malformed envelope, scorer mismatch, invalid -reason, incomplete usage, or repair attempt, also stops only that candidate. -Ordinary true/false quality misses run the complete statistical matrix. Reports -record completed and expected case counts explicitly. With `--json-out`, -`.partial` atomically records completed case metrics without prompts, -responses, credentials, or error text; the final report replaces it after the -run completes. Scorer output is bounded from the supplied finding count, up to -the supported maximum of 20 findings, and schema-repair context is byte-bounded -from the same output limit. - -## Diff-file live mode (opt-in, no forge) - -This live mode runs the real release binary against the same fixtures with a -real model and **no mocked model server**, so it measures authored-target detection -hit rate rather than pipeline fidelity. Each case runs in local diff-file mode -(`postil review --diff-file --output-json`), which does -no forge I/O at all, so no GitHub server, mock or real, is involved and nothing -is written to any repo. - -```sh -export MODEL_API_KEY=... # required; never logged or printed -REVIEW_MODEL=provider/qualified-model bun run bench:live -# Screen exact fixtures against the provisional Luna route. Repeat --case. -REVIEW_MODEL=openai/gpt-5.6-luna bun run bench:live -- \ - --run-id luna-azure-eu-screen-1 \ - --screen-profile ../provisional-models.json \ - --case prompt-injection-auth-bypass \ - --case near-duplicate-auth-clean -# Keep provider calls inside the live screen's 180-second case watchdog. -POSTIL_LLM_REQUEST_TIMEOUT_SECS=60 POSTIL_LLM_TOTAL_TIMEOUT_SECS=170 \ - REVIEW_MODEL=openai/gpt-5.6-luna bun run bench:live -- \ - --run-id luna-azure-eu-bounded-timeouts \ - --screen-profile ../provisional-models.json \ - --case prompt-injection-auth-bypass -# A profile with a scorerChain can exercise the production scorer path. -REVIEW_MODEL=provider/generator bun run bench:live -- \ - --screen-profile ./screen-profile.json \ - --scorer-model provider/scorer \ - --case prompt-injection-auth-bypass -# Exercise the production large-review selection and synthesis path. -REVIEW_MODEL=provider/qualified-model bun run bench:live -- --bounded -# --model is the equivalent command-line override -# --concurrency or BENCH_CONCURRENCY sets case parallelism (default 6) +bun install --frozen-lockfile +bun run bench:live -- --json-out .runs/live-screen.json ``` -The scorer receives generated findings rather than the complete review. A -silent generator gives it no work, so a clean envelope truthfully contains no -scorer call or scorer identity. A finding that is later suppressed does -exercise the scorer and must retain its exact identity and usage record. - -It refuses to run without `POSTIL_API_KEY`, `OPENROUTER_API_KEY`, `MODEL_API_KEY`, -or `LLM_API_KEY` and never logs or prints the key value. Live mode spends real -tokens and depends on an external provider, so it is **not run on ordinary -pushes or pull requests**. The release pipeline runs five sequential -full-corpus live samples against `bench/baseline.json` before every tagged -release (see "Release gate" below). Every live run writes its JSON report under -`.runs/live//` (gitignored), beside raw per-attempt stdout and stderr. -Set `--run-id ` or `POSTIL_BENCH_SCREEN_RUN_ID` to name the immutable -namespace; an omitted ID gets a unique generated value. Reusing an ID fails -before inference. `--json-out ` writes an additional report copy and -`--json` prints the report. `--bounded` (or -`POSTIL_BENCH_BOUNDED=1`) qualifies the deterministic risk-selection and -synthesis path used when a review exceeds five source batches. Every report -records `reviewMode` as `exhaustive` or `bounded` so admission tooling can -reject evidence from the wrong execution path. It also records the release -binary's SHA-256 digest so the report cannot be paired with a different -executable during admission. Fixture-corpus and evaluator-source digests bind -the results to the benchmark inputs and scoring code. - -`POSTIL_LLM_REQUEST_TIMEOUT_SECS` and `POSTIL_LLM_TOTAL_TIMEOUT_SECS` are -optional canonical positive integer seconds. Explicit values must expire before -the per-case process watchdog, and the request timeout cannot exceed the total -timeout. The harness forwards only these validated values to each isolated -child. Unset values remain unset so the CLI owns its defaults. `run.json` and -the aggregate report retain the exact overrides without recording credentials. - -`--case ` selects one exact fixture and may be repeated. Selected -cases require `--screen-profile `. The profile binds the model chain, -scorer chain, exact upstream provider, canonical managed endpoint, and price -ceilings. Requests deny provider data collection, require zero-data retention, -pin that provider without fallbacks, and enforce the profile prices. The report -records the selected IDs and marks the evidence as non-admission screening. -When the harness explicitly disables scoring, screening projects only the -profile's exact generator role; the unexercised scorer chain remains recorded -in the profile but does not have to be active in the child process. -Formal admission rejects `--case`, `--scorer-model`, and `--screen-profile`. -Every cost total says whether all calls supplied complete provider accounting. - -### Concurrency and retries +This screen runs every fixture in local diff-file mode. It does no forge I/O. The console labels defects as `HIT` or `MISS`, clean fixtures as `SILENT` or `NOISE`, and operational failures as `ERR`. It reports detection, silence on clean fixtures, false positives, latency, tokens, and observed provider cost. The explicit report is `.runs/live-screen.json`; per-case evidence is also retained in `bench/.runs/live//`. -Cases run through a bounded worker pool, `--concurrency ` (or -`BENCH_CONCURRENCY`, default 6) at a time, instead of strictly sequentially. -Each case still gets its own isolated run directory, and results are sorted by -case index before the report is written, so the output is byte-for-byte -deterministic in ordering regardless of completion order. Set `--concurrency 1` -to fall back to fully sequential execution. +Inspect the fixture IDs and source before a live run in [`fixtures/cases.ts`](fixtures/cases.ts). A selected-case screen requires `--case ` together with `--screen-profile `, which supplies the exact provider and price contract. A live screen is development evidence, not a hosted admission or a comparison with another model. -Exploratory live screens retry each case **once** by default (after a short -backoff) when its first attempt fails with a retryable operational error: -a non-zero exit whose stderr carries an HTTP 5xx/429, rate-limit, timeout, or -connection signature, or a run that produced no valid v1 envelope at all -(empty/garbled output, typically a dropped response). An envelope with an -unrecovered `review/invalidOutput` incident is also retried. `--retries ` -changes that outer retry count. A scored envelope is always a final result, -including a gate-failing exit or findings unrelated to the authored target. +## Managed qualification -Formal calibration and release cohort manifests pin the outer retry count to -one. Reports preserve each case's attempt count, recovered failure categories, -and aggregate duration, token, and exact provider-cost accounting. Every -accepted provider generation remains represented in the attested evidence. +Managed qualification exercises an ordered generator and scorer pair through the mock forge and a real provider. It requires an exact pair, provider identity and route, three complete repeats, and a release build whose embedded profile matches the worktree. Run the manual [managed admission workflow](../.github/workflows/bench-live.yml) for the attested hosted path. `bun run verify-admission` validates checked-in admission evidence. -### What live mode scores +
+Advanced qualification and calibration -- **Authored-target detection rate**: a finding counts only when its path and anchor line - match the authored target region and the atomic evaluator attributes the same defect. -- **Severity match (exact)** among attributed findings: strict equality between the found - severity and the fixture's ground-truth severity. -- **Severity match (+/-1 tier)** among attributed findings: a wider band that treats - adjacent tiers on the `info < warn < error` scale (i.e. `info`<->`warn` and - `warn`<->`error`) as a match, counting only the two-tier `info`<->`error` gap - as a real mismatch. -- **Silence on clean PRs**: a clean case should produce no findings. -- **Unrelated finding count**: any finding in a clean case, and any - non-matching finding in a defect case. -- **Confidence distribution** of attributed findings, and per-case duration / token - usage. +The managed workflow is the only path that produces an attested hosted-admission candidate. It runs from `main`, accepts the exact profile, upstream provider identity and route as workflow inputs, writes a report and candidate separately, encrypts private replay evidence, and attests the candidate. -Both severity numbers are reported, and the per-case detail always shows the -truth-vs-found severity. The exact figure uses strict equality. The +/-1-tier -figure is a deliberately softer matching rule that also accepts adjacent tiers -on the `info < warn < error` scale. +The release workflow runs the scorer screen with its exact configured profile: -These numbers are a **measured baseline for this CLI**: a single model, one run -per case, diff-only with no repository context or policy docs. **Neither -severity metric is a peer-comparison claim**: no competitor has been run on the -same fixtures. Results vary across runs because model inference is -nondeterministic. Treat them as internal evidence, not a published benchmark. - -## Release gate - -The `Release` workflow runs five uniquely named, sequential full-corpus -diff-file samples against the Luna profile in `provisional-models.json`. Every -sample uses the same preserved release binary. All five samples are attempted -even when one fails. The prepare job writes one five-slot cohort manifest bound -to the source commit, tag ref, workflow run, and attempt. GitHub OIDC attests the -exact manifest with public Sigstore provenance. Each sample reserves one -canonical slot directory and attests the running receipt before inference, -then attests its completed report and receipt together. Every accepted provider -response contributes its OpenRouter generation ID to the report. The fan-in job -verifies globally distinct generation IDs against OpenRouter's authenticated -generation API, including the exact canonical provider model pinned for each -logical profile model, provider, native token totals, and cost. It also verifies every -subject against the exact repository, release workflow, source commit, tag ref, -OIDC issuer, and GitHub-hosted runner before parsing it. -Only the unique first workflow run for -the version tag is authoritative. Tag-scoped concurrency, an existing-release -check, and duplicate-run rejection prevent a second publisher path. A failed -sample retains its terminal receipt but has no successful attestation. A -failed sample, missing or unverifiable subject, incomplete slot, or failed -comparison blocks the release because `build` depends on the `bench-live` job. - -The comparator accepts exactly one, three, or five distinct `--result` paths for -comparison. One- and three-report comparisons support smaller local checks; the -release workflow always supplies five. Multi-report paths must contain -byte-distinct reports with distinct immutable run IDs and timestamps. Byte -identity is established by SHA-256 over each raw file and authenticated by the -release attestations. -Five-report comparisons additionally require the original manifest and one -completed receipt for every slot. The comparator verifies the manifest against -the supplied binary, evaluator, corpus, profile, provider contract, workflow -run, and workflow attempt. It verifies every receipt's slot, nonce, run ID, -report digest, and timestamp interval. Semantically identical outcomes are -valid when their raw subjects have independent authenticated provenance. -Running, failed, missing, substituted, duplicate, and extra slots invalidate -the whole cohort. -Every report must be exhaustive full-corpus evidence with an empty -`selectedCaseIds`, an enforced provider contract, no operational errors, and -all cases scored. Summary and per-result cost accounting must be complete. -Every cost is a canonical nonnegative decimal, and the summary cost must equal -the exact sum of result costs. Binary, fixture-corpus, evaluator, screening -profile, and provider-contract hashes are exact 64-character lowercase SHA-256 -values. Provider identity, route, API, and scorer configuration fields must be -present and internally consistent. The comparator requires explicit release -binary and screening-profile paths, then recomputes the binary, fixture corpus, -evaluator source, screening profile, provider contract, and exact case cohort -instead of trusting hashes asserted by the reports. - -A five-report comparison also requires identical binary, corpus, evaluator, -model, provider, API, scorer, route, profile, contract, timeout, fixture -identity, and case-count fields across the release cohort. Structural, -operational, digest, and cohort failures block before metric comparison. Every -report count applies the same fail-closed report validation. +```sh +bun run scorer-eval --json-out +``` -`compare-baseline.ts` compares five aggregate metrics against the matching model -entry in `bench/baseline.json`: mean authored-target detection, median false or -unrelated finding count, median gate-verdict correctness, maximum per-run mean -provider cost per case, and median per-run nearest-rank p95 review latency. -Detection uses exact count arithmetic over the 57 defect fixtures. A five-report -release candidate passes the detection non-inferiority check when its cohort -mean is no more than two defect detections below the recorded calibration mean. -The two-defect margin is applied to counts, not a rounded percentage. Detection, -p95 latency, and cost are blocking checks; false findings and gate-verdict -correctness remain informational with their medians and complete observed -ranges. The CLI's per-operation cost cap remains the deterministic spending -boundary. +It receives the scorer models, repeat count, provider identity, route, credential, and release binary from the [release workflow](../.github/workflows/release.yml). The scorer screen can reject a scorer but cannot admit a hosted profile. -Baseline recording uses a predeclared calibration cohort of exactly ten -independent complete reports from one frozen binary, corpus, evaluator, provider -profile, and case cohort. A calibration report is not replaced because its -outcome is inconvenient: the ten-report cohort is fixed before execution, and -missing, duplicate, failed, interrupted, or incomplete evidence fails closed. -The `Benchmark calibration` workflow runs only once for the current `main` -commit. Before model execution it creates an immutable, server-protected -`postil-calibration-` registry tag; a failed source cannot be -rerun. It builds and attests one release binary and the ten-slot manifest. -Each slot runs in a separate GitHub-hosted job. The job attests its running -receipt before inference starts, executes the full corpus, and attests the -terminal report and receipt. The fan-in job verifies every offline Sigstore -bundle against the exact repository, workflow, source commit, branch, OIDC -issuer, and GitHub-hosted runner. It also verifies the running-to-completed -receipt transition and independently audits every globally distinct provider -generation before recording the baseline as a workflow artifact. The -baseline records the manifest and source digests, workflow run identity, -each slot and nonce, report and receipt SHA-256 values, normalized outcome -digests, per-run metric distribution, fixture-corpus and evaluator-source -digests, complete case counts, exact provider profile, and the maximum sampled -run cost as a canonical decimal with its case count. Checksums bind content; -GitHub attestations authenticate build and execution provenance. A release -candidate requires the populated baseline and its attestation bundle committed -together. The release verifies that bundle against the calibration workflow, -the recorded source commit, and the immutable calibration registry tag before -using any threshold. The candidate must match the recorded corpus, evaluator, -provider profile, and case cohort. Its five reports must share one candidate -binary. A mismatch blocks comparison across unrelated or incomplete evidence. +Release and calibration use an immutable cohort before model calls: ```sh -# Compare one complete report for a local check. -bun run bench:compare -- \ - --binary \ - --screen-profile ../provisional-models.json \ - --expected-run-id \ - --result - -# Inside the release workflow, create and attest a run-bound manifest before -# any candidate sample starts, then execute each canonical slot once. bun run bench:cohort-create -- \ - --purpose release \ - --binary \ + --purpose \ + --binary \ --screen-profile ../provisional-models.json \ --run-prefix \ - --out -bun run bench:cohort-run -- \ - --mode reserve \ - --manifest \ - --slot <1-through-5> \ - --binary \ - --screen-profile ../provisional-models.json -bun run bench:cohort-run -- \ - --mode execute \ - --manifest \ - --slot <1-through-5> \ - --binary \ - --screen-profile ../provisional-models.json + --out +bun run bench:cohort-run -- --mode reserve --manifest --slot --binary --screen-profile ../provisional-models.json +bun run bench:cohort-run -- --mode execute --manifest --slot --binary --screen-profile ../provisional-models.json +``` -# Run the release comparison over five complete candidate reports and receipts. -bun run bench:compare -- \ - --binary \ - --screen-profile ../provisional-models.json \ - --cohort-manifest \ - --expected-run-id \ - --expected-run-id \ - --expected-run-id \ - --expected-run-id \ - --expected-run-id \ - --result \ - --result \ - --result \ - --result \ - --result \ - --receipt \ - --receipt \ - --receipt \ - --receipt \ - --receipt +Use [the release workflow](../.github/workflows/release.yml) for the five-sample comparison and [the calibration workflow](../.github/workflows/benchmark-calibration.yml) for the ten-sample recorded baseline. Both verify attestations, receipts, and provider generation evidence before comparison or recording. -# The Benchmark calibration workflow invokes the record operation after it -# verifies the attested binary, manifest, reservations, reports, and receipts. -bun run bench:compare -- \ - --binary \ - --screen-profile ../provisional-models.json \ - --cohort-manifest \ - --expected-run-id \ - --expected-run-id \ - --expected-run-id \ - --expected-run-id \ - --expected-run-id \ - --expected-run-id \ - --expected-run-id \ - --expected-run-id \ - --expected-run-id \ - --expected-run-id \ - --result \ - --result \ - --result \ - --result \ - --result \ - --result \ - --result \ - --result \ - --result \ - --result \ - --receipt \ - --receipt \ - --receipt \ - --receipt \ - --receipt \ - --receipt \ - --receipt \ - --receipt \ - --receipt \ - --receipt \ - --record -``` +
+ +## Boundaries and deeper reference + +Live reports record detection, false positives, gate behavior, latency, and provider cost. A successful live run is evidence for its exact binary, fixture bundle, provider route, and model profile. It does not generalize to other models, routes, or source revisions. Public screening results and raw report are at [postil.dev/bench](https://postil.dev/bench). -`--record` accepts exactly ten reports and is the only operation that writes -`bench/baseline.json`. The release workflow never records a baseline. The -comparison table shows the calibration baseline, candidate cohort observation, -verdict, and complete sample range directly in the job log. +Qualification and admission rules, output schemas, evidence retention, scoring limits, and provider constraints are documented in [Model providers](../docs/model-providers.md) and [Architecture](../ARCHITECTURE.md). From f11e39c941a89b5234a723b10b3242f29472e48f Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Sun, 6 Sep 2026 10:29:53 +0000 Subject: [PATCH 2/3] Delegate stored-login inference policy to the hosted service --- docs/configuration.md | 11 ++-- docs/model-providers.md | 2 +- src/cli.rs | 6 +- src/config.rs | 93 ++++++++++++++++++++------- src/llm.rs | 105 +++++++++++++++++++++++++++--- src/main.rs | 11 ++-- src/review.rs | 43 +++++++++---- tests/e2e.rs | 139 +++++++++++++++++++++++++++++++++++++++- 8 files changed, 354 insertions(+), 56 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 080d7aa..1a787fa 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,12 +1,13 @@ # Configuration -Postil resolves model settings in this order: +With an explicit provider API key (BYOK), Postil resolves model settings in this order: 1. Command-line flags 2. Environment variables 3. `.postil.yaml`, `.postil.yml`, or `.postil.json` -4. Stored `postil login` routing -5. Built-in defaults +4. Built-in defaults + +With a stored `postil login` credential, the hosted service controls model, reasoning effort, and provider settings. Local model configuration, environment overrides, and model or effort flags are ignored with a warning. An explicit provider API key takes precedence over a stored login and enables local model settings. Translated `.coderabbit.yaml` settings can supply review policy but do not select a model. @@ -83,7 +84,7 @@ postil logout `postil login` authenticates against the configured login server over a device-authorization flow (open the printed URL, enter the code) and stores a renewable credential at `${XDG_CONFIG_HOME:-~/.config}/postil/credentials.json`, mode `0600` in a `0700` directory. An ambiguous polling failure retries the same device code so the service can recover the already-issued pair during its 60-second recovery window. The credential records its canonical issuing server. Refresh and logout use that stored issuer; a conflicting explicit `POSTIL_LOGIN_SERVER` fails before any login credential is sent. Issuer-free credentials from older CLI versions infer `https://postil.dev` only when their normalized `apiBase` is the canonical Postil inference endpoint. An issuer-free credential for any custom endpoint requires `postil login` again before refresh or logout. -The stored credential is a fallback used only when none of `POSTIL_API_KEY`, `OPENROUTER_API_KEY`, `MODEL_API_KEY`, or `LLM_API_KEY` is set. Postil rotates it before access expiry and persists the replacement; explicit API keys never trigger a refresh. Its `apiBase` is the only endpoint that may receive its bearer. A different `POSTIL_API_BASE` fails before network access unless an explicit API key is set. `REVIEW_MODEL` may still select a model at the stored endpoint. A legacy access-only login, an expired refresh inactivity window, or a rejected refresh instructs the user to run `postil login` again. A valid `Retry-After` value on a temporary refresh rate limit is reported in seconds. Missing or malformed values use a generic retry message, and all temporary refresh failures retain the credential. +The stored credential is a fallback used only when none of `POSTIL_API_KEY`, `OPENROUTER_API_KEY`, `MODEL_API_KEY`, or `LLM_API_KEY` is set. Postil rotates it before access expiry and persists the replacement; explicit API keys never trigger a refresh. Its `apiBase` is the only endpoint that may receive its bearer. A different `POSTIL_API_BASE` fails before network access unless an explicit API key is set. Its stored model is a request hint; review output records the model returned by the service when available. A legacy access-only login, an expired refresh inactivity window, or a rejected refresh instructs the user to run `postil login` again. A valid `Retry-After` value on a temporary refresh rate limit is reported in seconds. Missing or malformed values use a generic retry message, and all temporary refresh failures retain the credential. `postil logout` removes the active local credential only after remote revocation succeeds. A failed revocation retains its sole retry handle and asks the user to run `postil logout` again. A new login stages the newly issued family and any overwritten family in a private pending-revocation queue before replacing the active credential. A local replacement failure therefore retains the new remote family's revocation handle. Pending retries never block normal review work and never follow `POSTIL_LOGIN_SERVER` overrides. @@ -97,4 +98,4 @@ postil doctor `postil config` prints the resolved non-secret configuration and separate provenance for the model, reviewer reasoning effort, and scorer reasoning effort. `postil doctor` validates endpoint reachability, credential acceptance, and repository setup without printing credential values. Both commands identify renewable logins, access expiry, refresh inactivity expiry, and legacy access-only logins. -Use `--reasoning-effort` and `--scorer-reasoning-effort` for one review. These flags override the matching environment variables, which override `model.reasoningEffort` and `model.scorerReasoningEffort`. The built-in reviewer and scorer defaults are both `low`. Every request carries the resolved value, including retries and repair calls. +With BYOK, use `--reasoning-effort` and `--scorer-reasoning-effort` for one review. These flags override the matching environment variables, which override `model.reasoningEffort` and `model.scorerReasoningEffort`. The built-in reviewer and scorer defaults are both `low`. Every request carries the resolved value, including retries and repair calls. Stored-login reviews use the hosted service's reasoning policy. diff --git a/docs/model-providers.md b/docs/model-providers.md index 3506fbd..e6d91d7 100644 --- a/docs/model-providers.md +++ b/docs/model-providers.md @@ -1,6 +1,6 @@ # Model providers -Postil speaks either an OpenAI-compatible chat-completions interface or the native Anthropic Messages API. Provider requests do not follow redirects, credentials are never written to logs, and private-network endpoints require an explicit opt-in. +With BYOK, Postil speaks either an OpenAI-compatible chat-completions interface or the native Anthropic Messages API. Provider requests do not follow redirects, credentials are never written to logs, and private-network endpoints require an explicit opt-in. A stored hosted login delegates model, reasoning, and provider selection to the service; the local overrides described here require an explicit provider API key. ## Model admission diff --git a/src/cli.rs b/src/cli.rs index 0f9f638..75680d2 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -145,13 +145,13 @@ pub enum Command { /// Explicit config file (bypasses discovery). #[arg(long)] config: Option, - /// Model override (else REVIEW_MODEL, else config, else default). + /// BYOK model override. Ignored when a stored login supplies hosted inference. #[arg(long)] model: Option, - /// Reviewer reasoning effort: max|xhigh|high|medium|low|minimal|none (else REVIEW_REASONING_EFFORT, else config, else low). + /// BYOK reviewer effort: max|xhigh|high|medium|low|minimal|none. Ignored with a stored login. #[arg(long, value_name = "EFFORT")] reasoning_effort: Option, - /// Scorer reasoning effort: max|xhigh|high|medium|low|minimal|none (else REVIEW_SCORER_REASONING_EFFORT, else config, else low). + /// BYOK scorer effort: max|xhigh|high|medium|low|minimal|none. Ignored with a stored login. #[arg(long, value_name = "EFFORT")] scorer_reasoning_effort: Option, /// Keep detailed provider, retry, and batch telemetry in interactive terminals. diff --git a/src/config.rs b/src/config.rs index c7e2d39..654cbbb 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,8 +1,9 @@ //! Resolved review configuration. //! -//! Model precedence: CLI flags > environment > `.postil.{yaml,yml,json}` > -//! stored login routing > embedded defaults. Translated `.coderabbit.yaml` -//! settings do not select a model. +//! BYOK model precedence: CLI flags > environment > +//! `.postil.{yaml,yml,json}` > embedded defaults. A stored login supplies the +//! hosted endpoint and model hint; local model, reasoning, and provider settings +//! are ignored. Translated `.coderabbit.yaml` settings do not select a model. //! //! Exception: `model.apiBase` from a config file is repo-controlled, and the //! resolved base URL receives the deployment's bearer key. It is ignored by @@ -15,9 +16,10 @@ //! valid, unexpired credential exists at //! `${XDG_CONFIG_HOME:-~/.config}/postil/credentials.json`, //! `login::resolve_stored_token` supplies its bearer key and its `apiBase`/ -//! `model` provide the baseline below trusted project configuration and -//! environment overrides. A resolved endpoint override is accepted only with -//! an explicit API key; a stored login remains bound to its issuing endpoint. +//! `model` supply the hosted endpoint and model hint. The service selects the +//! actual model. A resolved endpoint override is accepted +//! only with an explicit API key; a stored login remains bound to its issuing +//! endpoint. //! An expired or unreadable stored credential is left alone here; //! `resolve_api_key` reports it as one actionable "run `postil login` again" //! error, while `postil config` reports the stored-login state without needing @@ -1151,6 +1153,9 @@ pub struct Config { pub source: String, /// Winning source for the generator chain, shown by `postil config`. pub model_source: String, + /// Whether the effective inference credential comes from `postil login`. + /// Explicit provider API keys leave this false even when a login is stored. + pub(crate) stored_login_authority: bool, } impl Default for Config { @@ -1188,6 +1193,7 @@ impl Default for Config { content_policy_disabled: false, source: "defaults".to_string(), model_source: "embedded default".to_string(), + stored_login_authority: false, } } } @@ -1265,8 +1271,8 @@ impl Config { #[cfg(test)] let _environment_guard = crate::test_env_lock().lock().unwrap(); let mut cfg = Config::default(); - // Login supplies a local baseline, below trusted project configuration - // and environment overrides but above the embedded provider defaults. + // Resolve credential origin before applying local model configuration. + // A stored login delegates inference policy to its issuing service. cfg.apply_stored_login_credential(); if let Some(path) = explicit { let file = Self::read_postil_file(path) @@ -1328,7 +1334,11 @@ impl Config { } pub fn apply_file(&mut self, f: FileConfig) -> Result<()> { - self.apply_file_inner(f, allow_config_api_base(), repository_model_config_locked()) + self.apply_file_inner( + f, + allow_config_api_base(), + repository_model_config_locked() || self.stored_login_authority, + ) } /// Core of [`apply_file`]. `allow_api_base` decides whether a @@ -1339,7 +1349,7 @@ impl Config { &mut self, f: FileConfig, allow_api_base: bool, - hosted_mode: bool, + model_config_locked: bool, ) -> Result<()> { if let Some(v) = f.enabled { self.enabled = v; @@ -1404,9 +1414,9 @@ impl Config { } } if let Some(m) = f.model { - if hosted_mode { + if model_config_locked { crate::progress::notice(format_args!( - "postil: ignoring repository model configuration in hosted mode; hosted inference selects the provider and model roster" + "postil: ignoring repository model configuration while using hosted inference; the hosted service selects model, reasoning effort, and provider settings" )); } else { if let Some(n) = m.name { @@ -1542,13 +1552,15 @@ impl Config { ), }; } - if let Ok(value) = std::env::var("REVIEW_REASONING_EFFORT") + if !self.stored_login_authority + && let Ok(value) = std::env::var("REVIEW_REASONING_EFFORT") && !value.trim().is_empty() { self.reasoning_effort = ReasoningEffort::parse("REVIEW_REASONING_EFFORT", &value)?; self.reasoning_effort_source = "environment".to_string(); } - if let Ok(value) = std::env::var("REVIEW_SCORER_REASONING_EFFORT") + if !self.stored_login_authority + && let Ok(value) = std::env::var("REVIEW_SCORER_REASONING_EFFORT") && !value.trim().is_empty() { self.scorer_reasoning_effort = @@ -1595,6 +1607,39 @@ impl Config { self.model_source = "qualification candidate profile".to_string(); return Ok(()); } + if self.stored_login_authority { + let ignored = [ + "REVIEW_MODEL", + "REVIEW_MODEL_CASCADE", + "REVIEW_MODEL_CONSENSUS", + "REVIEW_REASONING_EFFORT", + "REVIEW_SCORER_MODEL", + "REVIEW_SCORER_MODEL_CASCADE", + "REVIEW_SCORER_REASONING_EFFORT", + "POSTIL_DISABLE_SCORER", + "POSTIL_API_FORMAT", + "POSTIL_ENDPOINT_AUTH_HEADER", + "POSTIL_ENDPOINT_AUTH_VALUE", + ] + .into_iter() + .filter(|name| std::env::var(name).is_ok_and(|value| !value.trim().is_empty())) + .collect::>(); + if !ignored.is_empty() { + crate::progress::notice(format_args!( + "postil: ignoring local hosted-inference settings {} while using a stored login; set an explicit provider API key to use local model settings", + ignored.join(", ") + )); + } + // Preserve the existing endpoint-binding failure for a conflicting + // POSTIL_API_BASE. Runtime credential resolution compares this + // value with the endpoint recorded by the stored login. + if let Ok(base) = std::env::var("POSTIL_API_BASE") + && !base.is_empty() + { + self.api_base = base; + } + return Ok(()); + } if let Ok(m) = std::env::var("REVIEW_MODEL") && !m.trim().is_empty() { @@ -1658,11 +1703,10 @@ impl Config { Ok(()) } - /// A stored `postil login` credential supplies a local fallback API base - /// and, when present, a model. Legacy credentials with an empty model keep - /// the embedded default. Project configuration and environment variables - /// are applied after this method and therefore retain their documented - /// precedence. + /// A stored `postil login` credential supplies the hosted API base and, + /// when present, model. Legacy credentials with an empty model keep the + /// embedded default. Local model, reasoning, and provider settings are + /// ignored while this credential remains effective. /// Runtime credential resolution rejects an endpoint override when no /// explicit API key is set. The cascade is cleared with the model because /// a BYOK fallback chain does not describe the hosted gateway. @@ -1686,6 +1730,7 @@ impl Config { }; self.api_base = creds.api_base; self.cascade.clear(); + self.stored_login_authority = true; if !creds.model.trim().is_empty() { self.model = creds.model; self.model_source = "stored login".to_string(); @@ -1693,6 +1738,10 @@ impl Config { true } + pub(crate) fn uses_stored_login(&self) -> bool { + self.stored_login_authority + } + /// All models to try, in order, deduplicated. pub fn model_chain(&self) -> Vec { let mut chain = Vec::new(); @@ -3079,7 +3128,7 @@ scorer = { enabled = true, default_model = "provider/scorer", reasoning_effort = } #[test] - fn project_model_provenance_replaces_stored_login_provenance() { + fn stored_login_keeps_credential_model_authority_over_project_config() { let _lock = env_lock().lock().unwrap(); let directory = tempfile::tempdir().unwrap(); let xdg = EnvRestore::capture("XDG_CONFIG_HOME"); @@ -3110,8 +3159,8 @@ scorer = { enabled = true, default_model = "provider/scorer", reasoning_effort = assert_eq!(config.model_source, "stored login"); let file: FileConfig = yaml_serde::from_str("model:\n name: project/model\n").unwrap(); config.apply_file(file).unwrap(); - assert_eq!(config.model, "project/model"); - assert_eq!(config.model_source, "trusted project config"); + assert_eq!(config.model, "hosted/model"); + assert_eq!(config.model_source, "stored login"); drop(saved_keys); drop(xdg); diff --git a/src/llm.rs b/src/llm.rs index 33631d4..5365e40 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -594,6 +594,7 @@ pub struct LlmClient { request_api_base: String, api_key: String, endpoint_auth: Option, + stored_login_authority: bool, request_decorations: RequestDecorations, request_timeout: Duration, timeout_retry_timeout: Duration, @@ -2754,7 +2755,11 @@ impl LlmClient { review_deadline: Option, total_deadline: Option, ) -> Result { - let endpoint_auth = endpoint_auth_from_env(cfg.api_format)?; + let endpoint_auth = if cfg.uses_stored_login() { + None + } else { + endpoint_auth_from_env(cfg.api_format)? + }; let request_decorations = RequestDecorations::from_config(cfg)?; let request_api_base = qualification_request_api_base(&cfg.api_base)?; Ok(LlmClient { @@ -2765,6 +2770,7 @@ impl LlmClient { request_api_base, api_key, endpoint_auth, + stored_login_authority: cfg.uses_stored_login(), request_decorations, request_timeout, timeout_retry_timeout: request_timeout.min(Duration::from_secs(TIMEOUT_RETRY_CAP_SECS)), @@ -2779,6 +2785,14 @@ impl LlmClient { }) } + fn completed_call_model<'a>(&self, requested: &'a str, calls: &'a [ModelUsage]) -> &'a str { + if self.stored_login_authority { + calls.last().map_or(requested, |call| call.model.as_str()) + } else { + requested + } + } + fn model_usage_event( &self, model: &str, @@ -3681,7 +3695,7 @@ impl LlmClient { parsed } }; - let mut review = into_review(raw, model, usage); + let mut review = into_review(raw, self.completed_call_model(model, &call_usage), usage); review.model_usage = call_usage.clone(); review.model_incidents.append(&mut model_incidents); review.usage_accounting_complete = usage_accounting_complete; @@ -3743,7 +3757,11 @@ impl LlmClient { review.model_usage = call_usage.clone(); review.usage_accounting_complete = usage_accounting_complete; if let Ok(retried_raw) = parse_review(&retried) { - let mut candidate = into_review(retried_raw, model, retry_usage); + let mut candidate = into_review( + retried_raw, + self.completed_call_model(model, &call_usage), + retry_usage, + ); candidate.model_usage = call_usage.clone(); candidate.usage_accounting_complete = usage_accounting_complete; let still_contradictory = @@ -3849,7 +3867,11 @@ impl LlmClient { match retry { Ok(content) => match parse_review(&content) { Ok(raw) => { - let mut candidate = into_review(raw, model, retry_usage); + let mut candidate = into_review( + raw, + self.completed_call_model(model, &call_usage), + retry_usage, + ); candidate.model_usage = call_usage.clone(); candidate.model_incidents = review.model_incidents.clone(); candidate.usage_accounting_complete = retry_accounting_complete; @@ -4009,7 +4031,7 @@ impl LlmClient { }, revised_body: resolution.revised_body, evidence: resolution.evidence, - model_used: model.to_string(), + model_used: self.completed_call_model(model, &call_usage).to_string(), usage, model_usage: call_usage, model_incidents, @@ -4059,7 +4081,7 @@ impl LlmClient { })?; Ok(FindingCompressionReview { body: compression.body, - model_used: model.to_string(), + model_used: self.completed_call_model(model, &call_usage).to_string(), usage, model_usage: call_usage, model_incidents: Vec::new(), @@ -4168,7 +4190,7 @@ impl LlmClient { }; Ok(ScorerReview { scores, - model_used: model.to_string(), + model_used: self.completed_call_model(model, &call_usage).to_string(), usage, model_usage: call_usage, model_incidents, @@ -4441,8 +4463,14 @@ impl LlmClient { summary.failure_source = response.failure_source; summary.failure_reason = response.failure_reason; let elapsed = elapsed_text(attempt_started_at.elapsed()); + // The authenticated gateway owns the model choice. Read its + // actual identifier separately from the redacted log summary. + let hosted_response_model = self + .stored_login_authority + .then(|| actual_response_identity(&response.text).0) + .flatten(); call_usage.push(self.model_usage_event( - model, + hosted_response_model.as_deref().unwrap_or(model), phase, call_phase, attempt, @@ -7900,6 +7928,67 @@ mod tests { } } + #[tokio::test] + async fn stored_login_schema_repair_reports_each_returned_model() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .and(body_string_contains("You repair malformed JSON")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "model": "cloud/repair-model", + "choices": [{"finish_reason": "stop", "message": {"content": "{\"summary\":\"\",\"findings\":[]}"}}], + "usage": {"prompt_tokens": 20, "completion_tokens": 3} + }))) + .with_priority(1) + .expect(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "model": "cloud/initial-model", + "choices": [{"finish_reason": "stop", "message": {"content": "malformed"}}], + "usage": {"prompt_tokens": 10, "completion_tokens": 2} + }))) + .with_priority(2) + .expect(1) + .mount(&server) + .await; + let config = Config { + api_base: server.uri(), + model: "stored/stale-model".into(), + stored_login_authority: true, + scorer_enabled: false, + ..Config::default() + }; + let client = LlmClient::build( + &config, + "test-key".into(), + Duration::from_secs(2), + None, + None, + ) + .unwrap(); + *client.http.lock().unwrap() = Some(reqwest::Client::new()); + let result = client + .review_validated_with_safe_output_limit( + &config, + "system", + "user", + 4_000, + ReviewRequestRoute::Source, + |_| Ok(()), + ) + .await + .unwrap(); + assert_eq!(result.model_used, "cloud/repair-model"); + assert_eq!(result.model_usage.len(), 2); + assert_eq!(result.model_usage[0].model, "cloud/initial-model"); + assert_eq!(result.model_usage[1].model, "cloud/repair-model"); + assert_eq!(result.usage.prompt_tokens, 30); + assert_eq!(result.usage.completion_tokens, 5); + } + #[tokio::test] async fn synthesis_schema_repair_keeps_explicit_route_metadata() { let server = MockServer::start().await; diff --git a/src/main.rs b/src/main.rs index 073b7b5..7332cb8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -185,13 +185,13 @@ async fn dispatch(cli: Cli) -> anyhow::Result { println!(" This binary contains no standalone hosted qualification profile."); } println!("\nCheck the configured endpoint and model: postil doctor"); - println!("Override once: postil review --model provider/model"); + println!("BYOK override once: postil review --model provider/model"); println!( - "Override reasoning once: postil review --reasoning-effort high --scorer-reasoning-effort none" + "BYOK reasoning override once: postil review --reasoning-effort high --scorer-reasoning-effort none" ); - println!("Override persistently: REVIEW_MODEL=provider/model postil review"); + println!("BYOK persistent override: REVIEW_MODEL=provider/model postil review"); println!( - "Persist reasoning: REVIEW_REASONING_EFFORT=high REVIEW_SCORER_REASONING_EFFORT=none postil review" + "BYOK persistent reasoning: REVIEW_REASONING_EFFORT=high REVIEW_SCORER_REASONING_EFFORT=none postil review" ); println!( "Native Anthropic config: set model.apiBase, model.apiFormat: anthropic, and model.name, then run postil doctor" @@ -199,6 +199,9 @@ async fn dispatch(cli: Cli) -> anyhow::Result { println!( "Config keys: model.name, model.reasoningEffort, and model.scorerReasoningEffort" ); + println!( + "Hosted login: the Postil service selects model, reasoning, and provider settings" + ); Ok(0) } Command::Plan { envelopes, config } => { diff --git a/src/review.rs b/src/review.rs index 42779cc..bb412bd 100644 --- a/src/review.rs +++ b/src/review.rs @@ -632,18 +632,37 @@ pub async fn run(args: ReviewArgs) -> Result { ); } let mut cfg = Config::load(&cwd, args.config.as_deref())?; - if let Some(m) = &args.model { - cfg.model = m.clone(); - cfg.model_source = "command line".to_string(); - } - if let Some(effort) = &args.reasoning_effort { - cfg.reasoning_effort = ReasoningEffort::parse("--reasoning-effort", effort)?; - cfg.reasoning_effort_source = "command line".to_string(); - } - if let Some(effort) = &args.scorer_reasoning_effort { - cfg.scorer_reasoning_effort = - ReasoningEffort::parse("--scorer-reasoning-effort", effort)?; - cfg.scorer_reasoning_effort_source = "command line".to_string(); + if cfg.uses_stored_login() { + let ignored = [ + args.model.as_ref().map(|_| "--model"), + args.reasoning_effort.as_ref().map(|_| "--reasoning-effort"), + args.scorer_reasoning_effort + .as_ref() + .map(|_| "--scorer-reasoning-effort"), + ] + .into_iter() + .flatten() + .collect::>(); + if !ignored.is_empty() { + crate::progress::notice(format_args!( + "postil: ignoring {} while using a stored login; the hosted service selects model and reasoning settings. Set an explicit provider API key to use these options", + ignored.join(", ") + )); + } + } else { + if let Some(m) = &args.model { + cfg.model = m.clone(); + cfg.model_source = "command line".to_string(); + } + if let Some(effort) = &args.reasoning_effort { + cfg.reasoning_effort = ReasoningEffort::parse("--reasoning-effort", effort)?; + cfg.reasoning_effort_source = "command line".to_string(); + } + if let Some(effort) = &args.scorer_reasoning_effort { + cfg.scorer_reasoning_effort = + ReasoningEffort::parse("--scorer-reasoning-effort", effort)?; + cfg.scorer_reasoning_effort_source = "command line".to_string(); + } } cfg.require_model()?; if let Some(fo) = &args.fail_on { diff --git a/tests/e2e.rs b/tests/e2e.rs index 5741e09..47d26ba 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -1418,11 +1418,148 @@ async fn explicit_byok_key_remains_valid_with_an_api_base_override() { .env("POSTIL_API_BASE", server.uri()) .env("MODEL_API_KEY", provider_key) .env("POSTIL_DISABLE_SCORER", "1") - .args(["review", "--diff-file"]) + .args([ + "review", + "--model", + "byok/model", + "--reasoning-effort", + "high", + "--diff-file", + ]) .arg(&diff) .args(["--output", "json"]) .assert() .success(); + + let requests = server.received_requests().await.unwrap(); + let body: Value = requests[0].body_json().unwrap(); + assert_eq!(body["model"], "byok/model"); + assert_eq!(body["reasoning"], json!({"effort": "high"})); +} + +#[test] +fn stored_login_ignores_repository_and_environment_model_policy() { + let dir = tempfile::tempdir().unwrap(); + let config_home = dir.path().join("config"); + let credentials_dir = config_home.join("postil"); + std::fs::create_dir_all(&credentials_dir).unwrap(); + std::fs::write( + credentials_dir.join("credentials.json"), + serde_json::to_vec(&json!({ + "version": 3, + "issuer": "https://postil.dev", + "token": "pcli_e2e-access-not-a-real-secret", + "expiresAt": "2999-01-01T00:00:00.000Z", + "apiBase": "https://postil.dev/api/inference/v1", + "org": "example", + "model": "hosted/model" + })) + .unwrap(), + ) + .unwrap(); + std::fs::write( + dir.path().join(".postil.yaml"), + "model:\n name: repository/model\n reasoningEffort: max\n cascade: [repository/fallback]\n scorer: repository/scorer\n scorerReasoningEffort: high\n apiFormat: anthropic\n consensus: 3\n", + ) + .unwrap(); + + let assertion = isolated_postil() + .current_dir(dir.path()) + .env("XDG_CONFIG_HOME", &config_home) + .env("REVIEW_MODEL", "environment/model") + .env("REVIEW_REASONING_EFFORT", "turbo") + .env("REVIEW_SCORER_MODEL", "environment/scorer") + .env("POSTIL_API_FORMAT", "anthropic") + .env("POSTIL_ENDPOINT_AUTH_HEADER", "x-provider-auth") + .env("POSTIL_ENDPOINT_AUTH_VALUE", "fixture-value") + .arg("config") + .assert() + .success(); + let stdout = String::from_utf8(assertion.get_output().stdout.clone()).unwrap(); + let stderr = String::from_utf8(assertion.get_output().stderr.clone()).unwrap(); + + assert!(stdout.contains("model.name: hosted/model")); + assert!(stdout.contains("model.source: stored login")); + assert!(stdout.contains("model.reasoningEffort: low")); + assert!(stdout.contains("model.reasoningEffort.source: embedded default")); + assert!(stdout.contains("model.cascade: []")); + assert!(stdout.contains("model.apiFormat: openai-compatible")); + assert!(!stdout.contains("repository/")); + assert!(!stdout.contains("environment/")); + assert!(stderr.contains("ignoring repository model configuration")); + assert!(stderr.contains("ignoring local hosted-inference settings")); + assert!(stderr.contains("REVIEW_MODEL")); + assert!(stderr.contains("REVIEW_REASONING_EFFORT")); + assert!(stderr.contains("POSTIL_API_FORMAT")); + assert!(stderr.contains("POSTIL_ENDPOINT_AUTH_HEADER")); +} + +#[tokio::test] +async fn stored_login_ignores_command_line_model_and_reasoning_policy() { + let server = MockServer::start().await; + let mut response = llm_content(json!([])); + response["model"] = json!("cloud/current-model"); + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with(ResponseTemplate::new(200).set_body_json(response)) + .mount(&server) + .await; + let dir = tempfile::tempdir().unwrap(); + let config_home = dir.path().join("config"); + let credentials_dir = config_home.join("postil"); + std::fs::create_dir_all(&credentials_dir).unwrap(); + std::fs::write( + credentials_dir.join("credentials.json"), + serde_json::to_vec(&json!({ + "version": 3, + "issuer": server.uri(), + "token": "pcli_e2e-access-not-a-real-secret", + "expiresAt": "2999-01-01T00:00:00.000Z", + "apiBase": server.uri(), + "org": "example", + "model": "hosted/model" + })) + .unwrap(), + ) + .unwrap(); + let diff = write_diff(dir.path()); + + let assertion = isolated_postil() + .current_dir(dir.path()) + .env("XDG_CONFIG_HOME", &config_home) + .env("POSTIL_ALLOW_PRIVATE_API_BASE", "1") + .env("POSTIL_ENDPOINT_AUTH_HEADER", "x-provider-auth") + .env("POSTIL_ENDPOINT_AUTH_VALUE", "fixture-value") + .args([ + "review", + "--model", + "command/model", + "--reasoning-effort", + "turbo", + "--scorer-reasoning-effort", + "max", + "--diff-file", + ]) + .arg(&diff) + .args(["--output", "json"]) + .assert() + .success(); + + let requests = server.received_requests().await.unwrap(); + let body: Value = requests[0].body_json().unwrap(); + assert_eq!(body["model"], "hosted/model"); + assert_eq!(body["reasoning"], json!({"effort": "low"})); + assert_eq!(body["max_tokens"], 8_000); + assert_eq!(body["temperature"], 0.1); + assert!(body.get("provider").is_none()); + assert!(body.get("response_format").is_none()); + assert!(requests[0].headers.get("x-provider-auth").is_none()); + let envelope: Value = serde_json::from_slice(&assertion.get_output().stdout).unwrap(); + assert_eq!(envelope["modelUsed"], "cloud/current-model"); + assert_eq!(envelope["modelUsage"][0]["model"], "cloud/current-model"); + let stderr = String::from_utf8(assertion.get_output().stderr.clone()).unwrap(); + assert!(stderr.contains("ignoring --model, --reasoning-effort, --scorer-reasoning-effort")); + assert!(stderr.contains("hosted service selects model and reasoning settings")); } #[tokio::test] From c9bbe747d98ac282f1277fbb0fbbc042e1c38637 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Sun, 6 Sep 2026 11:28:36 +0000 Subject: [PATCH 3/3] Fix stored login authority assignment --- src/config.rs | 17 +++++++++++------ tests/e2e.rs | 2 +- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/config.rs b/src/config.rs index 654cbbb..e6cffb5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1273,7 +1273,7 @@ impl Config { let mut cfg = Config::default(); // Resolve credential origin before applying local model configuration. // A stored login delegates inference policy to its issuing service. - cfg.apply_stored_login_credential(); + cfg.stored_login_authority = cfg.apply_stored_login_credential(); if let Some(path) = explicit { let file = Self::read_postil_file(path) .with_context(|| format!("reading config {}", path.display()))?; @@ -1730,7 +1730,6 @@ impl Config { }; self.api_base = creds.api_base; self.cascade.clear(); - self.stored_login_authority = true; if !creds.model.trim().is_empty() { self.model = creds.model; self.model_source = "stored login".to_string(); @@ -3115,9 +3114,13 @@ scorer = { enabled = true, default_model = "provider/scorer", reasoning_effort = unsafe { std::env::remove_var(other) }; } EnvRestore::set(name, "provider-fixture-key"); - let mut cfg = Config::default(); + let mut cfg = Config { + stored_login_authority: true, + ..Config::default() + }; + cfg.stored_login_authority = cfg.apply_stored_login_credential(); assert!( - !cfg.apply_stored_login_credential(), + !cfg.uses_stored_login(), "{name} must suppress login routing" ); assert_eq!(cfg.api_base, model_defaults().api_base, "{name}"); @@ -3155,7 +3158,8 @@ scorer = { enabled = true, default_model = "provider/scorer", reasoning_effort = .unwrap(); let mut config = Config::default(); - assert!(config.apply_stored_login_credential()); + config.stored_login_authority = config.apply_stored_login_credential(); + assert!(config.uses_stored_login()); assert_eq!(config.model_source, "stored login"); let file: FileConfig = yaml_serde::from_str("model:\n name: project/model\n").unwrap(); config.apply_file(file).unwrap(); @@ -3196,7 +3200,8 @@ scorer = { enabled = true, default_model = "provider/scorer", reasoning_effort = .unwrap(); let mut config = Config::default(); - assert!(config.apply_stored_login_credential()); + config.stored_login_authority = config.apply_stored_login_credential(); + assert!(config.uses_stored_login()); assert_eq!(config.api_base, "https://postil.dev/api/inference/v1"); assert_eq!(config.model, model_defaults().default_model); assert_eq!(config.model_source, "embedded default"); diff --git a/tests/e2e.rs b/tests/e2e.rs index 47d26ba..a3730c7 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -1399,7 +1399,7 @@ async fn explicit_byok_key_remains_valid_with_an_api_base_override() { serde_json::to_vec(&json!({ "version": 3, "issuer": "https://postil.dev", - "token": "pcli_e2e-access-not-a-real-secret", + "token": "fixture-stored-login-token", "expiresAt": "2999-01-01T00:00:00.000Z", "refreshToken": "fixture-e2e-refresh-not-a-credential", "refreshExpiresAt": "2999-12-01T00:00:00.000Z",