diff --git a/.claude/skills/commit-and-pr/SKILL.md b/.claude/skills/commit-and-pr/SKILL.md index 69f36dc3..340ae899 100644 --- a/.claude/skills/commit-and-pr/SKILL.md +++ b/.claude/skills/commit-and-pr/SKILL.md @@ -25,10 +25,10 @@ Turn finished work into a clean, auditable commit history and a reviewable PR. R ## Workflow 1. **Gate first.** Run ../standards-check/SKILL.md and `pnpm turbo run lint typecheck test`. Do not commit red. -2. **Branch if on the default branch.** Never commit straight to `main`. +2. **Commit on `development`, never straight to `main`.** Work reaches `main` through a PR from `development`; create no other branch unless the maintainer asks (the [Git workflow](../../../AGENTS.md#git-workflow) in AGENTS.md is the canonical rule). ```bash git rev-parse --abbrev-ref HEAD - # if main: git switch -c feat/- + # if main: git switch development ``` 3. **One scope per commit.** Stage only one package's changes. A change spanning packages is usually two commits; if genuinely atomic, pick the primary scope and name the rest in the body. Do not bundle generated/formatting churn with logic. 4. **Write the message** in Conventional Commits form. Summary is imperative, lower-case, no trailing period, ≤ ~72 chars. Body (expected for anything non-obvious) explains the *why*, not a restatement of the diff. Reference the ADR in a trailer; end with the required co-author trailer. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a50a68e..6e224e37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,10 +3,12 @@ # Runs lint + typecheck + test across every workspace in dependency order with the # Turborepo cache, plus the formatting check and the no-vendor-type-across-the-seam fence. # -# Branch protection (set in the GitHub repo settings, not here): `ci` and `coverage` are the REQUIRED -# status checks to merge into `main`. Wire, usage, filesystem, secrets and packed CLI jobs feed `ci` through an explicit -# success-only aggregation; they cannot silently skip or fail while `ci` stays green. The other jobs -# self-label `(advisory)` — `floor-check`, `peer-dep-gate` and `windows-concurrency`. +# Branch protection (set in the GitHub repo settings, not here): `ci`, `coverage` and `windows-concurrency` +# are the REQUIRED status checks to merge into `main` (the Windows job since 2026-11-04: ADR-0106's +# acceptance names required CI for the platform-scoped claims it carries). Wire, usage, filesystem, secrets, +# packed CLI and product map control jobs feed `ci` through an explicit success-only aggregation; they cannot +# silently skip or fail while `ci` stays green. The other jobs self-label `(advisory)` — `floor-check` and +# `peer-dep-gate`. # # `coverage` enforces the testing.md >=90% line+branch floor for `@relavium/llm` and `@relavium/mcp` only. # `packages/core` is measured and printed by the same run but does not fail it: its branch margin is +0.83 @@ -57,7 +59,8 @@ env: jobs: ci: - needs: [wire-mutations, usage-mutations, fs-mutations, secrets-mutations, packed-cli] + needs: + [wire-mutations, usage-mutations, fs-mutations, secrets-mutations, packed-cli, map-controls] if: ${{ always() }} name: lint · typecheck · test runs-on: ubuntu-latest @@ -102,17 +105,20 @@ jobs: PACKED_CLI_RESULT: ${{ needs.packed-cli.result }} run: node -e "require('node:assert/strict').equal(process.env.PACKED_CLI_RESULT, 'success')" + - name: Require complete product map controls + env: + MAP_CONTROLS_RESULT: ${{ needs.map-controls.result }} + run: node -e "require('node:assert/strict').equal(process.env.MAP_CONTROLS_RESULT, 'success')" + # A squash/main-based checkout need not contain development's immutable predecessor object. # Fetch only this canonical pin during CI setup; the acceptance harness itself stays offline. - name: Ensure immutable replay predecessor is available shell: bash run: | BASELINE=$(node --input-type=module -e "import { baselineCommit } from './tools/budget-replay-compat/baseline-provenance.mjs'; process.stdout.write(baselineCommit)") - if ! git --no-replace-objects cat-file -e "$BASELINE^{commit}"; then - if ! git fetch --no-tags origin "$BASELINE"; then - echo "::error::Immutable replay predecessor $BASELINE is unavailable from origin; restore that exact object before running the offline gate." - exit 1 - fi + if ! git --no-replace-objects cat-file -e "$BASELINE^{commit}" && ! git fetch --no-tags origin "$BASELINE"; then + echo "::error::Immutable replay predecessor $BASELINE is unavailable from origin; restore that exact object before running the offline gate." + exit 1 fi git --no-replace-objects cat-file -e "$BASELINE^{commit}" @@ -245,13 +251,21 @@ jobs: # Offline, uncached semantic removal/restoration evidence. Required indirectly through ci.needs. # ADR-0105 isolated, current-source removal/restoration evidence, required through ci.needs. fs-mutations: - name: Wave 2 filesystem causal controls + name: Wave 2 filesystem causal controls (shard ${{ matrix.shard }} of 2) # ADR-0106/0109 extend the original 26 controls with complete media/lifetime workloads. # The 20-minute job historically reached only 46 of 93 controls; the later budget covered 99. - # The current cohort is 110 controls (2026-10-15); this comment does not certify its execution. + # The current cohort is 115 controls (2026-10-25); this comment does not certify its execution. # Per-phase 90-second/test 15-second deadlines and output/evidence ceilings remain unchanged. + # Every phase retains a manifest of the whole source graph (about 200 KB each), so the 330 phase + # manifests of one export passed the unchanged 64-MiB retained-evidence bound once Wave 3's PR95 + # increment grew that graph (2026-10-22). As for the secrets controls, two legs each run and + # export a disjoint round-robin half (`--shard k/2`) under the unchanged bounds. timeout-minutes: 60 runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2] steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: @@ -268,23 +282,32 @@ jobs: - name: Run filesystem removal and restoration controls env: RELAVIUM_ACCEPTANCE_EVIDENCE: ${{ runner.temp }}/wave2-acceptance - run: pnpm check:fs-mutations + run: pnpm check:fs-mutations --shard ${{ matrix.shard }}/2 - name: Retain bounded filesystem control metadata if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: wave2-filesystem-${{ github.run_id }}-${{ github.run_attempt }} + name: wave2-filesystem-${{ matrix.shard }}-of-2-${{ github.run_id }}-${{ github.run_attempt }} path: ${{ runner.temp }}/wave2-acceptance/filesystem if-no-files-found: warn retention-days: 7 secrets-mutations: - name: Wave 2 secrets causal controls + name: Wave 2 secrets causal controls (shard ${{ matrix.shard }} of 2) # The historical 35-control, three-phase capture exceeded the former aggregate budget. - # The current cohort is 57 controls (2026-10-18); this comment does not certify its execution. - # Per-phase, case/hook, output and retained-evidence bounds remain unchanged. - timeout-minutes: 30 + # Wave 3's PR95 increment grows the cohort to 130 controls (2026-10-21): each scrubber + # control runs its eight-MiB-heavy file in three phases (about 24 s locally). One export + # of the whole cohort no longer fits the unchanged 64-MiB retained-evidence bound, so two + # legs each run and export a disjoint round-robin half (`--shard k/2`) under the + # unchanged per-phase, case/hook, output, 2,000-file and 64-MiB bounds. A leg took up to 48 minutes + # by 2026-11-04 (runner variance, not imbalance), so its allowance is 75 minutes rather than the + # filesystem job's 60. This comment does not certify execution. + timeout-minutes: 75 runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2] steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: @@ -301,12 +324,12 @@ jobs: - name: Run secrets removal and restoration controls env: RELAVIUM_ACCEPTANCE_EVIDENCE: ${{ runner.temp }}/wave2-acceptance - run: pnpm check:secrets-mutations + run: pnpm check:secrets-mutations --shard ${{ matrix.shard }}/2 - name: Retain bounded secrets control metadata if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: wave2-secrets-${{ github.run_id }}-${{ github.run_attempt }} + name: wave2-secrets-${{ matrix.shard }}-of-2-${{ github.run_id }}-${{ github.run_attempt }} path: ${{ runner.temp }}/wave2-acceptance/secrets if-no-files-found: warn retention-days: 7 @@ -365,11 +388,9 @@ jobs: shell: bash run: | BASELINE=$(node --input-type=module -e "import { baselineCommit } from './tools/budget-replay-compat/baseline-provenance.mjs'; process.stdout.write(baselineCommit)") - if ! git --no-replace-objects cat-file -e "$BASELINE^{commit}"; then - if ! git fetch --no-tags origin "$BASELINE"; then - echo "::error::Immutable replay predecessor $BASELINE is unavailable from origin; restore that exact object before running the offline gate." - exit 1 - fi + if ! git --no-replace-objects cat-file -e "$BASELINE^{commit}" && ! git fetch --no-tags origin "$BASELINE"; then + echo "::error::Immutable replay predecessor $BASELINE is unavailable from origin; restore that exact object before running the offline gate." + exit 1 fi git --no-replace-objects cat-file -e "$BASELINE^{commit}" @@ -472,8 +493,9 @@ jobs: runs-on: ubuntu-latest # One file worker isolates unchanged admitted-size case budgets from instrumentation # contention. A measured 1,481-second complete run plus setup exhausted 25 minutes; - # reserve finite whole-suite headroom without changing case bounds or coverage thresholds. - timeout-minutes: 30 + # reserve finite whole-suite headroom without changing case bounds or coverage thresholds. By + # 2026-11-04 the job took 15-27 minutes, up to 91% of the former 30, so the allowance is 45. + timeout-minutes: 45 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: @@ -520,7 +542,7 @@ jobs: # perm assertions are NOT exercised here (a documented Windows no-op — ADR-0050); nor is ink's raw-mode code, # which is TTY-gated (the driver-selection gate picks the plain driver without a TTY — see the smoke below). windows-concurrency: - name: windows · db concurrency + filesystem + headless smoke (advisory) + name: windows · db concurrency + filesystem + headless smoke runs-on: windows-latest timeout-minutes: 25 steps: @@ -619,7 +641,11 @@ jobs: node-version-file: .nvmrc - name: Install (frozen lockfile) run: pnpm install --frozen-lockfile - # The runner builds inlined dependencies fresh, then prepack performs the sole final CLI build. + # The runner builds inlined dependencies fresh in an owned candidate tree, then prepack performs the sole + # final CLI build, where original-TS maps are built, validated, staged privately and discarded (Wave 3 + # Step 9a.1); the seal then writes the private manifest and symbolicates actual packed frames, both + # discarded too (Step 9a.2a); only the tarball and its package_test receipt leave this job. The mode is a + # claim: the map-controls job below runs the negative controls, and `ci` requires its success (Step 9a.2b). - name: Pack once and independently inspect the public package env: RELAVIUM_PACKAGE_SOURCE_SHA: ${{ github.sha }} @@ -634,6 +660,35 @@ jobs: retention-days: 7 if-no-files-found: error + # Product map controls (Wave 3 Step 9a.2b): a second, independent real production at this commit whose + # every named corruption must refuse with its exact code; its private maps, manifests and copies stay in + # the producer's owner-only root and are removed with it. Uncached, and nothing is uploaded. + map-controls: + name: product map controls + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + env: + TURBO_TOKEN: '' + TURBO_TEAM: '' + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + - name: Set up pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + - name: Set up Node + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: .nvmrc + - name: Install (frozen lockfile) + run: pnpm install --frozen-lockfile + - name: Corrupt product copies and require each exact refusal + env: + RELAVIUM_PACKAGE_SOURCE_SHA: ${{ github.sha }} + run: pnpm check:map-controls + packed-cli-consumer: name: installed CLI (${{ matrix.id }}) needs: pack-cli diff --git a/.github/workflows/models-catalog.yml b/.github/workflows/models-catalog.yml index 44359506..7c637a93 100644 --- a/.github/workflows/models-catalog.yml +++ b/.github/workflows/models-catalog.yml @@ -113,6 +113,33 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} + # One configured key keeps the run from being a false green, but each missing key still skips its + # provider's lane while the job stays green (a PR #96 review: only DEEPSEEK_API_KEY was configured). + # Name every lane as live or skipped, as a warning and in the job summary, until all four keys exist + # (decision D37) and a missing one can fail the run instead. Only emptiness is tested; no value prints. + - name: Report which provider lanes run live + shell: bash + run: | + { + echo '### Live effort conformance lanes' + echo '' + echo '| Provider | Lane |' + echo '|---|---|' + } >> "$GITHUB_STEP_SUMMARY" + for provider in ANTHROPIC OPENAI GEMINI DEEPSEEK; do + variable="${provider}_API_KEY" + if [ -n "${!variable}" ]; then + echo "| ${provider} | live |" >> "$GITHUB_STEP_SUMMARY" + else + echo "::warning::${variable} is not configured; the ${provider} live effort probes are skipped, not verified." + echo "| ${provider} | skipped: no key |" >> "$GITHUB_STEP_SUMMARY" + fi + done + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} # The conformance suite (fixtures + the `it.skipIf( === '')` live cases). With the provider keys set here # the live cases execute — and a catalog that claims a tier a model rejects fails red — while a missing key # simply skips that provider's lane. Scoped to `src/conformance` so a flaky live case cannot widen the red diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 83c0ea3e..0e1d17a9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,6 +3,19 @@ # Step 7b reuses the required installed-package runner, including the exact Node 22.0.0 runtime floor. # Step 9 retains ownership of original-TS maps, release custody and their permission/discovery redesign. # +# PUBLICATION IS REFUSED until that custody exists (Wave 3 Step 9a.1, the Interim release guard ruling of +# 2026-10-19): `publish` first runs a fail-closed guard that refuses every artifact. Only a verified release +# candidate (exact tarball, original ciphertext and release binding receipt) may publish, and Step 9d's custody +# validator, which replaces the guard, is what will verify one; Wave 4b supplies the production recipient. Until +# both exist every tag push stops at the guard, after the cross-OS smoke, so not even a v0.1.x patch publishes. +# The flow description below is the pre-custody shape the guard now blocks. +# +# Amended 2026-11-04 (decision D59, release path B): the condition above is to be superseded by a pre-1.0 +# release ADR that publishes the smoke-verified package-test tarball, with a sunset at 1.0 or when Steps +# 9c/9d land. That ADR must explicitly override ADR-0111's "missing custody cannot downgrade to test mode". +# Until it is Accepted and its guard change passes a security sitting, the guard below stays as it is. +# `publish` runs in the `release` deployment environment, so the npm token can be scoped to it. +# # Flow: `pack` builds the engine-inlined bundle and `pnpm pack`s it (resolving catalog:/workspace: to # concrete versions) into a tarball artifact → `smoke` installs THAT SAME tarball outside the workspace on ubuntu / # macOS / Windows and the minimum Node runtime and asserts the binary works (prebuilt native addons load, the bundled engine runs) → @@ -92,8 +105,12 @@ jobs: cache: pnpm - name: Install (frozen lockfile) run: pnpm install --frozen-lockfile - # The same required package runner owns fresh dependencies and the sole prepack CLI build. - # Original-TS maps and encrypted release custody are staged separately in Wave 3 Step 9. + # The same required package runner owns fresh dependencies and the sole prepack CLI build in an owned + # candidate tree, where original-TS maps are built, validated, staged privately and discarded (Wave 3 + # Step 9a.1), with the private manifest and the packed-frame symbolication (Step 9a.2a); their + # encrypted release custody is Steps 9b–9d. The receipt records package_test, a claim: the product map + # controls run only in the CI workflow, and back an artifact only where a successful CI run exists at + # exactly its commit (Step 9a.2b; packaging.md). - name: Pack once and independently inspect the public package env: RELAVIUM_PACKAGE_SOURCE_SHA: ${{ github.sha }} @@ -210,18 +227,26 @@ jobs: # manually dispatched run publish. Every manual run now stops after the cross-OS smoke. if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} runs-on: ubuntu-latest + # The deployment environment that holds `NPM_TOKEN` and its required reviewer (repo settings); a + # repository-level secret is readable by any workflow a writer pushes. It sets no job environment. + environment: release timeout-minutes: 10 permissions: contents: read id-token: write # npm provenance attestation steps: - # Checkout ONLY `.nvmrc` (sparse, non-cone) so `setup-node` can read the single Node-version source the - # pack/smoke jobs use too; publish needs no other repo files — it pushes the downloaded tarball, not a - # fresh build. (fetch-depth defaults to 1, so it is not set again.) + # Checkout ONLY `.nvmrc` and the dependency-free publication guard, its entry point and its rule (sparse, + # non-cone): `setup-node` reads the single Node-version source the pack/smoke jobs use too, and the guard + # runs before any publish. + # Publish needs no other repo files — it pushes the downloaded tarball, not a fresh build. + # (fetch-depth defaults to 1, so it is not set again.) - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: persist-credentials: false - sparse-checkout: .nvmrc + sparse-checkout: | + .nvmrc + tools/cli-package/release-guard.mjs + tools/cli-package/release-guard-core.mjs sparse-checkout-cone-mode: false - name: Set up Node (npm registry) uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 @@ -233,6 +258,13 @@ jobs: with: name: ${{ needs.packed-cli.outputs.artifact-name }} path: artifact + # Interim fail-closed guard (Wave 3 Step 9a.1, the Interim release guard ruling of 2026-10-19): only a + # verified release candidate may publish, and none can exist before Step 9d's custody graph and Wave 4b's + # production recipient. Every artifact, including this package-test tarball, therefore stops here; the + # entry point sets its exit code to 1 before it evaluates anything, so a rule that returns, throws or + # cannot load still refuses, and no way of invoking it can skip the refusal. + - name: Refuse anything but a verified release candidate + run: node tools/cli-package/release-guard.mjs ./artifact # Publish the EXACT artifact the smoke matrix proved — not a fresh build. catalog:/workspace: are already # resolved in the packed manifest, so `npm publish ` is correct here (pnpm-pack did the resolving). - name: Publish diff --git a/AGENTS.md b/AGENTS.md index 8d7dc760..8ecf7add 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,105 +22,15 @@ Turborepo + pnpm monorepo (`packages/shared`, `packages/llm`, `packages/core`, **cloud** (BYOK-central, Phase 2), and **managed** (Relavium's own keys via a metered egress gateway; engine stays local, Phase 2) — split across build phase 5 (managed inference) and phase 6 (cloud execution + portal); the engine is identical across all three (ADR-0012..0015). -**Status: Phase 1 is complete; Phase 2 (CLI) is feature-complete (published as v0.1.1); -Phase 2.5 (CLI Consolidation) is complete (M2.5-4, 2026-07-08); Phase 2.6 -(Conversational Authoring and the First-Class CLI) is in progress, with an interlude — -Phase 2.6.5 (Core reliability remediation) — completed between remediation Waves 1 and 2: -49 of 51 items closed on `main`, with approved deferrals retained; W8 merged 2026-10-11 in PR #92 (`CR-96` and `CR-97` were opened 2026-09-14, `CR-98` on 2026-09-18; `W7` merged to `main` on 2026-10-10 (PR #90), with ADR-0095–ADR-0103 -accepted) — all eight P0 blockers (ADR-0078–ADR-0084, merged 2026-08-24), the `W2` -liveness-and-deadlines wave (ADR-0085, merged 2026-08-28 as PR #85), `W3` resource governance and bounds (ADR-0086), merged 2026-08-30 as PR #86 **with a live blocker and nine open findings at that merge** — the blocker is now closed by W8/PR #92; scoped residuals remain in docs/roadmap/deferred-tasks.md — and `W4`, the hostile MCP boundary (ADR-0088), merged 2026-09-01 as PR #87 — a systematic review of the branch found five merge blockers, all reproduced and fixed before it merged. `W5`, media correctness (ADR-0089 + ADR-0090), **merged 2026-09-02 as PR #88** — a systematic review of that branch returned six merge blockers, all reproduced and fixed before the merge. `W6`, authoring correctness (ADR-0091–ADR-0094), **MERGED 2026-09-04 as PR #89** — `merge_strategy: first` means first DECLARED, a widened tool grant is refused when the plan is BUILT, an expression sees only its transitive closure with a literal out-of-closure read refused at parse, and an `output_schema` is compiled at parse and enforced at run time with **no new dependency**. Seven internal review rounds found that most defects were in the FIXES rather than the code they repaired; a systematic maintainer review then returned two merge blockers, both breaking a headline claim of the wave, plus eight further High findings — all reproduced and fixed before the merge. Each is a dated correction inside its own ADR. -**W7 merged to `main` on 2026-10-10 as PR #90, including final whole-wave Step 12.** -Steps 1–11, all five genuine overflow records, ADR-0102 request ownership and consolidated ADR-0103 -host integration have independent acceptance. ADR-0095–ADR-0103 are Accepted. All six W7 register -items are closed (47/51 at the W7 merge); PR #90 is merged (`46a6a701`); W7 has no open implementation or review gate. The -[closing register](docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md#w7-closing-register--2026-10-10) -joins evidence and every submitted review disposition. W8 subsequently merged in PR #92; approved deferrals remain. -No user decision, billing setup, credential or provider capture is pending. -**W8 merged to `main` on 2026-10-11 as PR #92 (`ef0ffa54`); Phase 2.6.5 is complete with approved deferrals.** -All seven steps, CR-81/82 and ADR-0087 §1 are complete on `main`; the register is **49/51**. -The [merge checkpoint](docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md#w8-merge-checkpoint--2026-10-11) -joins original whole-wave acceptance, maintainer corrections and final reviewed-head `3d4491df` -PR/push CI and SonarCloud success. Published ordinary/enforced coverage records 511 files / -11,026 passing tests / 12 established skips; all four wire and 23 usage mechanisms pass. -CR-70/93, CR-95's long continuation and exact-capacity/scoped residuals remain approved deferrals. -The joint [Wave 2 — Shut the doors](docs/roadmap/current.md#next-work--wave-2--shut-the-doors) merged to `main` on 2026-10-15 as PR #94; -next is [Wave 3 — Clear the ground](docs/roadmap/current.md#wave-3--clear-the-ground). -Phase 2.5.5 and Phase 2.6 remain in progress. No W8 implementation or landing gate remains. -**Post-closure corrections are independently accepted, 2026-10-10:** core, CLI/DB and tooling/docs -each have two NEW clean complete cumulative review rounds. The supplemental publication-tool -correction also has two NEW complete clean rounds. The [post-closure register](docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md#post-closure-systematic-review--2026-10-10) -joins every report disposition, new permanent/causal evidence and current gates. Published-head checks -passed at reviewed head `69ac80b0` before PR #90 merged. No W7 implementation, provider, -billing, key or user decision is pending. The [merge checkpoint](docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md#w7-merge-checkpoint--2026-10-10) -records W7 closure. W8 subsequently merged in PR #92; the register is 49/51 on `main`. -Existing approved limits are unchanged. - -**PR #92 maintainer corrections:** the [correction register](docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md#w8-pr-92-systematic-review-corrections--2026-10-11) -records all four increments and the subsequent publication correction accepted after two NEW -clean complete cumulative rounds each. Fresh full CI and bare/enforced coverage pass at `6139563e`, -including four wire and 23 usage controls. -Final reviewed head `3d4491df` passed PR/push CI and SonarCloud before PR #92 merged. -No merge-commit execution or new whole-PR review is inferred. Two nonblocking Sonar complexity -suggestions remain visible; no new deferral or decision is added. - -**Joint Wave 2 merged to `main` — 2026-10-15, PR #94 (`ab08209e`).** -All eight implementation steps and the maintainer corrections are complete on `main` -under Accepted ADR-0105–0109. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) -joins forty item dispositions, security sittings, the final reviewed source `4cdfe011`, -two NEW clean complete cumulative correction rounds and successful PR/push CI, -coverage, native-platform and independently inspected physical controls. Earlier -whole-wave and correction reviews retain their exact source scopes; no new whole-PR -review or merge-commit execution is inferred. Detailed review records remain local -and Git-ignored. - -Exit criteria 1–3 and the D half of M2.5.5-2 are satisfied on `main`. The I half, -whole Phase 2.5.5/2.6, full M/N production lifecycle, future live provider headers -and existing approved deferrals remain open in their canonical homes. No Wave 2 -implementation, review, publication, ADR approval, provider capture, credential, -billing or user-decision gate remains. Next is [Wave 3 — Clear the ground](docs/roadmap/current.md#wave-3--clear-the-ground), -whose ADR-0110–0113/contracts/nineteen-increment plan were approved on 2026-10-16. -Its serialized multi-file Step 1 is independently accepted on `development` -(2026-10-16); Step 2 is also accepted after two NEW clean complete cumulative rounds -following its historical-reader corrections. Step 3 (ADR-0113 secret provenance) -is independently accepted after two NEW complete cumulative clean rounds and a -separate clean security sitting. Step 4 (error/envelope substrate) is independently -accepted after two NEW complete cumulative clean rounds. Step 5 (tools and engine -tails) is independently accepted after two NEW complete cumulative clean rounds -following its registry-documentation correction and a separate clean security -sitting. Step 6 (pure process/discovery substrate) is independently accepted after -two NEW complete cumulative clean rounds and a separate clean security sitting. -Step 7a (lazy runtime and finite compiled graph) is independently accepted after -two NEW complete cumulative clean rounds and a separate clean security sitting. -Step 7b and whole Step 7 are independently accepted at `c836ffe6` after two NEW -complete cumulative clean rounds, a clean security sitting and successful exact-tarball -Linux/macOS/Windows plus Node 22.0.0 consumers. Broader CI retains its separately -attributed status. Step 8a implements origin adapters and an inert native foundation; -it is independently accepted at `e0a5ae19` after two NEW complete cumulative -clean rounds and a separate clean security sitting. -Step 8b (creation-owned native processes) is independently accepted at -`e16a7659` after two NEW complete cumulative clean rounds and a separate clean -security sitting following its canonical-reference correction. Its unchanged -runtime/control evidence and successful full CI plus all four same-tarball -platform consumers remain attributed to `ab50be11`, with independently verified -source/dependency equivalence. Step 8c workflow/preparation/terminal integration -is independently accepted at `dbab25d8` after two NEW complete cumulative clean -rounds, a separate clean security sitting and successful full CI. All four -same-tarball platform consumers pass at that reviewed source. -**Step 8d and whole Step 8 are independently accepted on `development` at -`c83c8575` — 2026-10-17.** Two NEW complete cumulative CLEAN rounds (R25/R26) -each cover all 342 paths / 1,842 ordered hunks after the final fixture correction. -A NEW separate clean security sitting and all fifteen full-CI jobs pass at that -reviewed source, as do all four actual same-tarball platform consumers. Original -failed reviewer-preparation records and source/native/platform limits remain -explicit; source CI does not certify later documentation commits. The -[acceptance register](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) -joins exact evidence and historical corrections. Steps 1–8 are accepted; -Steps 9–19 have NOT STARTED. After the private handoff, the maintainer authorized -an initial PR for completed Steps 1–8. This is a scoped publication checkpoint; -whole-wave acceptance remains Step 19. Later work stays with the next agent. -Check that PR's status before committing Step 9 on `development`, because new -commits extend its review scope. No provider, credential, billing or new ADR -approval is pending for the approved implementation plan. +**Status (2026-11-04): the CLI MVP comes first.** Phases 1, 2 and 2.5 are complete (`relavium@0.1.1` is +published); Phase 2.6.5 is complete with approved deferrals (49/51, PR #92). Phase 2.5.5's Waves 0–2 and Wave 3 +Steps 1–8 are merged (PR #94, PR #95), and Steps 9a.1–9b.2 and NI are accepted on `development`. On 2026-10-26 +the maintainer paused Wave 3 (Steps 9c–19 pending, not declined) and put the CLI MVP first: `relavium` v0.2.0 +(milestone M3.5), ordered as MVP-1…MVP-10 under decisions D58–D72 in +[current.md](docs/roadmap/current.md#next-work--the-cli-mvp). Phase 2.6 is in progress (2.6.C and 2.6.F done, +2.6.Q P1–P5 landed). The interim release guard blocks npm publication until the pre-1.0 release ADR (D59) is +Accepted; the acceptance bar is tiered (D64), with security-touching increments keeping a full review round and +a separate security sitting. CLAUDE.md carries the full summary. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. @@ -166,7 +76,8 @@ Never `npm` or `yarn`. `workspace:*` for inter-package deps; no circular deps. ## Git workflow -- Trunk-based on `main`; short-lived feature branches via PR. +- Work lands on the long-lived `development` branch and reaches the protected `main` (required CI + checks) through a PR from `development`. Do not create other branches unless the maintainer asks. - Conventional Commits with a per-package scope (`feat(core):`, `fix(llm):`, `docs(decisions):`). Reference the ADR or task the change advances. See [docs/standards/commit-style.md](docs/standards/commit-style.md). diff --git a/CLAUDE.md b/CLAUDE.md index 8c828f2a..de059aaa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -39,177 +39,24 @@ A run executes in one of **three execution modes** behind the one `LLMProvider` engine is identical across all three. See [ADR-0012](docs/decisions/0012-managed-inference-dual-mode.md) to [ADR-0015](docs/decisions/0015-managed-mode-data-handling-and-compliance.md) and [docs/architecture/managed-inference.md](docs/architecture/managed-inference.md). -**Status: Phase 1 is complete** (2026-06-21) — the engine runs end-to-end behind the -`LLMProvider` seam with all three adapters, `AgentSession`, multimodal I/O, and the full -run-loop. **Phase 2 (CLI) is feature-complete** (published as v0.1.1): -`relavium chat` / `chat-resume` / `agent run`, `relavium run` with `--json`, the -inbound MCP client, and the full YAML lifecycle (`create`/`import`/`export`). -**Phase 2.5 (CLI Consolidation) is complete** (M2.5-4, PR #69, 2026-07-08): the bare-invocation -Home, the two-registry command system (`/` palette + shell commands), per-tool -approval/modes (ask/plan/accept-edits/auto), `@`-mention file injection, context -compaction, the onboarding wizard + live model catalog, reasoning rendering, and regression -hardening. **Phase 2.6 (Conversational Authoring and the First-Class CLI) is in progress** — two -workstreams are merged to `main`: **2.6.F** (the platform floor + the full-screen TUI renderer, -PR #74, 2026-07-11) and **2.6.C** (the reseat transcript-carry + the `/cost` per-model breakdown, -PR #75, 2026-07-13). The phase is a full-screen Home-managed CLI with conversational workflow authoring, management -browsers, competitor-breadth tools, settings/theming/`en`+`tr` localization, and the -run-ops resume follow-up. - -**The interlude between remediation Waves 1 and 2 is complete: Phase 2.6.5 (Core reliability), -49 of 51 items closed on `main`, with approved CR-70/93 and scoped deferrals retained. -W8 merged on 2026-10-11 as PR #92; W7 had restored the non-deferrable criterion** — `W0` (PR #82, 2026-08-11), `W1`, the eight P0 blockers plus `CR-92`, merged -2026-08-24 (PR #83) behind [ADR-0078](docs/decisions/0078-ordered-durable-append-and-the-terminal-outbox.md)–[ADR-0084](docs/decisions/0084-consent-before-a-local-mcp-spawn.md): -ordered durable append, cross-process run ownership, the durable effect journal, untrusted compaction summaries, -the stream-grammar seam obligation, engine-side input admission and resume identity, and consent before a local -MCP spawn. **`W2` — liveness and deadlines — merged 2026-08-28 (PR #85)** behind -[ADR-0085](docs/decisions/0085-the-node-executor-owes-liveness-and-the-engine-enforces-it.md): the authored -agent `timeout_ms` now bounds the node, every media call is bounded, a resume re-arms the run cap and a -pending gate at their REMAINING time rather than renewing them (the node bound is the deliberate exception — -ADR-0085 §2 records why), and an executor that ignores its signal can no longer leave a run without a -terminal. **`W3` — resource governance and bounds — merged 2026-08-30 (PR #86) behind -[ADR-0086](docs/decisions/0086-absolute-admission-ceilings-on-authored-values.md)**: authored values carry -absolute admission ceilings, node output / workflow state / durable events are size-bounded, and a long-lived -engine no longer retains every finished run. **It merged with a live blocker** — an un-pulled event stream -drops the terminal event, breaching ADR-0036's gap-free contract — plus nine verified findings, all recorded -in the `W3` residuals of [deferred-tasks.md](docs/roadmap/deferred-tasks.md). -[ADR-0087](docs/decisions/0087-consumed-streams-size-bounds-and-run-retention.md) records the fix and, after a -review that refused acceptance until nine dated corrections existed, was **Accepted (2026-09-04) with its §1 -unimplemented**. That blocker is now closed by W8/PR #92. Its §3 turned out to be implemented already, in `W5`. **`W4` — the hostile MCP boundary — merged 2026-09-01 (PR #87) -behind [ADR-0088](docs/decisions/0088-the-mcp-boundary-is-hostile.md)**: every MCP call is bounded and -cancellable, `http`/`sse` connect by validated pinned IP, a redirect is refused, a remote `websocket` is -refused at admission, a server's ingress is bounded at two levels, and its tool DEFINITIONS are treated as -untrusted. No new product surface in either wave — only the invariants an existing surface already claims. -A systematic review of PR #87 found **five merge blockers** in `W4` — all reproduced and fixed before it -merged; the phase doc's `W4` section records what they were and what the pattern says. **`W5` — media correctness — MERGED 2026-09-02 (PR #88)** behind -[ADR-0089](docs/decisions/0089-media-correctness-four-boundaries.md) and -[ADR-0090](docs/decisions/0090-a-continuation-token-rides-the-part-it-belongs-to.md): `read_media` delivers -over the media-input rail on a marked, fenced synthesized message; tool and attachment capability is gated on -the MODEL; Gemini's function-call continuation token rides the part it belongs to; a media body streams from -the network into the store under a size ceiling, an idle deadline and the run signal; a `url` output is pinned -to a content-addressed handle at first resolution; and a missing media rate is unpriced rather than a price of -zero. A systematic maintainer review of the branch returned **six merge blockers**, all reproduced and fixed -before it merged. Two items closed one half of a two-part obligation and say so; every residual is written out -rather than left inside a checked box. **`W6` — authoring correctness — MERGED 2026-09-04 (PR #89)** behind [ADR-0091](docs/decisions/0091-first-means-first-declared-not-first-to-finish.md)–[ADR-0094](docs/decisions/0094-a-tool-grant-is-checked-when-the-plan-is-built.md): -`merge_strategy: first` means first DECLARED and the plan field no longer claims otherwise; a widened tool grant -is refused when the plan is BUILT rather than mid-run; an expression sees only its transitive dependency closure -and a literal out-of-closure `run.outputs` read is refused at parse (as is `edges[].condition`, which nothing -ever read); and an authored `output_schema` is compiled at parse in an allowlist-strict mode and enforced at run -time — with **no new dependency**, because the JSON-Schema→Zod compiler the deferral claimed we needed already -existed in `packages/mcp` and simply moved to `@relavium/shared`. Seven internal review rounds found that **most defects -were in the FIXES rather than the code they repaired** — the branch-order search was wrong three times, the -expression scan's no-false-refusal claim was false in eight ways, and a stand-down gate silently removed the -check it was added to protect; each is a dated correction inside its own ADR. A systematic maintainer review -then returned **two merge blockers**, both breaking a headline claim of the wave — `required` was not enforced -when a property's schema constrained nothing, and the expression scanner's regex was quadratic and synchronous -on a parser that accepts 2 MiB — plus eight further High findings, all reproduced and fixed before the merge. - - -**Between `W6` and `W7`, the two remaining non-deferrable items closed (2026-09-06)**: `CR-73` — `invoke_agent` -was advertised to every model granted it while `ctx.invokeAgent` is wired nowhere in the tree, so the -advertise-filter now checks a tool's declared dispatch DELEGATE and not only its `ToolHost` arm (`read_media` -rides along, for the same reason and by `CR-50`'s own argument) — and `CR-80` — a rejected custom provider -`base_url` was caught and the DEFAULT adapter left standing, which is the official API, so a drifted config -silently sent prompts and keys to `api.openai.com`; it now installs a refusing adapter naming the URL's shape, -never its credentials. The test that was supposed to pin `CR-80` asserted only `toBeDefined()`, which a -refusing adapter satisfies too, so it passed either way — the finding under the defect was the test. -[ADR-0087](docs/decisions/0087-consumed-streams-size-bounds-and-run-retention.md) was also **accepted -(2026-09-04) with nine dated corrections** after a review refused it as written; §1 was still unimplemented -at that checkpoint, while §3 turned out to have shipped in `W5` under a different item number. -**W8 merged to `main` on 2026-10-11 as PR #92 (`ef0ffa54`); Phase 2.6.5 is complete with approved deferrals.** -All seven steps, CR-81/82 and ADR-0087 §1 are complete on `main`; the register is **49/51**. -The [merge checkpoint](docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md#w8-merge-checkpoint--2026-10-11) -joins the original whole-wave acceptance, all maintainer-correction dispositions and final reviewed-head -`3d4491df` PR/push CI plus SonarCloud success. The ordinary/enforced-coverage inventory is -511 files / 11,026 passing tests / 12 established skips; four wire and 23 usage mechanisms pass. -Every correction increment has two NEW independent complete clean cumulative rounds after its final fix; -scoped correction reviews are not a new whole-PR certification. Fresh full local CI and bare/enforced -coverage at `6139563e` retain their separate source attribution. No merge-commit execution is inferred. -Two nonblocking Sonar complexity suggestions remain visible. Historical counts and pending states -throughout the registers describe their original checkpoints. - -CR-70 (cross-turn tool-history carrying), CR-93 (tenant-scoped mutable state), CR-95's long continuation -and existing exact-capacity/scoped residuals retain their approved limits in -[deferred-tasks.md](docs/roadmap/deferred-tasks.md). No new deferral, provider capture, billing, -credential or W8 user decision is pending. The joint [Wave 2 — Shut the doors](docs/roadmap/current.md#next-work--wave-2--shut-the-doors) merged to `main` on 2026-10-15 as PR #94; -next is [Wave 3 — Clear the ground](docs/roadmap/current.md#wave-3--clear-the-ground). -Phase 2.5.5 and Phase 2.6 remain in progress. - -**W7 merged to `main` on 2026-10-10 as PR #90, including final whole-wave Step 12.** -Steps 1–11, all five genuine overflow records, owned measured requests and consolidated provider/MCP/host -lifetimes have independent acceptance. ADR-0095–ADR-0103 are Accepted. The six W7 register items are closed -(47/51 at the W7 merge); PR #90 is merged (`46a6a701`); W7 has no open implementation or review gate. The -[closing register](docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md#w7-closing-register--2026-10-10) -joins the accepted scopes, causal controls and systematic-review dispositions. Existing approved deferrals -stay in [deferred-tasks.md](docs/roadmap/deferred-tasks.md); W8 subsequently merged in PR #92 and closed the phase with approved deferrals. No user decision, -billing setup, credential or provider capture is pending. - -**Post-closure corrections are independently accepted, 2026-10-10:** core, CLI/DB and tooling/docs -each have two NEW clean complete cumulative review rounds. The supplemental publication-tool -correction also has two NEW complete clean rounds. The [post-closure register](docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md#post-closure-systematic-review--2026-10-10) -joins every report disposition, new permanent/causal evidence and current gates. Published-head checks -passed at reviewed head `69ac80b0` before PR #90 merged. No W7 implementation, provider, -billing, key or user decision is pending. The [merge checkpoint](docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md#w7-merge-checkpoint--2026-10-10) -records W7 closure. W8 subsequently merged in PR #92; the register is 49/51 on `main`. -Existing approved limits are unchanged; W8 has no open implementation or landing gate. - -**Joint Wave 2 merged to `main` — 2026-10-15, PR #94 (`ab08209e`).** -All eight implementation steps and the maintainer corrections are complete on `main` -under Accepted ADR-0105–0109. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) -joins forty item dispositions, security sittings, the final reviewed source `4cdfe011`, -two NEW clean complete cumulative correction rounds and successful PR/push CI, -coverage, native-platform and independently inspected physical controls. Earlier -whole-wave and correction reviews retain their exact source scopes; no new whole-PR -review or merge-commit execution is inferred. Detailed review records remain local -and Git-ignored. - -Exit criteria 1–3 and the D half of M2.5.5-2 are satisfied on `main`. The I half, -whole Phase 2.5.5/2.6, full M/N production lifecycle, future live provider headers -and existing approved deferrals remain open in their canonical homes. No Wave 2 -implementation, review, publication, ADR approval, provider capture, credential, -billing or user-decision gate remains. Next is [Wave 3 — Clear the ground](docs/roadmap/current.md#wave-3--clear-the-ground), -whose ADR-0110–0113/contracts/nineteen-increment plan were approved on 2026-10-16. -Its serialized multi-file Step 1 is independently accepted on `development` -(2026-10-16); Step 2 is also accepted after two NEW clean complete cumulative rounds -following its historical-reader corrections. Step 3 (ADR-0113 secret provenance) -is independently accepted after two NEW complete cumulative clean rounds and a -separate clean security sitting. Step 4 (error/envelope substrate) is independently -accepted after two NEW complete cumulative clean rounds. Step 5 (tools and engine -tails) is independently accepted after two NEW complete cumulative clean rounds -following its registry-documentation correction and a separate clean security -sitting. Step 6 (pure process/discovery substrate) is independently accepted after -two NEW complete cumulative clean rounds and a separate clean security sitting. -Step 7a (lazy runtime and finite compiled graph) is independently accepted after -two NEW complete cumulative clean rounds and a separate clean security sitting. -Step 7b and whole Step 7 are independently accepted at `c836ffe6` after two NEW -complete cumulative clean rounds, a clean security sitting and successful exact-tarball -Linux/macOS/Windows plus Node 22.0.0 consumers. Broader CI retains its separately -attributed status. Step 8a implements origin adapters and an inert native foundation; -it is independently accepted at `e0a5ae19` after two NEW complete cumulative -clean rounds and a separate clean security sitting. -Step 8b (creation-owned native processes) is independently accepted at -`e16a7659` after two NEW complete cumulative clean rounds and a separate clean -security sitting following its canonical-reference correction. Its unchanged -runtime/control evidence and successful full CI plus all four same-tarball -platform consumers remain attributed to `ab50be11`, with independently verified -source/dependency equivalence. Step 8c workflow/preparation/terminal integration -is independently accepted at `dbab25d8` after two NEW complete cumulative clean -rounds, a separate clean security sitting and successful full CI. All four -same-tarball platform consumers pass at that reviewed source. -**Step 8d and whole Step 8 are independently accepted on `development` at -`c83c8575` — 2026-10-17.** Two NEW complete cumulative CLEAN rounds (R25/R26) -each cover all 342 paths / 1,842 ordered hunks after the final fixture correction. -A NEW separate clean security sitting and all fifteen full-CI jobs pass at that -reviewed source, as do all four actual same-tarball platform consumers. Original -failed reviewer-preparation records and source/native/platform limits remain -explicit; source CI does not certify later documentation commits. The -[acceptance register](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) -joins exact evidence and historical corrections. Steps 1–8 are accepted; -Steps 9–19 have NOT STARTED. After the private handoff, the maintainer authorized -an initial PR for completed Steps 1–8. This is a scoped publication checkpoint; -whole-wave acceptance remains Step 19. Later work stays with the next agent. -Check that PR's status before committing Step 9 on `development`, because new -commits extend its review scope. No provider, credential, billing or new ADR -approval is pending for the approved implementation plan. +**Status (2026-11-04): the CLI MVP comes first.** Phase 1 (engine and LLM) is complete; Phase 2 +(CLI) is feature-complete and `relavium@0.1.1` is the published version; Phase 2.5 (CLI Consolidation) is +complete (M2.5-4, PR #69, 2026-07-08); Phase 2.6.5 (Core reliability) is complete with approved deferrals +(49/51, PR #92, 2026-10-11). Phase 2.5.5 (Hardening and Remediation): Waves 0–2 are merged (Wave 2: PR #94), +Wave 3 Steps 1–8 are merged (PR #95, 2026-10-18), and PR95, 9a.1, 9a.2a, 9a.2b, NI, 9b.1 and 9b.2 are +accepted on `development`. **On 2026-10-26 the maintainer paused Wave 3 — Steps 9c–19 are pending, not +declined — and put the CLI MVP first:** `relavium` v0.2.0 on npm (milestone M3.5), ordered as MVP-1…MVP-10 in +[current.md](docs/roadmap/current.md#next-work--the-cli-mvp) under decisions D58–D72 (ruled 2026-11-04). +Phase 2.6 (Conversational Authoring and the First-Class CLI) is in progress: 2.6.C and 2.6.F are done +(2.6.F's v0.2.0 release clause closes with the MVP) and 2.6.Q P1–P5 landed; the MVP takes slices of the rest. + +Two rules shape every MVP increment. Step 9a.1's interim release guard on `development` refuses any npm +publication until the superseding pre-1.0 release ADR (D59) is Accepted. The acceptance bar is tiered (D64): +an increment touching an approval or consent path, egress, keys or secrets, filesystem writes, a new terminal +sink for untrusted content, the lease or resume paths, or release keeps a rigor pass, one NEW complete +cumulative round with no blocker, high or medium finding, and a separate security sitting; other UX, +rendering and command increments are reviewed per milestone. `development` is pushed after each accepted +increment. For live status, per-PR history, milestone dates, and open obligations, see the canonical home [docs/roadmap/current.md](docs/roadmap/current.md); diff --git a/README.md b/README.md index c28b08d6..77123328 100644 --- a/README.md +++ b/README.md @@ -158,9 +158,11 @@ Relavium is under active development. The **CLI is published as v0.1.1**; the pu agent-session entry point, workflow runtime, inbound MCP client, and CLI management surface are implemented. **Phase 2.6.5 — Core Reliability Remediation is complete with approved deferrals**, with W8 merged in PR #92. The joint Phase 2.5.5/2.6 security remediation -Wave 2 merged to `main` on 2026-10-15 as PR #94. The next -engineering wave is “Clear the ground”: CLI safety, shared guards and decomposition -of the hot files needed by later product work. +Wave 2 merged to `main` on 2026-10-15 as PR #94, and the first eight steps of Wave 3 +(“Clear the ground”) followed on 2026-10-18 as PR #95. Since 2026-10-26 the **CLI MVP +comes first**: Wave 3 is paused, not dropped, while the work a CLI user feels — visible +run results, a chat agent that can edit and run commands, readable replies, conversational +authoring, web, the Home hub and MCP management — leads to `relavium` v0.2.0. Status changes quickly, so this README intentionally stays high-level. The canonical source for the exact active wave, completed work, and open reliability obligations is diff --git a/apps/cli/package.json b/apps/cli/package.json index 101ab679..f6842501 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -39,8 +39,8 @@ }, "scripts": { "build": "tsup", - "prepack": "tsup && pnpm -w run lint:bundle-closure", - "typecheck": "tsc -p tsconfig.json --noEmit", + "prepack": "node ../../tools/cli-package/prepack.mjs", + "typecheck": "tsc -p tsconfig.json --noEmit && tsc -p tsconfig.config.json --noEmit", "lint": "eslint src", "test": "vitest run" }, diff --git a/apps/cli/src/authoring/snapshot-notice.test.ts b/apps/cli/src/authoring/snapshot-notice.test.ts index 35cf71dc..fbe6a0ac 100644 --- a/apps/cli/src/authoring/snapshot-notice.test.ts +++ b/apps/cli/src/authoring/snapshot-notice.test.ts @@ -48,6 +48,30 @@ describe('authored snapshot credential notice', () => { expect(err()).toBe(''); }); + it('keeps placeholder passwords and header tokens quiet and reports a literal http(s) username', () => { + const quiet = captureIo(); + warnSnapshotLiterals( + { + env: { + DATABASE_URL: 'postgres://app:{{secrets.db_password}}@db.example/app', + CACHE_URL: 'redis://:{{secrets.cache_password}}@cache.example:6379', + AUTH_HEADER: 'x-api-key: Bearer {{secrets.provider_key}}', + QUOTED_HEADER: 'x-api-key: Bearer "{{secrets.provider_key}}"', + QUOTED_BASIC: "x-api-key: Basic '{{secrets.provider_key}}'", + }, + }, + quiet.io, + ); + expect(quiet.err()).toBe(''); + // http(s)/ws(s)/ftp(s) mask any userinfo, so a literal username beside a placeholder still counts. + const reported = captureIo(); + warnSnapshotLiterals( + { env: { REPOSITORY: 'https://app:{{secrets.token}}@git.example/repo.git' } }, + reported.io, + ); + expect(reported.err()).toContain('1 string(s) matching known credential shapes'); + }); + it('does not exempt credential literals beside placeholders or inside filter arguments', () => { const literal = 'sk-templateSyntheticCredential123456789'; const { io, err } = captureIo(); diff --git a/apps/cli/src/commands/fs-invocation.test.ts b/apps/cli/src/commands/fs-invocation.test.ts index 71d7cb7d..1574e977 100644 --- a/apps/cli/src/commands/fs-invocation.test.ts +++ b/apps/cli/src/commands/fs-invocation.test.ts @@ -519,10 +519,10 @@ it('revalidates a revoked policy before binding an agent or connecting MCP durin ); if (options === undefined) throw new Error('seed builder was not reached'); // Keep the admitted directory alive so Linux cannot recycle its inode for the replacement. - const original = statSync(outside); + const original = statSync(outside, { bigint: true }); renameSync(outside, `${outside}-old`); mkdirSync(outside); - const replacement = statSync(outside); + const replacement = statSync(outside, { bigint: true }); expect([replacement.dev, replacement.ino]).not.toEqual([original.dev, original.ino]); const startMcpClient = vi .fn>() diff --git a/apps/cli/src/commands/native-forwarding.test-support.ts b/apps/cli/src/commands/native-forwarding.test-support.ts index e80f460f..e58f100b 100644 --- a/apps/cli/src/commands/native-forwarding.test-support.ts +++ b/apps/cli/src/commands/native-forwarding.test-support.ts @@ -10,13 +10,6 @@ import { type NativeProcessController, } from '../process/native-process-owner.js'; -/** - * These waits observe a REAL spawned child (or the retirement reaction of one). vitest's default - * `waitFor` timeout is 1 s, which a cold or loaded worker can exceed before a child has booted. - * Kept under the 5 s default test timeout so a genuine failure surfaces as its own assertion. - */ -const REAL_CHILD_WAIT_MS = 4_000; - /** A physical child witness for host-only forwarding, with an independent tool timeout. */ export async function witnessNativeToolProcess( nativeProcesses: NativeProcessController | undefined, @@ -38,7 +31,6 @@ export async function witnessNativeToolProcess( // queued. Require its actual removal before measuring the stable native-parent baseline. await vi.waitFor(() => expect(invocation.registry.activeOwners).toBe(expectedParentOwners), { interval: 1, - timeout: REAL_CHILD_WAIT_MS, }); const pending = host.process.spawn( process.execPath, @@ -51,12 +43,11 @@ export async function witnessNativeToolProcess( expect(invocation.registry.activeOwners, 'NATIVE_PROCESS_ASSEMBLY_OWNERSHIP').toBe( expectedParentOwners + 1, ), - { interval: 1, timeout: REAL_CHILD_WAIT_MS }, + { interval: 1 }, ); expect(await pending).toMatchObject({ exitCode: 0, stdout: 'native-witness' }); await vi.waitFor(() => expect(invocation.registry.activeOwners).toBe(expectedParentOwners), { interval: 1, - timeout: REAL_CHILD_WAIT_MS, }); } @@ -103,7 +94,7 @@ export async function holdNativeToolProcess( expect(existsSync(marker)).toBe(true); // physical native entry, not only a reserved field expect(invocation.registry.activeOwners).toBe(before + 1); }, - { interval: 1, timeout: REAL_CHILD_WAIT_MS }, + { interval: 1 }, ); return { close }; } catch (error) { diff --git a/apps/cli/src/engine/tool-host/fs-policy.test.ts b/apps/cli/src/engine/tool-host/fs-policy.test.ts index eea82e41..c553ce37 100644 --- a/apps/cli/src/engine/tool-host/fs-policy.test.ts +++ b/apps/cli/src/engine/tool-host/fs-policy.test.ts @@ -7,6 +7,7 @@ import { realpathSync, renameSync, rmSync, + statSync, symlinkSync, writeFileSync, } from 'node:fs'; @@ -79,6 +80,9 @@ beforeEach(() => { }); afterEach(() => rmSync(base, { recursive: true, force: true })); +// A real replacement could compare equal one run in sixty on Windows before identities were exact (NI). +const REPLACEMENT_REPEATS = { repeats: 20 }; + function capability(policy: FsPathPolicy, tier: FsScopeTier = 'project') { return createNodeFsCapability({ tier, @@ -160,6 +164,7 @@ describe('invocation authority and owned native identities', () => { it.each(['directory', 'alias'] as const)( 'revokes a replaced %s after admission, before the first tool call', + REPLACEMENT_REPEATS, async (kind) => { const root = kind === 'directory' ? grant : join(base, 'alias'); if (kind === 'alias') symlinkSync(grant, root, 'junction'); @@ -180,24 +185,30 @@ describe('invocation authority and owned native identities', () => { }, ); - it('pins workspace and selected-subroot identities and owns caller arrays/config', async () => { - const roots = [grant]; - const input = { - tier: 'project' as const, - workspaceDir: workspace, - readOnly: true, - extraRoots: roots, - }; - const fs = createNodeFsCapability(input); - roots.length = 0; - input.readOnly = false; - expect((await fs.readFile('../grant/ok.txt', {})).content).toBe('grant'); - await expect(fs.writeFile('new.txt', 'denied', {})).rejects.toThrow(/read.only|not available/); - renameSync(workspace, `${workspace}-old`); - mkdirSync(workspace); - writeFileSync(join(workspace, 'ok.txt'), 'replacement'); - await expect(fs.readFile('ok.txt', {})).rejects.toThrow(/pinned identities/); - }); + it( + 'pins workspace and selected-subroot identities and owns caller arrays/config', + REPLACEMENT_REPEATS, + async () => { + const roots = [grant]; + const input = { + tier: 'project' as const, + workspaceDir: workspace, + readOnly: true, + extraRoots: roots, + }; + const fs = createNodeFsCapability(input); + roots.length = 0; + input.readOnly = false; + expect((await fs.readFile('../grant/ok.txt', {})).content).toBe('grant'); + await expect(fs.writeFile('new.txt', 'denied', {})).rejects.toThrow( + /read.only|not available/, + ); + renameSync(workspace, `${workspace}-old`); + mkdirSync(workspace); + writeFileSync(join(workspace, 'ok.txt'), 'replacement'); + await expect(fs.readFile('ok.txt', {})).rejects.toThrow(/pinned identities/); + }, + ); it('returns reusable extra-root paths through write, glob and mention without a doubled anchor', async () => { const fs = capability( @@ -410,6 +421,7 @@ it.each([ describe('exact private host scratch descriptors (no production allocator)', () => { it.each(['tmp', 'artifacts'])( 'permits only the issued %s tree through both jailed factory tiers', + REPLACEMENT_REPEATS, async (kind) => { const home = join(base, 'home'); const scratch = join(home, '.relavium', kind, 'parent'); @@ -520,18 +532,22 @@ it.each(['workspace', 'grant', 'selector'] as const)( }, ); -it('revokes a selected subdirectory replacement even when its enclosing grant is unchanged', async () => { - const selected = join(grant, 'selected'); - mkdirSync(selected); - const fs = capability( - selectInvocationFsPolicy(admitInvocationFsGrants([grant]), workspace, [selected]), - ); - renameSync(selected, selected + '-old'); - mkdirSync(selected); - writeFileSync(join(selected, 'a.txt'), 'replacement'); - await expect(fs.readFile(join(selected, 'a.txt'), {})).rejects.toThrow(/pinned identities/); - expect(observed.open).not.toHaveBeenCalled(); -}); +it( + 'revokes a selected subdirectory replacement even when its enclosing grant is unchanged', + REPLACEMENT_REPEATS, + async () => { + const selected = join(grant, 'selected'); + mkdirSync(selected); + const fs = capability( + selectInvocationFsPolicy(admitInvocationFsGrants([grant]), workspace, [selected]), + ); + renameSync(selected, selected + '-old'); + mkdirSync(selected); + writeFileSync(join(selected, 'a.txt'), 'replacement'); + await expect(fs.readFile(join(selected, 'a.txt'), {})).rejects.toThrow(/pinned identities/); + expect(observed.open).not.toHaveBeenCalled(); + }, +); it('never repairs a failed construction by later creating the missing workspace', async () => { const missing = join(base, 'missing-workspace'); @@ -573,3 +589,27 @@ it.runIf(process.platform === 'win32')( expect((await fs.readFile(join(grant, '*.txt'), { glob: true })).content).toContain('grant'); }, ); + +// NI: measures the diagnosed mechanism on the runner itself; the counts are evidence, not a pin. +it.runIf(process.platform === 'win32')( + 'native Windows replacement pairs stay distinct as exact identities', + () => { + let collisions = 0; + let negative = 0; + for (let index = 0; index < 200; index += 1) { + const path = join(base, `probe-${index}`); + mkdirSync(path); + const original = statSync(path, { bigint: true }); + // The original stays alive, so its file ID cannot be reused for the replacement. + renameSync(path, `${path}-old`); + mkdirSync(path); + const replacement = statSync(path, { bigint: true }); + expect([replacement.dev, replacement.ino]).not.toEqual([original.dev, original.ino]); + if (Number(replacement.ino) === Number(original.ino)) collisions += 1; + if (original.ino < 0n || replacement.ino < 0n) negative += 1; + } + console.log( + `NI probe: ${collisions} of 200 replacement pairs collide as Numbers; ${negative} read negative`, + ); + }, +); diff --git a/apps/cli/src/engine/tool-host/fs-policy.ts b/apps/cli/src/engine/tool-host/fs-policy.ts index 0269c0a2..77d3f89e 100644 --- a/apps/cli/src/engine/tool-host/fs-policy.ts +++ b/apps/cli/src/engine/tool-host/fs-policy.ts @@ -15,10 +15,10 @@ export class FsPolicyError extends HostDeniedError { interface DirectoryPin { readonly lexical: string; readonly canonical: string; - readonly dev: number; - readonly ino: number; - readonly lexicalDev: number; - readonly lexicalIno: number; + readonly dev: bigint; + readonly ino: bigint; + readonly lexicalDev: bigint; + readonly lexicalIno: bigint; readonly linked: boolean; } @@ -59,8 +59,9 @@ function pinDirectory(path: string): DirectoryPin { // Match fs/promises.realpath's native canonicalization. The legacy synchronous // walker preserves Windows 8.3 spellings, which are not equal to their long names. const canonical = realpathSync.native(lexical); - const stat = statSync(canonical); - const entry = lstatSync(lexical); + // Exact native identities: a 64-bit file ID (NTFS, overlayfs xino) exceeds a Number's precision. + const stat = statSync(canonical, { bigint: true }); + const entry = lstatSync(lexical, { bigint: true }); if (!stat.isDirectory()) throw new FsPolicyError(); return Object.freeze({ lexical, diff --git a/apps/cli/src/engine/tool-host/native-identity.test.ts b/apps/cli/src/engine/tool-host/native-identity.test.ts new file mode 100644 index 00000000..bebb2537 --- /dev/null +++ b/apps/cli/src/engine/tool-host/native-identity.test.ts @@ -0,0 +1,187 @@ +import { + lstatSync, + mkdirSync, + mkdtempSync, + realpathSync, + renameSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, + type BigIntStats, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { discoverCatalog } from '../../workflows/catalog.js'; +import { + admitInvocationFsGrants, + FsPolicyError, + fsPolicyScopeRoots, + selectInvocationFsPolicy, +} from './fs-policy.js'; + +/** + * POSIX inodes stay far below 2^53, so these cases substitute every identity the code reads: each real + * (dev, ino) becomes 2^60 + k. Distinct substitutes stay distinct as bigints but share one Number (the + * spacing there is 256), so a read without `{ bigint: true }` or a comparison through Number lets a real + * replacement compare equal, as a 64-bit Windows file ID can. + */ +const identities = vi.hoisted(() => { + const state: { fakes: Map; beforeOpen: (() => void) | undefined } = { + fakes: new Map(), + beforeOpen: undefined, + }; + return state; +}); +vi.mock('node:fs', async (original) => { + const native = await original(); + const substitute = (info: unknown, exact: BigIntStats | undefined): void => { + if (exact === undefined || typeof info !== 'object' || info === null || !('ino' in info)) + return; + const key = `${exact.dev}:${exact.ino}`; + const value = identities.fakes.get(key) ?? (1n << 60n) + BigInt(identities.fakes.size); + identities.fakes.set(key, value); + Object.defineProperty(info, 'ino', { + value: typeof info.ino === 'bigint' ? value : Number(value), + }); + }; + return { + ...native, + statSync: (...args: Parameters) => { + const info = native.statSync(...args); + substitute(info, native.statSync(args[0], { bigint: true, throwIfNoEntry: false })); + return info; + }, + lstatSync: (...args: Parameters) => { + const info = native.lstatSync(...args); + substitute(info, native.lstatSync(args[0], { bigint: true, throwIfNoEntry: false })); + return info; + }, + fstatSync: (...args: Parameters) => { + const info = native.fstatSync(...args); + substitute(info, native.fstatSync(args[0], { bigint: true })); + return info; + }, + openSync: (...args: Parameters) => { + const before = identities.beforeOpen; + identities.beforeOpen = undefined; + before?.(); + return native.openSync(...args); + }, + }; +}); + +let base: string; +let workspace: string; +beforeEach(() => { + identities.fakes.clear(); + base = realpathSync.native(mkdtempSync(join(tmpdir(), 'relavium-native-identity-'))); + workspace = join(base, 'workspace'); + mkdirSync(workspace); +}); +afterEach(() => { + identities.beforeOpen = undefined; + rmSync(base, { recursive: true, force: true }); +}); + +/** The fixture must make the original and its replacement collide as Numbers and only as Numbers. */ +function expectNumberCollision(original: bigint, replacement: bigint): void { + expect(replacement).not.toBe(original); + expect(Number(replacement)).toBe(Number(original)); +} + +function refused(check: () => unknown): boolean { + try { + check(); + return false; + } catch (error) { + if (error instanceof FsPolicyError) return true; + throw error; + } +} + +describe('filesystem policy pins compare exact native identities', () => { + it('refuses a replaced selected directory whose identity collides as a Number', () => { + const grant = join(base, 'grant'); + const selected = join(grant, 'selected'); + mkdirSync(selected, { recursive: true }); + const policy = selectInvocationFsPolicy(admitInvocationFsGrants([grant]), workspace, [ + selected, + ]); + const original = statSync(selected, { bigint: true }).ino; + // Keep the original alive so the platform cannot recycle its inode for the replacement. + renameSync(selected, `${selected}-old`); + mkdirSync(selected); + expectNumberCollision(original, statSync(selected, { bigint: true }).ino); + expect( + refused(() => fsPolicyScopeRoots(policy, workspace, 'project')), + 'FS_ROOT_IDENTITY_PRECISION', + ).toBe(true); + }); + + it('refuses an unchanged alias whose target was replaced (the canonical identity)', () => { + const target = join(base, 'target'); + const alias = join(base, 'alias'); + mkdirSync(target); + symlinkSync(target, alias, 'junction'); + const policy = selectInvocationFsPolicy(admitInvocationFsGrants([alias]), workspace, undefined); + const original = statSync(target, { bigint: true }).ino; + const entry = lstatSync(alias, { bigint: true }).ino; + renameSync(target, `${target}-old`); + mkdirSync(target); + expectNumberCollision(original, statSync(target, { bigint: true }).ino); + expect(lstatSync(alias, { bigint: true }).ino).toBe(entry); + expect( + refused(() => fsPolicyScopeRoots(policy, workspace, 'project')), + 'FS_ROOT_IDENTITY_PRECISION', + ).toBe(true); + }); + + it('refuses a replaced alias whose target is unchanged (the lexical identity)', () => { + const target = join(base, 'target'); + const alias = join(base, 'alias'); + mkdirSync(target); + symlinkSync(target, alias, 'junction'); + const policy = selectInvocationFsPolicy(admitInvocationFsGrants([alias]), workspace, undefined); + const canonical = statSync(target, { bigint: true }).ino; + const original = lstatSync(alias, { bigint: true }).ino; + renameSync(alias, `${alias}-old`); + symlinkSync(target, alias, 'junction'); + expectNumberCollision(original, lstatSync(alias, { bigint: true }).ino); + expect(statSync(target, { bigint: true }).ino).toBe(canonical); + expect( + refused(() => fsPolicyScopeRoots(policy, workspace, 'project')), + 'FS_ROOT_IDENTITY_PRECISION', + ).toBe(true); + }); +}); + +describe('the workflow catalog admits an open by exact native identity', () => { + const workflow = (id: string): string => + `schema_version: '1.0'\nworkflow:\n id: ${id}\n nodes:\n - { id: a, type: input }\n - { id: b, type: output }\n edges:\n - { from: a, to: b }\n`; + + it('refuses a catalog file swapped between its admission and its open', () => { + const configDir = join(base, '.relavium'); + const file = join(configDir, 'workflows', 'hello.yaml'); + mkdirSync(join(configDir, 'workflows'), { recursive: true }); + writeFileSync(file, workflow('hello')); + // The swap lands after the catalog's admitting stat and before its open. + identities.beforeOpen = () => { + renameSync(file, `${file}-old`); + writeFileSync(file, workflow('swapped')); + }; + const entries = discoverCatalog({ projectConfigDir: configDir, cwd: base, kind: 'workflows' }); + expect(identities.beforeOpen).toBeUndefined(); + expectNumberCollision( + statSync(`${file}-old`, { bigint: true }).ino, + statSync(file, { bigint: true }).ino, + ); + expect( + entries.map((entry) => ({ valid: entry.valid, error: entry.error })), + 'CATALOG_IDENTITY_PRECISION', + ).toEqual([{ valid: false, error: 'not the admitted regular file' }]); + }); +}); diff --git a/apps/cli/src/engine/tool-host/process-close.test.ts b/apps/cli/src/engine/tool-host/process-close.test.ts index 140d7510..c43ddb2c 100644 --- a/apps/cli/src/engine/tool-host/process-close.test.ts +++ b/apps/cli/src/engine/tool-host/process-close.test.ts @@ -127,39 +127,99 @@ describe('native child resource lifetime after an error (ADR-0103)', () => { expect(closed).toBe(true); }); - it('withholds the failed-spawn result until the owner reservation joins close', async () => { - // The child's own `close` event cannot pin this: the capability's handler only runs after it, so a - // rejection that skipped `reservation.whenClosed` would still see a closed child (and, on POSIX with no - // helper, an already-retired owner). Gate the reservation's join itself and observe the publication. - const real = nativeProcessesForInvocation(createCliInvocation()); - if (real === undefined) throw new Error('explicit invocation missing'); - const gate = deferred(); - const gated: NativeProcessController = { - ...real, - reserve: async (kind) => ({ ...(await real.reserve(kind)), whenClosed: gate.promise }), - }; - let closed = false; - vi.mocked(spawn).mockImplementationOnce((command, args, options) => { - const child = native.spawn(command, args, options); - child.once('close', () => { - closed = true; - }); - return child; - }); - let settled = false; - const call = createNodeProcessCapability({ workspaceDir: workspace, nativeProcesses: gated }) - .spawn(join(workspace, 'absent-executable'), [], {}, {}) - .catch((cause: unknown) => cause) - .finally(() => { - settled = true; + // The child's own `close` event cannot pin the owner join: the capability's handler only runs after it, so + // a publication that skipped `reservation.whenClosed` would still see a closed child (and, on POSIX with + // no helper, an already-retired owner). Each publication branch of `runChild` is therefore tested by + // gating the reservation's join itself and observing when the result reaches the caller. + const longLived = ['-e', 'setInterval(() => {}, 30000)']; + const branches = [ + { + name: 'failed-spawn', + command: () => join(workspace, 'absent-executable'), + args: [] as string[], + expected: { message: 'the command failed to run' }, + }, + { + name: 'natural-exit', + command: () => process.execPath, + args: ['-e', ''], + expected: { exitCode: 0 }, + }, + { + name: 'abort', + command: () => process.execPath, + args: longLived, + abort: true, + expected: { message: 'the command was aborted' }, + }, + { + name: 'timeout', + command: () => process.execPath, + args: longLived, + timeoutMs: 300, + expected: { message: 'the command timed out after 300ms' }, + }, + ] as const; + + it.each(branches)( + 'withholds the $name result until the owner reservation joins close', + async (branch) => { + const real = nativeProcessesForInvocation(createCliInvocation()); + if (real === undefined) throw new Error('explicit invocation missing'); + const gate = deferred(); + const gated: NativeProcessController = { + ...real, + reserve: async (kind) => ({ ...(await real.reserve(kind)), whenClosed: gate.promise }), + }; + const spawned = deferred(); + let closed = false; + let started: ChildProcess | undefined; + vi.mocked(spawn).mockImplementationOnce((command, args, options) => { + const child = native.spawn(command, args, options); + started = child; + child.once('close', () => { + closed = true; + }); + spawned.resolve(); + return child; }); - await vi.waitFor(() => expect(closed).toBe(true)); - // Microtasks queued by the close dispatch have run; give any ungated publication one more turn. - await new Promise((resolve) => setImmediate(resolve)); - expect(settled, 'PROCESS_OWNER_CLOSE_JOIN').toBe(false); - gate.resolve(); - const error = await call; - expect(error).toBeInstanceOf(ProcessCapabilityError); - expect(error).toMatchObject({ message: 'the command failed to run' }); - }); + const controller = new AbortController(); + let settled = false; + const call = createNodeProcessCapability({ workspaceDir: workspace, nativeProcesses: gated }) + .spawn( + branch.command(), + [...branch.args], + {}, + 'timeoutMs' in branch ? { timeoutMs: branch.timeoutMs } : {}, + controller.signal, + ) + .then( + (value: unknown) => value, + (cause: unknown) => cause, + ) + .finally(() => { + settled = true; + }); + try { + if ('abort' in branch) { + await spawned.promise; + controller.abort(); + } + await vi.waitFor(() => expect(closed).toBe(true), { timeout: 3_000 }); + // Hold the gate for a fixed margin, not one macrotask: a publication that is merely late (a timer + // or a retried step) must be caught too, and a single turn let a 20 ms-late one through. + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(settled, `PROCESS_OWNER_CLOSE_JOIN_${branch.name}`).toBe(false); + gate.resolve(); + expect(await call).toMatchObject(branch.expected); + } finally { + // A failing case must not leave the long-lived child (or a pending call) behind. + gate.resolve(); + controller.abort(); + if (started !== undefined && started.exitCode === null && started.signalCode === null) + started.kill('SIGKILL'); + await call; + } + }, + ); }); diff --git a/apps/cli/src/harness/mcp-orphan.e2e.test.ts b/apps/cli/src/harness/mcp-orphan.e2e.test.ts index 2e8d23b0..daf9076c 100644 --- a/apps/cli/src/harness/mcp-orphan.e2e.test.ts +++ b/apps/cli/src/harness/mcp-orphan.e2e.test.ts @@ -105,8 +105,10 @@ async function until(predicate: () => boolean, budgetMs = 10_000): Promise { diff --git a/apps/cli/src/process/supported-leaf-exports.test.ts b/apps/cli/src/process/supported-leaf-exports.test.ts index 57da3b71..c7518d32 100644 --- a/apps/cli/src/process/supported-leaf-exports.test.ts +++ b/apps/cli/src/process/supported-leaf-exports.test.ts @@ -70,10 +70,15 @@ describe('supported compiled dependency leaves', () => { } }); - it('loads the actual supported compiled leaf with only terminal-safe reachable', () => { + it('loads the actual supported compiled leaf with only terminal-safe and its own probe reachable', () => { const require = createRequire(import.meta.url); + const leaf = pathToFileURL(require.resolve('@relavium/llm/scrub-secrets')).href; const allowed = [ - pathToFileURL(require.resolve('@relavium/llm/scrub-secrets')).href, + leaf, + // The leaf's dependency-free, package-internal split-credential probe and fixed-point + // helper (no package exports). + new URL('./split-credential-probe.js', leaf).href, + new URL('./fixed-point.js', leaf).href, pathToFileURL(require.resolve('@relavium/shared/terminal-safe')).href, ]; // An isolated ESM loader refuses every other module, including schema/provider barrels. diff --git a/apps/cli/src/render/sanitize.test.ts b/apps/cli/src/render/sanitize.test.ts index 00a18161..6a96c9b5 100644 --- a/apps/cli/src/render/sanitize.test.ts +++ b/apps/cli/src/render/sanitize.test.ts @@ -259,6 +259,9 @@ describe('sanitizeUntrusted — the failure paths carry arbitrary text (#W15-8)' for (const separator of ['\n', '\t']) { expect(sanitize(`xhttps://us${separator}er:opaque-value@host/x`)).toBe('[REDACTED]'); } + // Glued scheme + split credential + a raw `@` in the credential: no partial mask may remain. + expect(sanitize('xhttps://user:p@ss\tword@host/x')).toBe('[REDACTED]'); + expect(sanitize('xhttps://u@e.com:pw\nxyz@host/x')).toBe('[REDACTED]'); } }); }); diff --git a/apps/cli/src/render/sanitize.ts b/apps/cli/src/render/sanitize.ts index 80fd1f66..9f409e96 100644 --- a/apps/cli/src/render/sanitize.ts +++ b/apps/cli/src/render/sanitize.ts @@ -64,8 +64,8 @@ export function sanitizeUntrustedInline(text: string): string { * Coarse on purpose: a boolean cannot locate the * span, and this function's whole posture (see {@link sanitizeUntrusted}) is that the cost of a false * positive is a `[REDACTED]` in a diagnostic, while the cost of a miss is a printed key. This broader CLI - * probe also checks split literal query names. The shared `scrubSecrets` separately protects its finite - * non-query credential shapes before query masking can destroy their identifying context; it does not + * probe also checks split literal query names. The shared `scrubSecrets` masks its finite shapes and + * its literal query values in one merged pass, so no mask erases another shape's context; it does not * join query names or ordinary query prose. */ function redactIncludingAcrossSeparators(normalized: string): string { diff --git a/apps/cli/src/render/tui/final-summary.test.ts b/apps/cli/src/render/tui/final-summary.test.ts index 7476649d..8bda7e32 100644 --- a/apps/cli/src/render/tui/final-summary.test.ts +++ b/apps/cli/src/render/tui/final-summary.test.ts @@ -221,4 +221,22 @@ describe('renderFinalSummary', () => { expect(lines[1]).toContain('✓ deploy completed'); expect(lines).not.toContain(' ✓ deploy completed'); }); + + it('masks a secret shape in the failure message before it reaches the terminal', () => { + // A PR #96 review: the summary row collapsed separators but did not mask, unlike every other + // untrusted diagnostic sink. + const key = `sk-${'Qz7'.repeat(8)}`; + const state: RunViewState = { + ...initialRunViewState(), + summary: { + outcome: 'failed', + errorMessage: `provider rejected Authorization: Bearer ${key}`, + }, + }; + + const output = renderFinalSummary(state); + expect(output).not.toContain(key); + expect(output).toContain('provider rejected'); + expect(output).toContain('[REDACTED]'); + }); }); diff --git a/apps/cli/src/render/tui/final-summary.ts b/apps/cli/src/render/tui/final-summary.ts index 535bf44f..7d7e820c 100644 --- a/apps/cli/src/render/tui/final-summary.ts +++ b/apps/cli/src/render/tui/final-summary.ts @@ -7,7 +7,7 @@ import { } from './format.js'; import { nodeSuffix } from './projection.js'; import type { RunViewState } from './run-view-model.js'; -import { sanitizeInline } from '../sanitize.js'; +import { sanitizeInline, sanitizeUntrustedInline } from '../sanitize.js'; /** * Render the **persistent** final summary the `ink` renderer writes after it unmounts (workstream **2.E**). @@ -57,7 +57,8 @@ export function renderFinalSummary(state: RunViewState): string { if (summary?.errorMessage !== undefined) { // A summary error occupies ONE structural row. Collapse line/column separators as well as terminal // controls, so provider/model text cannot forge a believable node-status row beneath the headline. - lines.push(` ${sanitizeInline(summary.errorMessage)}`); + // It is untrusted text (a provider or tool diagnostic), so its secret shapes are masked first. + lines.push(` ${sanitizeUntrustedInline(summary.errorMessage)}`); } for (const id of state.nodeOrder) { diff --git a/apps/cli/src/workflows/catalog.test.ts b/apps/cli/src/workflows/catalog.test.ts index 51c8c1d0..37862863 100644 --- a/apps/cli/src/workflows/catalog.test.ts +++ b/apps/cli/src/workflows/catalog.test.ts @@ -11,6 +11,7 @@ import { import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { MAX_SOURCE_CHARS } from '@relavium/core'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { isCliError } from '../process/errors.js'; @@ -195,6 +196,24 @@ describe('discoverCatalog', () => { expect(entries[0]?.valid).toBe(false); }); + it('refuses an oversized file from its admitting stat before reading a byte', () => { + write('workflows', 'big.yaml', 'a'.repeat(MAX_SOURCE_CHARS * 4 + 1)); + const entries = discoverCatalog({ projectConfigDir: configDir, cwd: proj, kind: 'workflows' }); + expect(entries.map((entry) => ({ valid: entry.valid, error: entry.error }))).toEqual([ + { valid: false, error: 'exceeds the size limit' }, + ]); + expect(vi.mocked(readSync)).not.toHaveBeenCalled(); + }); + + it('admits a dense multibyte file over the character cap in bytes but within it in characters', () => { + // Three bytes per character: half the cap in characters is one and a half times it in bytes. + write('workflows', 'dense.yaml', `# ${'€'.repeat(MAX_SOURCE_CHARS / 2)}\n${VALID_WORKFLOW}`); + const entries = discoverCatalog({ projectConfigDir: configDir, cwd: proj, kind: 'workflows' }); + expect(entries.map((entry) => ({ slug: entry.slug, valid: entry.valid }))).toEqual([ + { slug: 'hello', valid: true }, + ]); + }); + it('flags a symlinked .yaml that resolves to a non-regular file (no readFileSync hang)', () => { write('workflows', 'hello.relavium.yaml', VALID_WORKFLOW); // ensures workflows/ exists // A `.yaml` symlinked to a DIRECTORY (statSync follows it → not a regular file). The same guard rejects a diff --git a/apps/cli/src/workflows/catalog.ts b/apps/cli/src/workflows/catalog.ts index fa57e325..1b573a7e 100644 --- a/apps/cli/src/workflows/catalog.ts +++ b/apps/cli/src/workflows/catalog.ts @@ -126,7 +126,7 @@ function readEntry(path: string, cwd: string, kind: CatalogKind, root: string): const canonical = realpathSync.native(path); if (!isWithinFsRoot(root, canonical)) return invalidEntry(path, rel, 'outside the project catalog'); - const stats = statSync(canonical); + const stats = statSync(canonical, { bigint: true }); // statSync FOLLOWS a symlink, so a `.yaml` symlinked to a directory, FIFO, or character device (e.g. // /dev/zero) resolves here — and readFileSync on a FIFO/char-device BLOCKS INDEFINITELY (a local DoS via a // crafted symlink in the catalog dir). Reject anything that is not a regular file BEFORE reading; the size @@ -138,7 +138,7 @@ function readEntry(path: string, cwd: string, kind: CatalogKind, root: string): // bytes/char, so `chars * 4` is a byte ceiling that never false-rejects a within-limit file (a dense // multibyte file just under the char cap is not wrongly hidden) while still bailing before we slurp a // genuinely huge file into memory. The parser then re-applies the exact char cap. - if (stats.size > MAX_SOURCE_CHARS * 4) { + if (stats.size > BigInt(MAX_SOURCE_CHARS * 4)) { return invalidEntry(path, rel, 'exceeds the size limit'); } // Contained aliases are allowed; open their canonical target without following a newly planted final link. @@ -147,7 +147,9 @@ function readEntry(path: string, cwd: string, kind: CatalogKind, root: string): constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, ); try { - const opened = fstatSync(fd); + // Exact bigint identities: Windows has no O_NOFOLLOW, so this is its only swap guard, and its + // 64-bit file IDs exceed a Number's precision. + const opened = fstatSync(fd, { bigint: true }); if (!opened.isFile() || opened.dev !== stats.dev || opened.ino !== stats.ino) { return invalidEntry(path, rel, 'not the admitted regular file'); } diff --git a/apps/cli/tsconfig.config.json b/apps/cli/tsconfig.config.json new file mode 100644 index 00000000..dc1c1fba --- /dev/null +++ b/apps/cli/tsconfig.config.json @@ -0,0 +1,10 @@ +{ + // The build configuration is not under `src/`, so `tsconfig.json` does not check it, yet `tsup.config.ts` + // decides whether a build carries source maps (a candidate build) or is map-free. Typecheck it too. + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "noEmit": true, + "types": ["node"] + }, + "include": ["tsup.config.ts"] +} diff --git a/apps/cli/tsup.config.ts b/apps/cli/tsup.config.ts index 92e89c71..0d01471a 100644 --- a/apps/cli/tsup.config.ts +++ b/apps/cli/tsup.config.ts @@ -17,10 +17,18 @@ const { version } = JSON.parse( * are bundler-hostile). `tools/bundle-closure/check.mjs` guards that this `external` list equals the declared * runtime `dependencies`, so the two can never drift. * - * `sourcemap: false` + `minify: true`: the published `dist/` must ship no source tree or maps. This is the npm + * `minify: true` and no maps in `dist/`: the published package ships no source tree or maps. This is the npm * distribution boundary, not a claim that the public repository's sources are secret (ADR-0111). * The entry-source shebang makes `dist/index.js` directly executable as the `relavium` bin. + * + * Ordinary builds are map-free. A candidate build — this build with `RELAVIUM_CLI_CANDIDATE_STAGING` defined, which + * the prepack lifecycle (`tools/cli-package/prepack.mjs`, Wave 3 Step 9a.1) runs only after it validates that + * directory as private staging — emits external maps without embedded sources. The prepack finalizer + * (`tools/cli-package/finalize.mjs`) then checks that they chain to original TypeScript, moves them and the + * metafile out of the package and removes their link trailers before anything is hashed or packed; the ordinary + * closure guard refuses any map that a build outside prepack leaves in `dist`. */ +const CANDIDATE = process.env['RELAVIUM_CLI_CANDIDATE_STAGING'] !== undefined; const THIRD_PARTY_EXTERNAL = [ '@anthropic-ai/sdk', '@clack/prompts', @@ -51,7 +59,11 @@ export default defineConfig({ target: 'node22', platform: 'node', clean: true, - sourcemap: false, + sourcemap: CANDIDATE, + esbuildOptions(options) { + // Original sources never enter a candidate map; the finalizer refuses any that does. + if (CANDIDATE) options.sourcesContent = false; + }, minify: true, splitting: true, // Emit esbuild's metafile (`dist/metafile-esm.json`) — the authoritative record of every externalized import, diff --git a/docs/decisions/0051-cli-distribution-thin-bundle-private-engine.md b/docs/decisions/0051-cli-distribution-thin-bundle-private-engine.md index 1df8188f..ae0afef3 100644 --- a/docs/decisions/0051-cli-distribution-thin-bundle-private-engine.md +++ b/docs/decisions/0051-cli-distribution-thin-bundle-private-engine.md @@ -151,3 +151,5 @@ intended precedent for the desktop (`.dmg`) and VS Code (`.vsix`) surfaces. ## Wave 3 decision acceptance — 2026-10-16 > Amended 2026-10-16: [ADR-0111](0111-cli-meta-operations-use-a-lazy-packaged-runtime.md) is accepted by the maintainer. Only the single-output constraint is partially replaced by finite engine-inlined runtime chunks with complete closure and packed-platform checks. Licensing rather than source secrecy, private engine packages and no public plaintext source maps remain accepted. Exact release-map encrypted custody is a distribution policy, not secrecy for this public source repository. The approved [execution plan](../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) owns implementation and review gates; this note does not claim implementation landing. + +> Amended 2026-10-22: Wave 3 Step 9a.1 implements the candidate build under [ADR-0111](0111-cli-meta-operations-use-a-lazy-packaged-runtime.md). A candidate build — the CLI build with `RELAVIUM_CLI_CANDIDATE_STAGING` defined, which the prepack lifecycle runs only after it validates that directory as private staging — emits external maps without embedded sources, and its finalizer stages them outside the package before anything is hashed or packed, so the published artifact still carries no map or map link; the finalizer, the closure guard and the archive inspector each refuse one. "The publish build sets `sourcemap: false`" now describes ordinary builds. The [packaging contract](../reference/cli/packaging.md#candidate-build-and-finalizer-implementation-boundary--step-9a1) records the implemented boundary; this note claims no symbolication, custody or release. diff --git a/docs/decisions/0105-filesystem-roots-are-invocation-grants-not-project-authority.md b/docs/decisions/0105-filesystem-roots-are-invocation-grants-not-project-authority.md index 4e6d623c..390d70de 100644 --- a/docs/decisions/0105-filesystem-roots-are-invocation-grants-not-project-authority.md +++ b/docs/decisions/0105-filesystem-roots-are-invocation-grants-not-project-authority.md @@ -323,3 +323,17 @@ future feature, deferral or claim of merge-commit execution. ## Wave 3 decision acceptance — 2026-10-16 > Amended 2026-10-16: [ADR-0111](0111-cli-meta-operations-use-a-lazy-packaged-runtime.md) is accepted by the maintainer. Meta operations deliberately skip directory existence/link/permission grant admission while retaining pure grammar/count/size checks; no filesystem authority is minted. Execution preserves filesystem admission before the one catalog seed and later model/cost admission. ADR-0112 separately narrows built-in chat HTTP authority without changing filesystem grants. The approved [execution plan](../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) owns implementation and review gates; this note does not claim implementation landing. + +## Exact native identity correction — 2026-10-25 + +> Amended 2026-10-25: The root pins this decision requires compared device and inode as JavaScript numbers. On +> Windows `ino` is a 64-bit NTFS file ID, and Linux overlayfs's `xino` feature sets high inode +> bits; neither fits a Number's 53 bits, so a replaced root could compare equal (Step 9a.2a's +> push CI observed it once on the Windows runner). The policy pins and the workflow catalog's +> open check now read and compare exact bigints; the catalog comparison is Windows' only swap +> guard, since `O_NOFOLLOW` is undefined there. The decision is unchanged: this restores the +> identity the pins already claimed, and permanent causal controls substitute identities that +> differ only beyond a Number's precision. ReFS's 64-bit file IDs are not guaranteed unique; +> that limit is recorded in [deferred-tasks.md](../roadmap/deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24). +> The [NI checkpoint](../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25) +> owns review and acceptance; this note claims neither. diff --git a/docs/decisions/0106-media-publication-requires-bounded-verified-bytes.md b/docs/decisions/0106-media-publication-requires-bounded-verified-bytes.md index c3d44b2f..3fcbcb34 100644 --- a/docs/decisions/0106-media-publication-requires-bounded-verified-bytes.md +++ b/docs/decisions/0106-media-publication-requires-bounded-verified-bytes.md @@ -547,3 +547,21 @@ as PR #94 (`ab08209e`). The [merge checkpoint](../roadmap/phases/phase-2.5.5-har joins final reviewed source `4cdfe011`, review scopes, execution evidence and preserved limits. Earlier pending/unmerged notes are historical; this landing adds no decision, future feature, deferral or claim of merge-commit execution. + +### Exact native identity correction — 2026-10-25 + +> Amended 2026-10-25: The 2026-10-13 correction made the persisted claim token exact but left every live comparison +> on numbers: the root pin, file snapshots and the owned-file check that guards deletion, the +> read's descriptor stats, the publish handle, and the sweep's claim-directory, child, marker +> and moved-blob checks; `nativeFile` read a bigint but compared it through `Number()`. Each now +> compares exact bigint device and inode, each read from a single stat; the file and directory +> checks share one projection, in which `nlink`, `size` and `mtimeMs` keep their number values +> (`mtimeMs` by Node's own formula). Node fills bigint stats from a signed array, so a 64-bit file +> ID with its top bit set read negative: `nativeFile`'s Number cross-check refused it, and the +> claim token would have too, retaining such a temporary forever. The token now encodes the +> unsigned value; +> every other value keeps a byte-identical token, so persisted claims stay compatible. The +> [physical storage contract](../reference/shared-core/built-in-tools.md#physical-media-storage-and-limits) +> owns the rules. Permanent causal controls and the +> [NI checkpoint](../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25) +> own the proof and its gates. The cooperative root/path floor is unchanged. diff --git a/docs/reference/cli/README.md b/docs/reference/cli/README.md index bb56d98f..f8c11ca3 100644 --- a/docs/reference/cli/README.md +++ b/docs/reference/cli/README.md @@ -13,5 +13,5 @@ Part of [reference/](../README.md). | [commands.md](commands.md) | The full `relavium` command surface (run, chat, chat-resume/list/export, agent run, list, create, import, export, logs, gate, init, provider) plus interactive TUI vs CI JSON mode. | | [home.md](home.md) | The bare-invocation interactive **Home** — the TTY gate, the read-only management strip over `history.db`, the Home↔chat mode machine + keymap, bracketed paste, the min-terminal degrade, and the signal/exit-code lifecycle (2.5.B / ADR-0054). | | [chat-session.md](chat-session.md) | The `relavium chat` agent-session REPL — entry, agent/model selection, the multi-turn loop, streaming, tool availability, the `--json` session-event stream, and exit code 4. | -| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; Step 9 source maps, encrypted custody and release retry remain staged. | +| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development` and Step 9a.2a's private manifest and symbolication are accepted on `development` and Step 9a.2b's product map controls and `package_test` receipt are accepted on `development`; Step 9b.1's private container, isolated GnuPG runner and encrypt-once library are accepted on `development` (synthetic recipients only); Step 9b.2's ciphertext reuse, release binding receipt v1, absent production loader and scope separation are accepted on `development`; the remaining Step 9 custody and release gates remain staged. | | [agent-run-fixture.md](agent-run-fixture.md) | The `relavium agent run --fixture` cassette format — a `StreamChunk[][]` JSON recording of an agent run's LLM stream, for deterministic offline replay. | diff --git a/docs/reference/cli/commands.md b/docs/reference/cli/commands.md index 7e9a1acd..eb289b54 100644 --- a/docs/reference/cli/commands.md +++ b/docs/reference/cli/commands.md @@ -1,6 +1,6 @@ # CLI Command Reference (`relavium`) -> Last updated: 2026-10-18 +> Last updated: 2026-10-20 - **Status**: Reference (partial — surface defined, exact flags to be finalized as the CLI is built) - **Surface**: CLI (`relavium`) @@ -232,9 +232,9 @@ Both are **surface-agnostic** git-native YAML operations — they read/write `.r `create` and `import` emit these notices only after their YAML file is written. A refused write emits no authored-content notice or success output. -The review visits at most 4,096 nodes through depth 64, checks strings of at most 16,384 UTF-16 code units and at most 262,144 checked code units in total, including property names. It reports only a count of strings matching the finite detector: prefixed OpenAI/Anthropic or Google keys, HTTP(S)/WS(S)/FTP(S) URL userinfo, exact `api-key`/`x-api-key`/`x-goog-api-key` headers, Bearer/Basic credentials and literal secret query names. It does not classify every credential format or distinguish arbitrary opaque credential fragments from following diagnostic prose. Over-budget strings or structure, repeated objects and unsupported object shapes produce a separate fixed incomplete-review notice. Object-key enumeration relies on the existing parser source ceiling; the node limit is not an independent bound on that enumeration. Neither notice includes values or arbitrary property names. +The review visits at most 4,096 nodes through depth 64, checks strings of at most 16,384 UTF-16 code units and at most 262,144 checked code units in total, including property names. It reports only a count of strings matching the finite detector, the [diagnostic secret backstop](../shared-core/diagnostic-secret-backstop.md): prefixed OpenAI/Anthropic or Google keys, URL userinfo (any userinfo for HTTP(S)/WS(S)/FTP(S), colon-bearing userinfo for the listed connection-string schemes), exact `api-key`/`x-api-key`/`x-goog-api-key` headers, Bearer/Basic credentials and literal secret query names. It does not classify every credential format or distinguish arbitrary opaque credential fragments from following diagnostic prose. Over-budget strings or structure, repeated objects and unsupported object shapes produce a separate fixed incomplete-review notice. Object-key enumeration relies on the existing parser source ceiling; the node limit is not an independent bound on that enumeration. Neither notice includes values or arbitrary property names. -For this detector probe only, a recognised bare template reference without a path or filter is projected to an already masked span, provided its raw reference contains no recognised credential shape. This keeps `Bearer {{secrets.token}}` and equivalent query placeholders quiet. Adjacent literals, credential-shaped identifiers, paths, filter arguments and unknown or malformed references retain their evidence. This projection neither resolves references nor changes authored content. +For this detector probe only, a recognised bare template reference without a path or filter is projected to an already masked span, provided its raw reference contains no recognised credential shape. This keeps `Bearer {{secrets.token}}`, a header such as `x-api-key: Bearer "{{secrets.token}}"` (quoted or not), equivalent query placeholders and a connection string whose password is a placeholder (`postgres://app:{{secrets.db_password}}@db`) quiet. An HTTP(S)/WS(S)/FTP(S) URL with a literal username beside a placeholder password still counts, because those schemes mask any userinfo. Adjacent literals, credential-shaped identifiers, paths, filter arguments and unknown or malformed references retain their evidence. This projection neither resolves references nor changes authored content. ### `relavium logs ` diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 3df41142..b991138c 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -1,6 +1,6 @@ # CLI Packaging Contract -- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; Step 9 map/custody and release gates remain staged +- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are independently accepted on `development` (2026-10-24); Step 9a.2a's strict decoder, private manifest and symbolication are independently accepted on `development` (2026-10-24); Step 9a.2b's product map controls, post-receipt rebuild refusals and `package_test` receipt are independently accepted on `development` (2026-10-24); Step 9b.1's private container, isolated GnuPG runner and encrypt-once library are independently accepted on `development` (2026-10-25; synthetic recipients only); Step 9b.2's ciphertext reuse, release binding receipt v1, absent production loader and scope separation are independently accepted on `development` (2026-10-26; synthetic recipients only); the remaining Step 9 custody and release gates remain staged - **Date**: 2026-10-15 - **Related**: [ADR-0111](../../decisions/0111-cli-meta-operations-use-a-lazy-packaged-runtime.md), [commands.md](commands.md), [release runbook](../../runbooks/release-a-surface.md) @@ -149,8 +149,8 @@ removal precedes success receipt/summary publication. Worker or cleanup failure prevents publication, and secondary cleanup/diagnostic failure preserves the primary outcome. Private worker identity stays outside public receipts. -Required workflows share one public tarball and a `compiled_package_test` receipt -with current `.nvmrc` Linux/macOS/Windows and exact Node 22.0.0 Linux. Each repeats +Required workflows share one public tarball and a package-test receipt (mode `package_test` +since Step 9a.2b, `compiled_package_test` before it) with current `.nvmrc` Linux/macOS/Windows and exact Node 22.0.0 Linux. Each repeats untouched-entry bootstrap controls and actual workflow, gate, budget, chat/resume, SQLite, native-load, MCP-version and media-cleanup behavior. Their aggregate checks all four actual success records and immutable artifact/source/origin identities. @@ -159,9 +159,11 @@ The [tooling guide](../../../tools/cli-package/README.md) describes invocation a bounded retry evidence. These runners are implemented; required platform execution and independent acceptance are recorded separately in the Wave 3 register. -This intermediate receipt does not claim the full map-bearing package-test mode -below. Original-TypeScript symbolication, private staging and release custody -remain Step 9. Native fatal-owner integration is covered by [Step 8 acceptance](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17). +Since Step 9a.2b the receipt records `package_test`: before writing it the producer verified its +maps and symbolicated actual packed frames (Steps 9a.1 and 9a.2a), and the `ci` workflow requires the +product map controls; an artifact is backed by them only where a successful `ci` run exists at exactly +its commit (below). Release custody remains Steps 9b–9d. Native fatal-owner integration is covered by +[Step 8 acceptance](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17). No intermediate local result closes Step 9 or certifies a release. ### Candidate finalization — Step 9 @@ -229,6 +231,274 @@ candidate for that version refuses instead of silently superseding it. The finalized JS/map receipt is sealed locally before encryption; the binding receipt is sealed after ciphertext exists. These are distinct, named artifacts. +### Candidate build and finalizer implementation boundary — Step 9a.1 + +Step 9a.1 implements items 1–3 above and the producer half of item 4. This section +describes the implemented mechanism; independent acceptance is recorded in the +[Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments). + +- **Owned candidate tree.** `pnpm package:cli:pack` never builds in the developer's + workspace. It records the original checkout's commit, full porcelain status and the + tracked files Git is told not to check (skip-worktree or assume-unchanged, which make + the identity `dirty` whatever their bytes), copies every file Git would ship (tracked + plus untracked-but-not-ignored, minus deletions, by the repository's ignore rules and + the developer's global excludes file, as the developer's own Git finds it (also through + command-scope `GIT_CONFIG_*` configuration) and passed explicitly; that Git reads no home, + global or system configuration or attributes and ignores replace refs) into a fresh + owner-only tree, and refuses if any of the three changed during the copy. A clean status is + then confirmed byte for byte (Step 9a.2a): each copied file's Git blob id — the hash of its raw + bytes, no filter — must equal the commit tree's entry, because line-ending conversion + (`core.autocrlf`, the Git for Windows default) and Git's stat cache can report converted bytes as + unchanged; a copy that differs makes the identity `dirty`, so the build is symbolicated against + the bytes it was built from and a trusted origin refuses it. A link, as a listed file or as any parent directory of one + below the checkout, refuses the copy, so a tracked directory replaced by an ignored link + cannot bring in bytes from outside the checkout; so does an untracked nested repository, + which Git lists only as its directory. Every build and install pnpm call runs in the + private home's isolated environment (only the store discovery below also sees the + caller's store-location variables), and a corepack-managed pnpm runs from the developer's + own corepack cache (`COREPACK_HOME`, as corepack computes it), never from the temporary + directory, where another local user could plant a module on its lookup path. + Ignored output, caches, dependencies and private notes never enter it. The locked + closure is installed there offline, frozen and with `--ignore-scripts` from the + existing content-addressed store, which pnpm discovers in the repository with only + `HOME`, `USERPROFILE`, `XDG_DATA_HOME`, `XDG_CONFIG_HOME`, `LOCALAPPDATA`, `APPDATA`, + `PNPM_HOME`, `npm_config_store_dir` and `NPM_CONFIG_STORE_DIR` of the caller's environment. Every `@relavium/*` + link of the CLI and the five inlined engine packages must resolve to that package inside + the tree, no other `@relavium/*` link is admitted, and the root `node_modules` must + resolve inside the tree too. +- **Fresh upstream maps.** The five engine packages compile in order with the installed + compiler's `--sourceMap true --inlineSources false --incremental false` overrides after + their `dist` and incremental files are removed. No Turbo or other build cache is + invoked, and a Turbo directory, `*.tsbuildinfo` or `node_modules/.cache` left in the + tree refuses before the seal. Ordinary builds keep their explicit `sourceMap: false`. +- **Sole prepack build.** `prepack` is the tooling orchestrator + `tools/cli-package/prepack.mjs`, which runs the installed tsup once. Without a staging + directory it then runs the ordinary closure guard, as before. The CLI configuration + emits external maps with `sourcesContent` disabled whenever + `RELAVIUM_CLI_CANDIDATE_STAGING` is defined; prepack first refuses unless it names an + absolute, canonical, empty directory outside the candidate (both compared by the real + paths and letter case the filesystem stores), owner-only where POSIX modes are + observable, and then runs the finalizer in place of the ordinary guard. That guard + refuses any map a build outside prepack leaves in `dist`. +- **Finalizer.** Before any write it accounts for every emitted file (each runtime JS + output paired with exactly one map, plus the metafile, nothing else). It validates every + consumed upstream tsc map: exact terminal map link, version 3, matching `file`, empty + `sourceRoot`, no embedded sources, and only that package's existing original + TypeScript; its mappings may be empty only when the module's types all erase, which tsc + emits as exactly `export {};`. It validates every final map: + version 3, no embedded sources or unreviewed fields, and sources resolving to existing + `apps/cli/src` or engine `src` TypeScript that equal exactly the original sources of + the inputs contributing bytes to that output, so a bundler fallback to compiled JS + refuses. A code-splitting facade with no contributing input must have an empty map. + It removes only tsup's exact terminal `//# sourceMappingURL=.map` trailer and + requires the remaining bytes to equal the size esbuild recorded. Then it stages the + rewritten maps (repository-relative sources; mappings and names unchanged) and the + metafile owner-only outside the package, re-runs the finite closure and the + migration-copy check on the finalized files, and only then hashes into a private record. + A refusal names a fixed code and at most a fixed label or a repository-relative path that + has passed validation: a metafile string that escapes the repository or is not a plain + path is left out, and no map or source content is ever printed. +- **Seal.** The seal takes the candidate root, the staging directory and the original + checkout's identity explicitly and never resolves a repository from its own location. + Public JS must be exactly the staged metafile's runtime outputs, byte-identical to the + finalizer's record; the actual tarball is inspected as in Step 7b. The receipt remained + `compiled_package_test` until Step 9a.2b flipped it to `package_test`. +- **Ordinary map-free control.** The ordinary closure guard refuses any map, map link or + unreviewed file in `dist`, and the five build configurations and the CLI configuration + are pinned map-free outside a named candidate. + +Private maps, the metafile, the hash record and the candidate tree are removed when the +producer succeeds, fails or is interrupted (SIGINT, SIGTERM, SIGHUP); an interrupt also stops +the build's child processes and empties the artifact directory, which was empty at the start, +before the producer exits with the signal's code — also when it arrives after the build's last +child exited, during the seal or the final removal. A failed production prints only prepack's own +fixed `✗ CLI …` lines and one fixed code, never a build child's captured output, which can quote a +malformed map. A step of an interrupt's cleanup that fails leaves one fixed line, never a path, +and the producer still exits with the signal's code. An interrupted consumer likewise stops its +child processes, including the bootstrap witness's, and removes its installation, the witness's +fixtures and the installed smoke's working directory; the native fixture targets the installed CLI +starts in their own process groups end on their own once the smoke worker that owns their case is +gone. On POSIX the local `pnpm smoke:cli-package` forwards an interrupt to the producer or +consumer it started rather than ending first (on Windows they share the console and receive +Ctrl-C themselves), starts no later stage once one arrived, and exits with the signal's code after +removing its root. On POSIX a stage past its deadline is interrupted (SIGTERM) first, and its whole +process group is killed when a grace period ends or as soon as the stage exits, whichever comes first, +so it stops its own build children before the root goes; on Windows its whole tree is killed at the deadline, and its private directories go +with the check's root. These Windows interrupt and deadline paths are not exercised by any CI job: +the tooling tests run on Linux only. SIGHUP aborts and cleans up like the other two signals even +under `nohup`, which these listeners replace; detach a long local run with `setsid` or a terminal +multiplexer instead. A producer killed outright, a crash or a lost host can leave the +owner-only root in the temporary directory, and no secure erasure is claimed. The local consumer +installs the CLI under the temporary directory, so on a host whose temporary directory other +local users can write to (a shared `/tmp`), a module another user plants in its `node_modules` +could satisfy an optional `require` of the installed CLI; CI runners are single-tenant and macOS's +per-user temporary directory is not shared. This is a named limit of the local consumer, owned +by [deferred-tasks.md](../../roadmap/deferred-tasks.md#wave-3-step-9a1--recorded-low-findings-2026-10-24). "Owner-only" in this +section means mode 0700 for directories and 0600 for staged files, set — and for the staging +directory verified together with this process's uid — where POSIX modes are observable. On Windows +no mode evidence is claimed: the candidate root and staging inherit the temporary directory's ACL, +which is not verified, as the graph checks above claim no unobservable POSIX mode bits. The +private map manifest and actual packed CLI and engine symbolication are Step 9a.2a (below); +`check:map-controls` and the `package_test` receipt flip are Step 9a.2b; +recipient custody, the release state controller and the workflow graph are Steps 9b–9d. +Until those and Wave 4b's production recipient exist, the release workflow's publish job +refuses every artifact (the [Interim release guard ruling](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) of 2026-10-19). + +> Amended 2026-11-04: decision D59 (release path B, in the +> [MVP priority reset](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#mvp-priority-reset--2026-11-04)) +> replaces "until those exist" with a pre-1.0 release ADR that publishes the smoke-verified package-test +> tarball, with a sunset at 1.0 or when Steps 9c/9d land. That ADR must explicitly override +> [ADR-0111](../../decisions/0111-cli-meta-operations-use-a-lazy-packaged-runtime.md)'s "missing custody cannot +> downgrade to test mode". The guard stays in force until the ADR is Accepted and its guard change passes a +> security sitting. The publish job runs in the `release` deployment environment, which holds the npm token. + +### Private manifest and symbolication implementation boundary — Step 9a.2a + +Step 9a.2a (a [recorded split](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24) +of 9a.2, maintainer-approved on 2026-10-24) implements items 4 and 5 above for the package-test +producer. The receipt stayed `compiled_package_test` until 9a.2b's product map controls existed. +"Owner-only" in this section has the meaning the 9a.1 section gives it: mode 0700 or 0600 where +POSIX modes are observable; on Windows the private root inherits the temporary directory's ACL, +which is not verified. The producer needs Node 22.15 or later, whose `module.registerHooks` the +preload uses (CI's pack jobs run the `.nvmrc` major); on an older Node it refuses with +`PACKED_CLI_PRODUCER_NODE` before anything is built. + +- **Strict maps.** The finalizer decodes every consumed upstream map's and every final map's + `mappings` strictly — base64 VLQ only, no truncated or oversized value, segments of one, four or + five fields, increasing generated columns, in-range source and name indices, no negative position — + and requires every generated position inside the text the map was generated for and every + original position inside its source text (a position at a line's end is a caret position). A + violation refuses with its own code (`UPSTREAM_MAP_MAPPINGS`, `UPSTREAM_MAP_RANGE`, + `MAP_MAPPINGS`, `MAP_RANGE`) before anything is written; a well-formed charset alone no longer + decides, and the bundler's silent fallback for a malformed upstream map is not relied on. Node's + built-in source map decoder is lenient, so it serves only as an independent test oracle. +- **Private manifest.** After the archive inspection the seal writes `manifest.json` into private + staging (owner-only, never overwritten): version 1; the source commit, the commit's Git tree and + the `dirty` flag; the package version; the build target; Node, pnpm, TypeScript, tsup and esbuild + versions from the installed tools; SHA-256 digests of the lockfile, every package manifest and + every build configuration with the configurations it extends; every final JS file keyed as + `dist/.js` with its SHA-256, its map's package-relative path (`dist/.js.map`) and + the staged map's SHA-256; a SHA-256 for every original source a final map names + (repository-relative, no content); and the tarball's SHA-256. The staged maps must still be the + bytes the finalizer recorded and each JS digest must be the tarball entry's. It holds no host path, + credential or source text and is removed with the staging. +- **Actual frames.** The seal then runs the finalized `dist` — which the inspection has proven + byte-equal to the tarball entry by entry — under a tooling-only `node --import` preload while it + imports a canary workflow (`relavium import`, with its home, project and captured frames in an + owner-only directory of the private root). The project's own `.relavium/` exists before the run, + so the CLI's upward search for a project stops there and never adopts an ancestor's — a + developer's real project on Windows, where the temporary directory lies below the home — and the + run must have imported the canary into it (`CAPTURE_PROJECT`). The preload wraps the external + `yaml` package's `parse`, records V8's structured call sites for the canary document alone, + restores the stack formatter and limit, replaces the keyring package with a module that refuses + every use, and makes any native addon load refuse, so the run loads no native module and reads no + secret. Through a synchronous resolution hook that sees `require` as well as `import`, it also + refuses any module that resolves outside the candidate, so an optional dependency the candidate + does not install (ws's `bufferutil`, for example) can never be satisfied by a `node_modules` that + another local user creates above it in a shared temporary directory. These refusals are installed + in the capture process itself: a thread or subprocess the candidate starts without the preload (an + eval Worker or a `node -e` child, for example) is not covered, and `relavium import` starts none, + though nothing enforces that. Exactly one captured stack is required. +- **Symbolication.** Each frame inside the finalized `dist` must be a file the manifest and the + tarball record (re-hashed after the run), its map must be the staged bytes the manifest records, + and only a frame that lands exactly on a mapping segment counts. Five named call sites must be + reached — three in the CLI and two in the inlined `@relavium/core`, each the one call to a named + function inside another, and the table itself refuses fewer than two distinct core or three + distinct CLI call sites — at the line and UTF-16 column that the matching source revision's + syntax tree gives: for a + clean candidate the checkout's Git objects at the recorded commit, read as the raw blob (no filter + or text conversion) by Git without home, global or system configuration and without replace refs + (the checkout's own configuration is still read), and for a dirty local one the candidate copy, + both bound to the manifest's per-source digests. Wrong JS, a wrong map, permuted map sources and a + wrong source revision therefore cannot reach the anchors. A refusal prints one fixed code + (`PACKED_CLI_MANIFEST_*`, `PACKED_CLI_SYMBOLICATION_*`) and never a frame, path or source text, and + no receipt is written. + +The capture child is not sandboxed: it runs the candidate CLI's runtime import closure as the +producer's user, with network access, beside the private staging, confined only by its isolated +environment, its owner-only work directory and, within the capture process only, the refusal of any +module outside the candidate and the refused keyring and native modules. `relavium +import` makes no network request, but nothing enforces that. This is a named limit of the +package-test producer, owned by +[deferred-tasks.md](../../roadmap/deferred-tasks.md#wave-3-step-9a2a--recorded-low-findings-2026-10-24). + +### Product map controls and the package-test receipt — Step 9a.2b + +Step 9a.2b (the [preflight decisions](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-preflight-decisions--2026-10-24), +maintainer-approved on 2026-10-24) closes ADR-0111's post-receipt rebuild refusal for the +package-test producer, adds the product map controls and only then flips the receipt. + +- **Rebuild refusals.** A second prepack into a sealed staging refuses before any build + (`STAGING_NOT_EMPTY`). `sealPackage` refuses with `PACKED_CLI_ALREADY_SEALED` at its entry, before + the inspection and any capture, when a receipt already lies beside the tarball or the staging already + holds a manifest (checked without following a link); any other extra file in the artifact directory + keeps the public artifact check's own refusal (`PACKED_CLI_UNEXPECTED_PUBLIC_ARTIFACT`), and a receipt + another seal wrote meanwhile refuses with the same code. The refusal is keyed on the artifact + directory and the staging, not on the candidate's identity. Immediately before the receipt is + written, `verifySealedDist` compares the whole `dist` inventory, every output's digest and the + tarball's digest with the manifest just sealed (`PACKED_CLI_MANIFEST_DIST_DRIFT`, + `PACKED_CLI_MANIFEST_TARBALL_CHANGED`), so a `dist` rebuilt or changed after the inspection refuses, + also in a chunk whose map names no anchor source. Release-mode retry and no-rebuild rules stay with + Steps 9c and 9d. +- **Product map controls.** `pnpm check:map-controls` (`tools/cli-package/map-controls.mjs`, an + unconditional entry point over `map-controls-core.mjs`) runs one real package-test production at the + checked-out commit through the producer's own ports. A half-configured CI environment refuses before + anything is built (`PACKED_CLI_MAP_CONTROL_ORIGIN`). Inside the seal, after the baseline receipt and + while the private staging still exists, each control corrupts one product copy — the owned candidate + tree, or a copy of its private staging re-recorded so that only the layer under test can refuse — + requires that control's exact refusal and restores every file it corrupted by digest (`dist` itself + is replaced by the next build): + - a second prepack into the sealed staging; a re-seal of the artifact directory against a copy of + the staging without its manifest, and of the sealed staging beside a tarball copied without its + receipt, each isolating one half of `PACKED_CLI_ALREADY_SEALED` with no capture run; + - on every anchor-bearing chunk, wrong JS by line and by column, a map from a build one line apart + and rotated map sources (`PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED`), each checked to have changed + exactly its own chunk in the way it names, after a re-recorded unchanged copy is accepted; + - a wrong source revision, clean (`PACKED_CLI_SYMBOLICATION_SOURCE_REVISION`) and dirty + (`ANCHOR_UNMAPPED`); + - a changed chunk whose map names no anchor source, and a real rebuild from a changed source, against + the sealed manifest (`PACKED_CLI_MANIFEST_DIST_DRIFT`), after the sealed `dist` verifies; and the + seal's own check, which refuses a chunk the run changes after the inspection, writing no receipt; + - an upstream map that is missing, malformed (an extra `sourcesContent`, an out-of-range source + index) or cut inside a VLQ value, each refused by the finalizer with its own fixed line after the + bundler silently succeeded (one build, nothing finalized, staged or packed); a byte-truncated + upstream map, which fails the bundler's own build; and a shifted but strictly valid upstream map, + which the finalizer accepts and symbolication refuses. + + A restoration production must then seal again; its tarball digest is not compared with the + baseline's (see deferred-tasks.md, "Two packs of one source"). Staging copies stay under the private + staging, which the producer removes first on success, failure and interrupt; a refused control + build's `dist`, which holds the bundler's maps, is removed at once. The developer's checkout is read, + never written (Git may refresh its own index's stat cache), and its identity must be unchanged + afterwards. The runner prints one JSON summary of control ids, expected and observed codes and + prepack's fixed lines with their repository-relative subjects — never an absolute path, frame, + position, map byte or raw child output — and exits 1 unless every expected control (seventeen, plus + four for each anchor-bearing chunk) ran exactly once with its expected outcome. It runs in + `pnpm run ci` right after `smoke:cli-package`, and as the required `map-controls` CI job: ubuntu, at + most 20 minutes, read-only permissions, no remote cache credentials, no pnpm cache, no build step, no + custom shell and no upload, inside `ci`'s needs with a guard that refuses every result except success + (the required contexts stay `ci` and `coverage`). That job's production is a second, independent + build that is never shipped, not a retry of the packed candidate, so ADR-0111's no-rebuild rule for a + candidate is not touched. +- **The `package_test` receipt.** The seal and the shared receipt validator (in its check and its + normalized record) use one constant, the earlier `compiled_package_test` is refused, and the receipt + keeps version 1 and its twelve fields; consumers and the aggregate verify identity through that + validator. The mode is a claim, not evidence of the map checks. The `ci` workflow at a commit + requires the controls, but a given artifact is backed only where a successful `ci` run exists at + exactly its commit: `pack-cli` uploads its artifact whatever the same run's `map-controls` job + concludes, a pull request's `ci` runs at its merge commit, not its head, and a release dispatch with + `allow_off_main` or of an intermediate commit has no `ci` run at all; a local dirty production writes + `package_test` too. Steps 9c and 9d must therefore never read the mode as proof. `release.yml` runs + no map controls of its own and its pack job keeps its `pnpm` dependency cache (public registry + packages only) until Step 9d rebuilds that graph, and the interim guard refuses every + release-workflow artifact. + +The runner keeps private maps and manifests longer than the producer does, always inside the +producer's owner-only root; a runner killed outright can leave that root behind, the producer's +existing named limit, and an interrupted runner can leave its own temporary directory, which may still +hold the baseline public tarball and its receipt (see deferred-tasks.md). The CI job runs on Linux only; a local run on Windows claims no POSIX mode-bit +evidence, as above. + ## Recipient custody The maintainer selected encrypted CI archival. GnuPG is release tooling, never a @@ -251,14 +521,236 @@ update for future candidates; preserve historical fingerprint/decryption capabil for supported versions. Existing public ciphertext cannot be recalled by revocation. This wave creates no automatic deletion policy or production key. +### Recipient custody implementation boundary — Step 9b.1 + +Step 9b.1 implements the container and the one-time encryption as a library with synthetic +controls only. It holds no production recipient, encrypts no product staging, and changes +neither `producePackage`, `sealPackage`, the package-test receipt, the interim release guard +nor either workflow; encryption in the originating job is Step 9d's, reuse, the release +binding receipt and the production loader are Step 9b.2's. Its review and acceptance are +recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments). + +- **One strict dialect.** `ustar.mjs` owns the USTAR dialect the archive inspector reads: + regular files only, every header field fixed except the modification time, whose syntax + alone is checked, zero owner ids and device numbers, an exact end and zero padding. The + inspector delegates to it unchanged. +- **The private container.** `container.mjs` builds one owner-only (0600) USTAR from the + sealed staging in its own layout: `finalized.json`, `manifest.json`, `metafile-esm.json` and + `maps/.js.map`, sorted, with no host path. The staging must hold exactly those records + and the manifest's maps as regular files of at most 16 MiB each, at most 259 entries and + 128 MiB in total (the total refused before the archive is assembled). Each record must also + be within the bound its own reader applies (`RECORD_BYTES`: `finalized.json` 1 MiB, + `manifest.json` and `metafile-esm.json` 4 MiB), so no record exceeds the size its own + reader accepts. Every map's digest must + be the manifest's, and the finalizer's record must name exactly the manifest's outputs and + maps with each map's size. The manifest records no digest for the metafile, which must parse + as a JSON object. The written container is read back before it is returned, so a path the + dialect's 100-byte name field cannot hold refuses (`CONTAINER_TAR_NAME`). Reading a container + back applies the same checks, and a decrypted container + extracted with `tar -x` is a staging the manifest reader consumes unchanged. +- **The isolated runner.** `gnupg.mjs` resolves `gpg` from absolute PATH entries only and + refuses one older than GnuPG 2.2.8; the installation — its binary, its `gpgconf` and its + system-wide configuration directory — is trusted. Every call runs in the C locale and an + isolated environment, bounded by `runCommand`; every `gpg` call also runs in a fresh + owner-only home the tool creates, with `--no-options --no-autostart --no-keyring`. GnuPG + still reads the installation's + `/gpg.conf` under `--no-options`, and its `[force]` options override the command + line, so before any key is handled the installation's `gpgconf --list-dirs sysconfdir` is + read — the one call without a home — and a `gpg.conf` or `common.conf` there refuses + (`GPG_CONFIG`): a global configuration + could add or hide a recipient, redirect the output or log the session key. `gpgconf` prints a + named directory raw, ended by one newline; anything but exactly one absolute path refuses. The recipient listing (`--show-keys`) uses its own home + and the default trust model, because GnuPG 2.4 cannot list an expired or revoked key under + `--trust-model always` without a trust database; that home may gain only GnuPG's own + `trustdb.gpg`. Encryption uses `--recipient-file`, `--trust-model always` (the pinned + fingerprints and the colon pre-check are the assurance), an AES-256 preference, no + compression and a fixed literal name, in a home that may gain only `random_seed`. +- **The recipient pre-check.** The recipient file is read bounded and copied, so the bytes + checked are the bytes encrypted to, and pins are exactly two strings, each a 40-digit + uppercase fingerprint (OpenPGP v4). An armored secret key refuses from its bytes; secret + material GnuPG lists refuses as secret, and a file GnuPG cannot list refuses as unreadable. + Secret material GnuPG neither lists nor reads, such as binary secret packets after the + armored block, is admitted and never reaches the ciphertext, the record or a log (a named + limit in [deferred-tasks.md](../../roadmap/deferred-tasks.md#wave-3-step-9b1--named-limits-2026-10-25)). + The colon listing then refuses, + in this order: secret material, a missing pinned primary, more than one listed key, a missing + pinned subkey, revocation, expiry at the current time, unusability (invalid or disabled, no + encryption capability, a creation time in the future, or an encryption subkey other than + cv25519 or RSA of at least 3072 bits) and another usable encryption subkey or additional + decryption subkey (an ADSK, which GnuPG 2.4.1 and later adds as a further recipient). GnuPG's listing + omits a key it cannot use, such as one created in the future or one without a valid user ID, + so such a key refuses as missing or goes uncounted; GnuPG's own refusal and the + `KEY_CONSIDERED` and recipient checks remain the independent second layer for it. +- **Encrypt once.** `encryptOnce` refuses before any GnuPG call when the destination already + holds `relavium--private-maps.tar.gpg`, when the destination and the owner-only + private root overlap (compared by device and inode, so one directory reached by a second + spelling, such as a case-insensitive variant or a bind mount of that directory, does not hide + the overlap; containment through a bind mount of a directory inside the other is not detected, + a [recorded limit](../../roadmap/deferred-tasks.md#wave-3-step-9b1--recorded-low-findings-2026-10-25)), or when either is missing or the root is not an owner-only + directory of this user. No root qualifies on Windows, where modes say nothing about access + and no ACL check exists. Success requires exit 0, exactly one + `KEY_CONSIDERED` for the pinned primary, one `BEGIN_ENCRYPTION 2 9` (SEIPDv1 with MDC and + AES-256; an AEAD profile refuses) before one `END_ENCRYPTION`, exactly the expected work + files, a ciphertext within the container's size plus 64 KiB whose public-key packets name + exactly the pinned subkey and no hidden recipient and whose file is still the bytes about to + be published, and homes holding only what GnuPG creates. The work directory and its + plaintext are removed before anything is published, so a failed removal publishes nothing. + The ciphertext is then published atomically: written to a fresh sibling, then hard-linked to + its final name, which never replaces an existing file, so a failed write never leaves a + partial file under that name. It returns a frozen custody record: name, size, SHA-256, pins + and GnuPG version. +- **Plaintext and diagnostics.** The plaintext container and the recipient copy exist only in + an owner-only work directory under the caller's private root, removed on every path; an + interrupt in the caller's `exitOnInterrupt` scope stops GnuPG and removes the root. No secure + erasure is claimed. Every refusal is a fixed `PACKED_CLI_CUSTODY_`, including a host + failure (`UNEXPECTED`, or `CONTAINER_UNREADABLE` for a staging that cannot be read) and a + work directory that cannot be removed (`CLEANUP`); GnuPG's stdout and stderr, paths and host + error text never surface. +- **Synthetic fixtures.** `custody-fixtures/` commits armored public v4 keys under + `fixture.invalid` for the classes GnuPG can produce — good cv25519 and RSA-3072 keys, one + advertising AEAD, an RSA-2048 subkey, an expired primary and subkey, a revoked primary and + subkey, a sign-only key and a key with two encryption subkeys — with their pins and a + maintainer-only generator that is never run in CI; the other classes are covered by crafted + listings. No secret key is committed: the decryption round trip generates one per + run in a short owner-only home and stops its agent afterwards, and a test refuses any tracked + PGP private-key block. +- **Where it runs.** The custody tests are part of `test:acceptance-tools`, so they run in local + `pnpm run ci` and the four Ubuntu jobs inside `ci`'s needs, against the runner's GnuPG; they + fail, never skip, without GnuPG ≥ 2.2.8 or with a global configuration + ([local-dev-setup.md](../../runbooks/local-dev-setup.md)). They do not run + on Windows: no Windows job runs `test:acceptance-tools`, the `gpg` on the image's PATH is + Git for Windows' MSYS2 build and MSYS2's own GnuPG under `C:\msys64` is installed off PATH, + neither exercised here, and the tests use POSIX process and permission + tools; `encryptOnce` itself refuses there. The 2.2.8 floor is enforced by the version check + without being exercised. + +### Recipient custody implementation boundary — Step 9b.2 + +Step 9b.2 implements the reuse of a retained ciphertext without re-encryption, the release binding +receipt's schema v1 with its validator and sealer, the production recipient loader (absent in Wave 3) +and the separation of synthetic and production scope +([decisions](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-preflight-decisions--2026-10-25)). +It is release tooling with synthetic controls only, in new modules: no Step 9b.1 module changes and +nothing outside its tests calls it. Step 9d wires it into the originating job, binds the receipt's +origin and map-check claims to the trusted run, and replaces the interim release guard, which still +refuses every artifact. Its review and acceptance are recorded in the +[Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments). + +- **The production recipient.** `release-recipient.mjs` loads it from one committed directory, + `tools/cli-package/release-recipient/`, holding exactly `recipient.asc` (the + `gpg --armor --export '!'` export, at most 64 KiB) and `pins.json`: + `{"version": 1, "recipients": [{"primary", "subkey", "status"}]}` with 1–32 entries of 40-digit + uppercase fingerprints, each `current` or `retired`, exactly one `current` and no subkey twice. An + entry must never be removed, so that a candidate sealed before a rotation still verifies; the + loader sees one revision and cannot detect a removal, so that rule is kept by review until a + control over the file's history exists (a recorded Wave 4b handoff). `loadProductionRecipient()` + takes no argument: there is no environment variable, flag or fallback. In Wave 3 the directory + does not exist and the loader refuses `PRODUCTION_RECIPIENT_ABSENT`; a linked or malformed + directory, and any host failure while loading, refuses `PRODUCTION_RECIPIENT_INVALID`. A history + naming a fingerprint that the committed fixtures' `pins.json` lists refuses `FIXTURE_RECIPIENT`, and + a malformed fixtures file refuses rather than weaken that denylist. `pins.json` must be exactly its + canonical serialization (`JSON.stringify(value, null, 2)` plus a newline, fields in the order above), + so a duplicated key, which `JSON.parse` would resolve silently to its last value, refuses and the file a + reviewer reads is the history loaded. The loaded value is frozen and + branded: only that value counts as a production recipient, never a copy. Tests load temporary + roots through the separately named `loadRecipientForTests`, which `release-scope.test.mjs` + permanently keeps out of every product and tooling file. Wave 4b commits the files. +- **Scope.** A synthetic candidate is one whose private manifest records the fixtures' sentinel + source, commit `c`×40 and tree `b`×40. `encryptCandidate` admits production scope only with the + loaded recipient and a clean, non-synthetic candidate, and encrypts to the recipient's current + pins (`RECIPIENT_SOURCE`, `DIRTY_SOURCE`, `SYNTHETIC_SOURCE`). It admits synthetic scope only with a + synthetic candidate (`FIXTURE_SCOPE`), so neither a fixture key nor the per-run round-trip key can + encrypt a product staging. An unknown scope refuses `SCOPE`. The destination and the private root + must be normalized absolute paths (`PLACEMENT`, `PRIVATE_ROOT`), so that a trailing separator, `.` or + `..` cannot make a link look like a directory. The destination must then be a plain, resolvable + directory (`PLACEMENT`; a link refuses before anything is read through it, as `encryptOnce` requires) + holding exactly the tarball the manifest names: a receipt there refuses `ALREADY_SEALED`, the ciphertext + `ALREADY_ENCRYPTED`, any other entry, or a tarball named otherwise, `CANDIDATE_ENTRIES`, and a different + tarball `TARBALL_DRIFT`. A ciphertext is therefore never encrypted where the sealer would refuse its + directory. Every one of these refuses before any GnuPG call. The gate reads the caller's scope once + and encrypts its own copy of the container, so neither can change midway; `encryptOnce` then runs + unchanged, on that copy and the caller's destination. +- **The release binding receipt v1.** The public `release-binding-receipt.json` beside the ciphertext, + the name the interim guard already keys on: + + | Field | Content | Checked here | Trust Step 9d adds | + | --- | --- | --- | --- | + | `version`, `kind` | `1`, `"release_binding"` | exact | — | + | `packageVersion` | the private manifest's version | equals the expectation | the tag | + | `source` | the manifest's `{commit, tree, dirty}` | the commit equals the expectation; production refuses `dirty` and the synthetic source, synthetic requires it | the run's commit | + | `tarball` | `{name, bytes, sha256}`, `relavium-.tgz`, 18 B–16 MiB | the name follows the version; the sealer binds the digest to the manifest's `tarballSha256` | — | + | `ciphertext` | `{name, bytes, sha256}`, `relavium--private-maps.tar.gpg`, 1 B–128 MiB + 64 KiB | the name follows the version; the sealer re-reads the file | — | + | `recipient` | `{primary, subkey}` | production: a pair of the loaded history; synthetic: the expected pins | — | + | `toolchain` | the manifest's node, pnpm, typescript, tsup and esbuild versions, and the GnuPG version | format | — | + | `target` | the manifest's target | format | — | + | `inputs` | the manifest's per-file lock and configuration digests, by path | format: 1–64 repository-relative paths | — | + | `mapCheck` | `{"status": "complete"}` | exact | a successful `ci` run with the product map controls at `source.commit` | + | `origin` | `{kind: "github", repository, workflow, runId, runAttempt, event: "push", source}` | format; `source` equals `source.commit` | the originating run and attempt | + + It carries no map, source, per-source inventory, host path or user ID, and no scope field: scope + always comes from the verifier's expectation. Every field is a claim until Step 9d binds the last + two. Release smoke on a reused candidate derives the public file inventory and package metadata from + the digest-verified tarball (Step 9d). +- **Canonical bytes.** The receipt is `JSON.stringify(receipt, null, 2)` plus a newline, its fields in + the table's order and its inputs sorted by path; an integer-like input name, which JavaScript would + order first, refuses. A verifier parses the exact bytes, at most 64 KiB, and refuses + `BINDING_BYTES` for anything but the canonical serialization of the value it validates: a + duplicated key, a reordering, other whitespace, a BOM or trailing data. A verifier accepts only + those exact bytes and never writes a receipt back. +- **The validator.** `validateReleaseBinding(value, expected)` and `parseReleaseBinding(bytes, expected)` + take a required expectation: `{scope: 'production', recipient, packageVersion, source}` with the + loaded recipient, whose whole history verifies, or `{scope: 'synthetic', pins, packageVersion}`. + Each reads the expectation once. They refuse `BINDING_VERSION`, `BINDING_SHAPE` (an unknown, missing or malformed field at any level), + `BINDING_CANDIDATE`, `BINDING_NAME`, `BINDING_SIZE`, `BINDING_DIRTY`, `BINDING_SCOPE`, + `RECIPIENT_SOURCE`, `BINDING_RECIPIENT`, `BINDING_MAP_CHECK` and `BINDING_ORIGIN`. +- **The sealer.** `sealReleaseBinding(record, {origin})` accepts only a record `encryptCandidate` + returned and not yet sealed (`RECORD_SOURCE`); a failed seal leaves the record usable. It refuses an + existing receipt before reading anything (`ALREADY_SEALED`), admits exactly the tarball and the + ciphertext in the destination (`CANDIDATE_ENTRIES`), and re-reads the ciphertext's size and SHA-256 + against the record (`CIPHERTEXT_DRIFT`) and the tarball's digest against the manifest + (`TARBALL_DRIFT`). It validates its own receipt, then publishes it once: written to a fresh sibling, + then hard-linked to its name, which never replaces an existing receipt. A sibling that cannot be + removed after the link is left behind rather than reported, because the receipt is then published; + the candidate is not reusable (`CANDIDATE_ENTRIES`) until that sibling is removed by hand. A + ciphertext whose seal never succeeds before its process exits — after a crash, or a refusal on an + input the gate does not check, such as the origin — can never be sealed and needs manual recovery. +- **Reuse.** `verifyRetainedCandidate({directory, expected})` admits a directory holding exactly the + receipt, the tarball and the ciphertext (`CANDIDATE_ENTRIES`) and parses the receipt. It checks each + asset's size (`CANDIDATE_SIZE`), reads it as a bounded ordinary file (`CANDIDATE_ASSET`), checks its + SHA-256 (`CANDIDATE_DIGEST`), and returns the verified bytes, never a path to open again. It writes, + renames and removes nothing. Its module imports nothing that can start a process, so reuse makes no + GnuPG call, and it cannot re-check the ciphertext's recipients: a named limit. +- **Diagnostics.** Every refusal is a fixed `PACKED_CLI_CUSTODY_`. A host failure is `UNEXPECTED`, + except on a named path, which carries that path's code: `PLACEMENT` when the destination cannot be + checked or resolved (one that resolves but cannot be listed is `UNEXPECTED`), `CANDIDATE_ASSET`, + `CIPHERTEXT_DRIFT` and `TARBALL_DRIFT` for the files read, and `PRODUCTION_RECIPIENT_INVALID` while + loading the recipient; inside encryption, `encryptOnce`'s own codes apply. No path, key or receipt content surfaces. +- **Where it runs.** `release-recipient.test.mjs`, `release-binding.test.mjs`, `release-custody.test.mjs` + and `release-scope.test.mjs` join `test:acceptance-tools`, as the 9b.1 tests do; the custody test + needs GnuPG ≥ 2.2.8. `release-scope.test.mjs` pins the separation statically: + - no product or tooling code reaches a test fixture, and only the loader reads the fixture pins; + - only the release gate mentions `custody.mjs`, the one tracked file of that name, anywhere in source, + workflow and package-script text, comments included, once `\x`, `\u`, `\u{…}` and percent escapes, + separators and case are normalized; a path built at run time from pieces, or spelled with another + escape, is beyond this control (a recorded limit); + - the loader is never called with an argument, and its test seam appears in no product or tooling + file (permanent controls); + - the receipt module's import closure, read from every `from` or `import` specifier whatever the + spacing, holds only its own files and listed Node builtins, with no package, process module or + `require`-like call, and every dynamic import takes exactly one string literal; + - the interim guard still refuses a complete, valid synthetic candidate; + - until Step 9d no product path or workflow names a release module, and until Wave 4b no recipient + directory is committed. + ## Package-test and release modes **Package-test mode** is required on PR/push and manual dry-run. In its isolated uncached originating pack job, build once, inspect exact tarball, verify maps and CLI/engine symbolication locally, then discard all private maps/manifests/metafiles. -Share only the tarball and a nonsecret package-test receipt (version, mode -package_test, source SHA, package version, tarball digest and originating workflow -identity) with platform consumers. It needs no production recipient. GPG fixture +Share only the tarball and a nonsecret package-test receipt (version 1 with mode +`package_test`; the source SHA and dirty flag; the package version; the tarball's name, +digest and size; the public file inventory and package metadata; the originating workflow +identity and the producer toolchain) with platform consumers. It needs no production recipient. GPG fixture controls remain separate and use synthetic map payloads only. A package-test artifact is never promoted to a release candidate after private diff --git a/docs/reference/cli/regression-harness.md b/docs/reference/cli/regression-harness.md index ad914aff..3d056e02 100644 --- a/docs/reference/cli/regression-harness.md +++ b/docs/reference/cli/regression-harness.md @@ -127,12 +127,14 @@ journey, and the read-path query shapes. These are additive `.e2e.test.ts` suite ### The Windows lane (2.5.I S6) -An advisory `windows-concurrency` job in [`.github/workflows/ci.yml`](../../../.github/workflows/ci.yml) runs +A required `windows-concurrency` job in [`.github/workflows/ci.yml`](../../../.github/workflows/ci.yml) runs the full `@relavium/db` test suite (the concurrency/retry white-box tests among it) + the CLI concurrency/perf/session-chain harness on `windows-latest` (the WAL locking, the retry's `Atomics.wait` sleep, the two-process child spawn + `file://` import, and the native `better-sqlite3` addon are the parts most likely to diverge), plus a headless no-TTY smoke. It is a -**separate, advisory** job — the required check stays the ubuntu `ci` job. POSIX `0600`/`0700` permission +**separate** job and, since 2026-11-04, a **required** check beside the ubuntu `ci` job, because it carries +ADR-0106's Windows filesystem-policy scenarios ([ADR-0106](../../decisions/0106-media-publication-requires-bounded-verified-bytes.md) +names required CI for them). POSIX `0600`/`0700` permission assertions are a documented Windows no-op ([ADR-0050](../../decisions/0050-cli-history-db-at-rest-posture.md)) and are not exercised there; `release.yml` separately smokes the built binary across ubuntu/macOS/Windows. diff --git a/docs/reference/shared-core/README.md b/docs/reference/shared-core/README.md index 8a583b4b..0adeb4f8 100644 --- a/docs/reference/shared-core/README.md +++ b/docs/reference/shared-core/README.md @@ -20,5 +20,6 @@ Part of [reference/](../README.md). | [run-plan.md](run-plan.md) | The `RunPlan` the DAG builder (1.M) compiles from a validated workflow — engine vertices, topological order, dependency edges, per-type config. | | [built-in-tools.md](built-in-tools.md) | Built-in tools available to local agents (read_file, run_command, web_search, git, …). | | [tool-registry.md](tool-registry.md) | The engine-side `ToolRegistry` + dispatch and the host-injected `ToolHost` capability seam — policy/mechanism split, dispatch lifecycle, guardrail enforcement, result bounding, untrusted-data taint (1.T / [ADR-0037](../../decisions/0037-engine-tool-execution-boundary.md)). | +| [diagnostic-secret-backstop.md](diagnostic-secret-backstop.md) | The finite secret shapes the LLM diagnostic backstop (`scrubSecrets`) masks in provider errors and CLI diagnostics, and the limits of that list. | | [mcp-integration.md](mcp-integration.md) | MCP: agents as MCP servers, and agents consuming MCP tools. | | [action-guard-seam.md](action-guard-seam.md) | The optional, host-injected `ActionGuard` seam an external action-governance control plane plugs into at the side-effecting tool boundary: decide/commit/compensate, verdict union, off-by-default ([ADR-0041](../../decisions/0041-external-action-governance-seam.md), Accepted — contract settled, implementation deferred to the Phase-2 enterprise track). | diff --git a/docs/reference/shared-core/built-in-tools.md b/docs/reference/shared-core/built-in-tools.md index 09662fbb..8dc999ad 100644 --- a/docs/reference/shared-core/built-in-tools.md +++ b/docs/reference/shared-core/built-in-tools.md @@ -93,17 +93,183 @@ This is conservative pattern masking, not proof that arbitrary sensitive content absent; raw dispatch/model results, full spills and historical rows retain their existing content-sensitive handling. -LF/TAB inside a recognized token-shaped credential extends the masked original span. -An ambiguous token-alphabet continuation after LF/TAB is conservatively masked with -that credential. Ordinary spaces and punctuation separate public prose; the masker -does not discard the whole diagnostic because a credential appears. Supported split -shapes are recognized **before** complete-text one-line normalization, followed by -ordinary recognition and length clipping. This preserves multiline quoted-value -recognition without exposing a split token's suffix. Public argument bytes are not +A raw `?` or `#` before a URL authority's `@` begins public query or fragment text only when the +text before it is host-shaped (a dotted name or IPv4, `localhost` or a bracketed IP literal, with an +optional port) **and** an `=` assignment follows it; otherwise the userinfo is masked. This is the +same predicate as the +[LLM diagnostic backstop](diagnostic-secret-backstop.md), including +its 260-character bound on the host-shaped prefix. Masked userinfo that holds such a raw `?` or `#` +leaves the authority ambiguous: the userinfo may also be a query that another rule reads +(`https://gateway?password=P@ss.word1`). The backstop masks through the authority's end unless a +host-shaped name follows its last `@`. This projection, like the backstop, finds every rule's ranges +on the same text and merges overlapping ranges before it replaces anything, so no rule's mask +removes the key name, scheme label, PEM marker or URL scheme another rule reads. Its key/value and +opaque-query rules read every key and parameter in a userinfo but its username's and mask each value +they recognize — a secret key's, unquoted from six characters, or an opaque one of 24 or more +characters — to its own end: through the last `@`, and on through the host, path and query when no +delimiter of theirs comes first (`https://gateway?password=P@ss.word1` → `https://[redacted]`; +`jdbc:sqlserver://db.example:1433;user=sa;password=P@ssw0rd2024;encrypt=true` → +`jdbc:sqlserver://[redacted];encrypt=true`; `https://user:password=@host.example/v1/a` → +`https://[redacted]`). Such a value is a recognized credential, so calls that differ only in the +host or path it runs through share a journal projection. A value no rule recognizes is masked only +as far as the userinfo's last `@`, so its text after that `@` reads as the host +(`https://svc?a=1&code=Adm1n@2024prod` → `https://[redacted]@2024prod`), where the backstop masks an +authority with no host-shaped name after its last `@` to its end. The username is the text before +the userinfo's first `:` when no raw `?` or `#` comes before that colon; a userinfo with no such +colon has none, and a username never reaches past an LF or TAB. A key in the username names the +user, so its value does not run on through the public host and path +(`https://x-access-token:@github.com/o/r` → `https://[redacted]@github.com/o/r`), and a key on +the line after a split URL masks its value to its own end +(`see https://\nDB_PASSWORD=P@ssw0rd123 now` → `see https://\n[redacted] now`). The projection +otherwise differs from that backstop by design: its authority also ends at `"`, `'`, `}` and `]`, +the delimiters of the JSON and quoted texts it reads, so userinfo that contains one of them — an +already-masked `[REDACTED]` password among them — is left as it is; it recognizes any scheme of up to +32 characters rather than a finite list; and a userinfo with no password is kept as a vanity handle +below 16 characters and masked from 16 on, under every scheme. Since 2026-11-04 a `,` or `/` in a +password no longer ends the authority: the userinfo is read on to the first `@` within 512 characters +(`https://user:pa,ss@host` and `https://user:pa/ss@host` → `https://[redacted]@host`). A port +(`https://host:8080/a@b`, `https://host:8080,bob@x.example`), a public query contact as above, another +scheme, or a quote, `}`, `]` or whitespace first keeps the text public; the backstop still stops at a +`/` (a recorded low). + +In an argument vector, the element after a credential flag, or the value of its `--flag=value` +spelling, is masked whole. A flag is a credential flag when it is one of the anchored names (`-p`, +`--password`, `--token`, `--api-key`, …) or its last word names a credential, read with the object-key +rule's words as written (`--auth-token`, `--access-token`, `--db-password`, `--pwd`, `--x-api-key`, +`--credentials`). A flag that only mentions a credential (`--password-stdin`, `--token-file`, +`--max-tokens`) or begins with `no-` does not count, so the argument after it keeps its own identity. +Only `-p` keeps a port exception. curl's `-u`/`--user` value, glued (`-ualice:secret`) or not, is +masked when it holds a `:`, unless it is a `uid[:gid]` or a URL. + +LF/TAB inside a recognized token-shaped credential extends the masked original span. An ambiguous +token-alphabet continuation after LF/TAB is conservatively masked with that credential; an +LF/TAB-split header label that begins a line inside such a continuation is still read (`sk-…` ending +a line before a split `api-k\tey:` label). A credential that begins such a line is masked as far as +the continuation reaches, so when the continuation takes the start of its prefix and the credential +is itself split again, its rest stays visible (`hf_…` ending a line before an LF-split `glpat-…`). +An LF or TAB right after a URL's `://`, right before its userinfo's `@` or inside its `scheme://` +belongs to the split userinfo, which is masked as one span. A credential glued to the end of another +starts at the earlier mask's boundary, so it is read on the next pass; a run of three or more glued +credentials (`AKIA…ASIA…AKIA…`) therefore does not settle (below). When the earlier mask takes the +start of the glued credential's prefix — for example a variable-length token masked to the end of +its alphabet (`ghp_ghp_`) or its LF/TAB continuation, a scheme label's token, or a key's +value masked to its delimiter — the rest of that credential stays visible, as it did before this +projection's corrections; so does the first line of a glued credential split by an LF or TAB whose +continuation another rule masks in the same pass, which leaves that line below its family's minimum +(`AKIA…ghp_<15 characters>\ntoken=…`). A quoted value closes on its line, at the later +of two readings: one with the masks before and inside it filled, as the next pass reads them, and one +where each mask keeps the backslashes it ends with, as the text has them. So a mask that takes the +backslash escaping a later quote neither closes the value early +(`password="Bearer\nabcdefgh token=xyz\"tail" more` → `[redacted] more`) nor, when that quote is the +value's only closer, leaves it open (`password="say \"…?sig=<24 or more characters>\"` → +`[redacted]`), in a summary too. Both readings are also taken with the scheme labels' tokens filled, +as the key rule read them before this projection merged its rules, so a token that takes the value's +line break lets it close at its real quote. One that does not close on its line is decided once the +pass's other masks are known: when one of them takes its line break (a scheme label's token, a nested +key's value), it closes as the next pass reads it and is masked whole, as before this rule, and it is +read with and without the key rule's own masks, so another key's value that takes its closing quote, +or ends at its opening one, does not keep it open (`password="a b Bearer\nxyzxyzxy token="q" more` → +`[redacted] more`); otherwise it is read as an unquoted value from after its quote — a run +of at least six characters up to whitespace, a quote, `,`, `;` or `&` — and no further than its +line's end, since its quote may instead close a string the key sits in: the text after such a quote +stays (`stop logging "token=" values`), so calls that differ only there keep distinct projections, +and the later words or lines of a value that never closes or closes on a later line stay visible +(`password="firstline\nsecond"` → `[redacted]\nsecond"`; a first line under six characters stays as +well). A run is masked from its key in the pass that finds it, so a key or a line break that only a +later pass reads, behind a glued credential's mask +(`password="AKIA<16>abcdefghijklm_token: \n…"`), goes with the run and the value behind it stays. +A value opened by an escaped quote (`\"…\"`, as JSON inside a JSON string writes it) is read one +decoding level down, also behind escaped tabs and line breaks after its separator +(`password:\t\"…\"`), where an apostrophe, which JSON leaves unescaped, opens a value too +(`password:\n '…'`); a raw double quote behind them, as `util.inspect` or `repr` print one +(`password:\t"…"`), does not, a documented limit. A key inside an opaque query value loses its value +to that value's mask, which runs to the escaped quote and takes its backslash +(`?state=<24 or more characters>;password=\"…\"` keeps the quoted text), a documented limit. Only +its escaped quote closes it: a raw line break, behind a backslash or not, or a raw quote of its own +kind ends the string around it, so a value that has not closed by then is read as a run, like one +that does not close on its line (`"stop logging \"token=\" values"` stays as it is); a raw quote of +the other kind does not end that string, since an apostrophe is ordinary text in a JSON string, so +inside a string delimited by that kind the value runs on. One quoted two or more JSON levels down is +outside the shapes, and so is a single-quoted value inside a JSON string whose apostrophe is escaped +(`'ab\'…'` written as `'ab\\'…'`): JSON doubles the backslash, so the value closes at that +apostrophe and the rest stays. A key's value runs through every span an earlier rule masks, line +breaks included, as through an applied mask, so a quoted value that carries a split credential or a +PEM block is masked whole. A key word must start within the first 33 characters of its word, counted +from the last word boundary (a `-` starts one, a `_` does not), so a longer name is no key +(`MY_APPLICATION_PRODUCTION_DATABASE_PASSWORD=…` stays). A key's mask starts where its word starts, +so a credential glued to the front of a key name (`password=…`) is masked with that key's +value and stays when the value is empty or too short to mask; a glued token of 33 or more characters +puts the key word past that window, so no key is read and the value stays visible unless another rule +recognizes it (`ghp_<36 characters>password=…`). A key inside another key's value, and +a scheme label inside another label's token, start a credential of their own +(`token=a>password: ` → `[redacted]`; `Bearer a/Bearer ` → +`Bearer [redacted] [redacted]`), and an LF/TAB-split header label or a scheme label may start right +after a credential or a PEM block, as after an applied mask. Core's scheme rule reads one unquoted +label followed by whitespace, so a self-quoted label (`"Bearer" `) or a concatenated run +(`BearerBearer `), which the backstop masks, is outside its shapes. A scheme label that takes a +URL scheme as its token no longer hides that URL's password (`token postgres://app:pw@db` → +`token [redacted]://[redacted]@db`). The projection repeats until its output stops changing, at most +three passes, and becomes `[redacted]` if it has not settled — as a run of glued credentials or a +key whose short value names the next key can need more — so projecting a projection changes nothing; +a display field that does not settle (three glued fixed-length ids suffice) therefore shows as +`[redacted]` whole, and of such calls only a tiered one is refused. A tiered call whose journal +projection has not settled, or whose arguments hold two member names of one object that project +alike, which leaves one of the members, is refused before it is prepared +([tool registry](tool-registry.md#resolution--the-dispatch-lifecycle)), so neither that constant nor +a lost member stands for two different arguments in the journal. Every rule's scan stays linear in +the admitted text, and none uses a backtracking pattern whose stack an admitted value can exhaust: a +quoted value is read by a character scan no further than its line, and a summary decides its values +that stop at a line break by one backward scan. Ordinary spaces and punctuation separate public +prose; the masker does not discard the whole diagnostic because a credential appears. A summary uses +the projection with the differences below, since a summary is shown, never compared. A quoted value +that one of its readings stops at a line break is masked through the nearest later quote that could +close it outside the values the rule masks or, when every such quote is inside one, through the last +of them, and only else to the end of its line, in whichever reading reaches further, and through the +closing of a reading that closes too; one that reaches the end of the string around it or of the text +without closing is masked to that end. A key's value whose escaped opening quote ends an earlier +mask's backslashes opens where that mask keeps them, so a query value at the end of a line that runs +on through the next line's key in a structured result's JSON does not hide that key's value (a +`cat .env` result's `?sslmode=require\nPOSTGRES_PASSWORD=\"…\"` → `?sslmode=[redacted]\n`); the +journal projection keeps its reading, so the same JSON as a text argument still shows such a value, a +documented limit. A structured (non-string) result is summarized from its JSON with +every member under a credential-named key and every argument after a credential flag masked whole +(a count such as `input_tokens` too, the text rules then masking the member's name with it). The text +rules read that JSON once before those masks, which the same pass applies, so a quoted value or a PEM +block that runs into or out of a masked member is masked across members, as reading the unwalked JSON +masks it. The summary marks those values with the private-use character U+E000, so a result whose JSON +already holds one is summarized from the walk's masks alone and such a value or block is not masked +across members there, a documented limit. The text rules read the JSON as one text, a value one +JSON string down included, so a JSON-escaped line break is no delimiter and a PEM block or a quoted +value that runs across members keeps its key and markers. A summary also reads a private-key marker whose words a line break, a tab or more than +one space separates (`-----END RSA PRIVATE\nKEY-----`), as a block wrapped at a fixed width can split +one: one to eight whitespace characters between `PRIVATE` and `KEY`, and up to 40 characters of type +words and whitespace after `BEGIN` or `END`. A marker wrapped further is read only after the summary's +one-line normalization, which a key's value that takes the BEGIN marker first prevents +(`PRIVATE_KEY=-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE\n KEY-----` shows the body), +a documented limit. The journal projection reads such a marker as before, and a structured result's +summary does not read one whose line break or tab its JSON escapes +(`{"a":"…-----END RSA PRIVATE\nKEY-----"}`), a documented limit. A private-key block is masked only from a BEGIN marker to an END marker that starts +at most 20,000 characters after it, so a block cut off before its END marker (`head -n 5 key.pem`) +stays visible in every projection, a documented limit. A block ends at its first END marker even where +another rule's match is glued to it (`?sig=-----END …`): text after that marker stays, and a key's +quoted value whose closing quote lies inside the block shows its front +(`api_key='x -----BEGIN …' ?sig=-----END …` → `api_key='x [redacted]`); base masked these only +because its rules ran in turn and the glued match erased the marker, a documented relaxation. In a +summary, a key's unquoted value behind an escaped line break and indentation (`password:\n ` in a +structured result's JSON) is not read, a documented limit. Every summary also reads its text a second time with each +escaped line feed, carriage return or tab (`\n`, `\r`, `\t`, behind any run of backslashes) as that +separator, for the rules a separator decides: a credential or a scheme label right after one, and a +credential split by one, are masked as in raw text (`{"stdout":"keys:\nsk-…"}` → +`{"stdout":"keys:\n[redacted]"}`), while a header label split by one is not read; the journal +projection reads no escape. Core does not read a credential or a scheme label glued to a terminal +sequence's final byte (`ESC[1msk-…`), which leaves no word boundary there, a documented limit; the +backstop's terminal handling is in [error handling](../../standards/error-handling.md). Supported +split shapes are recognized **before** complete-text one-line normalization, by that projection's +full fixed point, followed by ordinary recognition and length clipping. This preserves multiline +quoted-value recognition without exposing a split token's suffix. Public argument bytes are not globally whitespace-normalized for journal identity. The -[journal contract](effect-journal.md#11-secrets-what-a-row-may-hold) owns recognized- -credential equality and persisted compatibility; diagnostic masking grants no -permission to repeat an effect. +[journal contract](effect-journal.md#11-secrets-what-a-row-may-hold) owns recognized-credential +equality and persisted compatibility; diagnostic masking grants no permission to repeat an effect. ## Subprocess environment @@ -124,7 +290,7 @@ A valid host-issued scratch descriptor conditionally adds only its exact subtree ### CLI authority and floors -The CLI owns an immutable, process-local policy with pinned lexical/canonical directory identities, rechecked at each capability operation. Read/list/glob, mention/completion, discovery and write boundaries check lexical provenance and canonical targets; recursive enumeration prunes denied directories **before descent**. An ordinary extra-root grant never overrides confidentiality or protected-write floors, including full tier and auto mode. +The CLI owns an immutable, process-local policy with pinned lexical/canonical directory identities, rechecked at each capability operation. Device and inode compare as exact bigints: a 64-bit file ID (NTFS, or an overlayfs `xino` inode) exceeds a Number's precision. Read/list/glob, mention/completion, discovery and write boundaries check lexical provenance and canonical targets; recursive enumeration prunes denied directories **before descent**. An ordinary extra-root grant never overrides confidentiality or protected-write floors, including full tier and auto mode. Synchronous root admission uses native canonical resolution, matching asynchronous file operations. Windows short-name and long-name spellings must resolve to the same canonical authority; lexical spellings remain separately pinned for provenance and confidentiality checks. Native path membership alone cannot equate two unresolved aliases. @@ -155,7 +321,7 @@ Synthetic tmp/artifact descriptor tests establish this prerequisite only. Automa ### Physical media storage and limits -The filesystem CAS pins its canonical root on the first admitted operation; reads/GC do not create an unused store, and later absence/replacement/retargeting revokes access. Shards/final descendants cannot be symlinks; files must be regular without extra hard links, with exclusive no-follow temporary ownership, one-fd read identity checks, atomic publication and the same physical containment checked for read, delete and GC. CAS has no pnpm exemption. Handle/port semantics live in the [MediaStore reference](llm-provider-seam.md). +The filesystem CAS pins its canonical root on the first admitted operation; reads/GC do not create an unused store, and later absence/replacement/retargeting revokes access. Every root pin, file snapshot, owned-file, claim-directory, marker and read identity check compares device and inode as exact bigints taken from one stat. Shards/final descendants cannot be symlinks; files must be regular without extra hard links, with exclusive no-follow temporary ownership, one-fd read identity checks, atomic publication and the same physical containment checked for read, delete and GC. CAS has no pnpm exemption. Handle/port semantics live in the [MediaStore reference](llm-provider-seam.md). An opened CAS read descriptor may become unlinked when a cooperating publisher atomically republishes the same handle. Reads validate the current root/shard/final-entry floor independently, retain the one opened regular descriptor (at most one remaining link), and verify its bytes against the requested hash before delivery. They do not require the final pathname to retain that descriptor's inode. Temporary ownership and destructive operations retain their stricter identity checks. @@ -197,7 +363,10 @@ the fixed primary acquisition error or expose native diagnostics. New `media-claim-v2` records encode native bigint device/inode identities as bounded canonical unsigned 64-bit tokens. Claim creation, post-move admission and final recovery -revalidation compare these exact identities without a Number conversion. Existing v1 +revalidation compare these exact identities without a Number conversion. Node reports a +64-bit file ID with its top bit set as a negative bigint; its token is the unsigned value +(`BigInt.asUintN(64, …)`), so such a temporary can be claimed, and every other value keeps +its token. Existing v1 claims retain their safe-integer/canonical validation; an ambiguous unsafe historical token is retained, never promoted to deletion authority. Producer, maintenance and recovery-marker layouts are unchanged. This targeted identity check supplements the diff --git a/docs/reference/shared-core/diagnostic-secret-backstop.md b/docs/reference/shared-core/diagnostic-secret-backstop.md new file mode 100644 index 00000000..d4f40b31 --- /dev/null +++ b/docs/reference/shared-core/diagnostic-secret-backstop.md @@ -0,0 +1,117 @@ +# Diagnostic secret backstop — the finite shape list + +- **Status**: Accepted — the behavior `packages/llm/src/scrub-secrets.ts` implements +- **Canonical home**: the finite secret shapes `scrubSecrets` masks in provider diagnostics (`makeLlmError`'s `message` and `code`) and in the CLI's display sanitizer, and the documented limits of that list +- **Related**: [error-handling.md](../../standards/error-handling.md#diagnostic-secret-backstop) (the rule: a defense-in-depth backstop behind an adapter's own redaction), [built-in-tools.md](built-in-tools.md#finite-diagnostic-credential-masking) (core's journal and display projection, which differs from this list by design), [llm-provider-seam.md](llm-provider-seam.md) (the error model `makeLlmError` belongs to), [../../standards/security-review.md](../../standards/security-review.md) + +Moved here from [error-handling.md](../../standards/error-handling.md#diagnostic-secret-backstop) on +2026-11-04, unchanged: a PR #96 review found an exact specification in a standard, against the rule that one +lives only in its `docs/reference/` home. The standard keeps the rule and links here. + +- **Shapes.** OpenAI/Anthropic `sk-…` and Google `AIza…` key prefixes; a finite list of literal + secret query names, after `?`, `&` or `#` (an OAuth implicit-flow `#access_token=`); the exact + `api-key`, `x-api-key` and `x-goog-api-key` header names, also in quoted, JSON-encoded and + JSON-in-JSON (`\"x-api-key\":\"…\"`) objects; and `Bearer`/`Basic` authentication values. A header + value may begin with up to eight (possibly quoted, escaped, punctuation-glued or LF/TAB-split) + `Bearer`/`Basic` labels — a doubled `Bearer Bearer` is a common misconfiguration. Its token may + begin with any run of (escaped) quotes and ends at a delimiter, at escapes before one, or before a + nested header name, which then masks its own value. A label may be closed by its own quote before + its token (`"Bearer" `), and a run of up to eight concatenated labels (`BearerBearer`) + counts as one label. No label takes as its token another label that whitespace or the end of the + text follows, even one glued behind punctuation, quoted or concatenated, so a standalone value + with any number of separately stacked labels masks the token after the last one, and a label glued + to a token behind a non-word character masks its own token; a second label that other punctuation + follows (`Bearer Bearer;`) is masked as a token, which only over-masks. +- **Masking.** Every rule, the literal query names included, finds its matches in the original + text, and the masked ranges of all rules are merged before anything is replaced, so no rule + can erase the context another rule needs. A header, label or query rule masks only its token: + the header name, the separator, any labels and the query name stay visible, an LF/TAB-split + label keeps its spelling, and quotes around a token stay outside it. Masking a token that is + already exactly `[REDACTED]` changes nothing, so a masked token is a fixed point, quoted, + escaped or after labels. After a `Bearer`/`Basic` label, text glued to the marker + (`Bearer [REDACTED];tail`) is masked with it; a header token ends at `,`, `;` or `}`. One + pass can still glue a marker to the text after another rule's token, so the scrub repeats + until its output stops changing, at most three passes, and blanks the whole diagnostic if it + has not settled: a second scrub changes nothing. +- **URL userinfo.** The `?`/`#` rule decides where userinfo ends: a raw `?` or `#` before the + authority `@` begins public query or fragment text only when the text before it is host-shaped (a + dotted name or IPv4, `localhost` or a bracketed IP literal, with an optional port, at most 260 + characters in all) **and** an `=` assignment follows it, as in + `https://host.example?contact=a@b.example`. Masked userinfo that holds such a raw `?` or `#` + leaves the authority ambiguous: unless a host-shaped name follows its last `@`, the text there may + be the rest of a query value (`https://svc?password=P@ss`), so the mask runs through the + authority's end, the next `/`, whitespace, quote, `,` or `}`. The scheme rule then decides what is + masked: `http(s)`, `ws(s)` and `ftp(s)` mask any userinfo, while the connection-string schemes + `postgres`, `postgresql`, `mysql`, `mariadb`, `redis`, `rediss`, `mongodb`, `mongodb+srv`, `amqp`, + `amqps`, `ssh`, `sftp`, `ldap`, `ldaps`, `smtp` and `smtps` mask only colon-bearing userinfo + (`user:pass@`, `:pass@`), so a bare user such as `ssh://git@host` (or a colonless + `redis://#TOKEN@host`) stays visible. Userinfo whose password is exactly masked stays as it is: + beside any username under a connection-string scheme, or beside a masked or empty username under + any scheme; an empty password, or text glued after a masked one, is still masked. The + authored-content detector's projection of `postgres://app:{{secrets.password}}@db` to + `postgres://app:[REDACTED]@db` is therefore a fixed point. Core's journal projection applies the + same `?`/`#` predicate and also merges its rules' ranges, but masks a userinfo together with the + values its key and opaque-query rules recognize in it, its username's excepted, each to its own + end, and ends a value no rule recognizes at the last `@`, whose text after it stays as the host, + rather than masking through the authority's end + ([built-in tools](built-in-tools.md#finite-diagnostic-credential-masking)). +- **Split and normalized forms.** Every rule except the literal query names also probes the + terminal-normalized text; a credential revealed only by normalization masks the **whole** + diagnostic. The key prefixes, URL userinfo and the exact header names are additionally probed in + the LF/TAB-joined view at original-text boundaries — of the raw text too when terminal + normalization changed it, since a terminal sequence there may take a split credential's first + letter (ESC `A` before `AIza…`) — and a credential revealed only by such a split masks the whole + diagnostic too. A `Bearer`/`Basic` label may itself be split by LF/TAB, but its opaque value is not + joined: a value split by an LF or TAB is masked up to the separator and its tail stays visible + (documented 2026-11-04, after a PR #96 review found the module comment claiming otherwise). +- **Documented trade-offs, all failing closed.** A scheme glued to a word character is masked in + place; an unlisted scheme ending in a listed one (`gftp://`, `jdbc:mysql://`) matches by its + suffix; prose that the whitespace-stripped residual probe glues to a path-less URL and a later `@` + blanks the whole diagnostic, and so does a key prefix (`sk-`, `AIza`) that the same probe joins + with the following words to its minimum length, at the start of the text or, since the scrub + repeats, right after another rule's mask (`Authorization: Bearer xyz sk-ant api key invalid`); a + path-less terminal hyperlink target (OSC 8) that the raw LF/TAB-joined probe joins with a later + `@` (a hyperlinked table cell before an e-mail address) blanks it as well; a host fragment with no + assignment (`https://host.example#sec@x`), a single-label host query and an over-long host-shaped + prefix are masked, and so is the text after the last `@` of such an ambiguous authority unless it + is host-shaped; a `Bearer`/`Basic` label that ends a line takes the next token with it; an opaque + value after a label ends only at whitespace, so in compact JSON it masks to the end of the run; + nine or more concatenated labels, read in the raw text as well as the normalized one, and a token + that begins with `,`, `;` or `}` after a `Bearer`/`Basic` label in a header value, blank the whole + diagnostic. Past eight separate labels after a header name, and in a value of labels alone glued + behind punctuation (`Bearer |Bearer`), the outcome depends on what precedes the header name: when + its nearest non-whitespace character is a word character, `-` or `_`, the diagnostic stays in + place (the last label masks any token); otherwise, the start of the text included, it is blanked. + No token is visible either way, and the output is stable under a second scrub. +- **Limits.** Userinfo under an unlisted scheme is not masked, including driver-qualified spellings + such as `postgresql+psycopg2://`. Opaque `Bearer`/`Basic` values, `api-key` header values and + literal secret query values end at any whitespace, LF and TAB included, so a fragment after such a + break stays visible, as it does after an ordinary space; a label glued to a quoted token without + whitespace (`Bearer""`) is likewise outside the shape, as is a token after a JSON escape + (`\n`, `\t`) that stands for the whitespace after a label. A JSON escape right after a header + name's colon starts its token, so a value glued to the escape is masked with the escape's letter + (`"x-api-key":"\n"` → `"x-api-key":"\[REDACTED]"`), while a quoted value behind such an escape + stays visible (`x-api-key:\t""`). A terminal sequence with a parameter whose final byte is a + shape's first letter (ESC `[` `1` `B` before `earer `), and a terminated device-control, + start-of-string or application-program string (ESC `P`, ESC `X` or ESC `_`, ended by ST or BEL) + glued before a shape, hide that shape from every view: in the raw text the sequence's last byte + glues to the shape's first letter, and normalization removes it. A word glued before `sftp://` on + the same line (`statusftp://user@h`) reads as `sftp`, so its colonless userinfo stays visible like + any `sftp` user: every whitespace-joined probe, the CLI's display residual among them, glues prose + to a scheme the same way, and reading it as `ftp` would blank an ordinary + `connect to sftp://user@host` diagnostic. (A word that ends a line before `ftp://` is no such + glue: an LF/TAB-split ftp credential after it blanks the diagnostic.) Malformed URLs + (`https:/u:p@h`), a raw `/` inside a password, invisible characters inside a scheme and JSON whose + quotes are themselves escaped as `\u0022` are outside the finite shapes. In a JSON-encoded + diagnostic, an escape written as two characters (`\n`, `\t`, `\r`) and glued before a key prefix, + a header name or a label defeats that shape's word boundary, so the credential after it stays + visible (`"key rejected:\nsk-…"`). Core's journal projection shares this limit for key prefixes + and scheme labels, while core's summary also reads each such escape as its separator and masks the + credential after it; core's key/value rule's word-character prefix still reads a header or + secret-key name glued after such an escape (`"err:\napi-key: …"`) and masks the value with it. A header value + that begins with `,`, `;` or `}` and has no label before it is an empty token, so the text after + it stays visible. The backstop has no generic key/value rule: a secret key inside a URL authority + whose value holds `@` — after `;` (`https://svc;a=1;password=P@ss…`), after `&` unless it is one + of the literal secret query names, or on the next line of a JSON-encoded file whose break is the + escape `\n` — leaves its value's text after the last `@` visible as the host. Core's key/value + rule reads that key and masks its value. diff --git a/docs/reference/shared-core/effect-journal.md b/docs/reference/shared-core/effect-journal.md index f1c3a8db..474db201 100644 --- a/docs/reference/shared-core/effect-journal.md +++ b/docs/reference/shared-core/effect-journal.md @@ -369,12 +369,52 @@ no fallback to the old digest or automatic retry permission. Secret and inline-media redaction remain deliberate projection equality; raw credential bytes are never added to the digest to distinguish calls. -Recognized URL userinfo is masked through the authority's last `@`, preserving the host, -path, query and fragment. Recognized opaque URL-query values are masked through their delimiter, including values -longer than 512 characters. Quoted secret-key values honor escaped quotes and backslashes -through the actual closing quote. A recognized value must not leave a credential suffix -in the projection. This remains a finite shape detector, not a claim to identify arbitrary -unlabelled secrets. Existing old digests are not retroactively erased or rewritten. +Recognized URL userinfo is masked through the authority's last `@`, preserving the host, path, query +and fragment; the values the key and opaque-query rules recognize in it, its username's excepted, +are masked to their own ends as well, through that `@` and on through the host, path and query when +no delimiter of theirs comes first, while a value no rule recognizes ends at that `@` and the text +after it stays as the host +([built-in tools](built-in-tools.md#finite-diagnostic-credential-masking)). Recognized opaque +URL-query values are masked through their delimiter, including values longer than 512 characters. +Quoted secret-key values honor escaped quotes and backslashes through the actual closing quote on +their line. A recognized value must not leave a credential suffix in the projection; what a rule +does not recognize whole, such as the later words of a quoted value that does not close on its line, +is listed with the shapes +([built-in tools](built-in-tools.md#finite-diagnostic-credential-masking)). This remains a finite +shape detector, not a claim to identify arbitrary unlabelled secrets. Existing old digests are not +retroactively erased or rewritten. + +A raw `?` or `#` before the authority's `@` is public only after a host-shaped prefix that an `=` +assignment follows ([built-in tools](built-in-tools.md#finite-diagnostic-credential-masking)). Wave +3's PR95 increment tightened that predicate without a new digest domain. It also merges the ranges +of every rule before replacing anything, so a userinfo is masked together with the recognized values +of the keys and query parameters in it but its username, which never reaches past an LF or TAB, and +no mask removes another credential's context; reads a key or scheme label inside another value as a +credential of its own, and a value through earlier masks; masks userinfo split by an LF or TAB at +either edge or inside its `scheme://`; reads a quoted value one JSON string down too, and closes a +quoted value at the latest of its readings, with earlier masks keeping their trailing backslashes and, +once a reading closes, with the scheme labels' tokens filled; reads one that does not close on its +line, or before the end of its string, as an unquoted run that stops there — unless another mask of +the pass takes its line break and it closes, as the next pass read it before (at the latest of its +readings, with and without the key rule's own masks and the masks' trailing backslashes) — so the +text after a quote that may close a string around the key keeps calls distinct; reads a +split header label or a scheme label right after a credential or a PEM block, and a header label at +a line start inside a split continuation; and repeats the projection until it is stable. These +change a `v5:` projection only for the arguments they project differently, such as userinfo +previously treated as public or a credential an earlier mask had hidden: a resumed run whose +committed effect carried one now computes a different digest and refuses matching prepare instead of +replaying, which is the fail-closed outcome of any digest difference. Every other `v5:` row keeps +its digest. Text a rule reads as part of a credential's value — such as the host, path and query a +password-position key's value runs through — is that credential's, so calls that differ only there +share a projection, as recognized credentials always do. A projection that has not settled after +three passes becomes the constant `[redacted]` in diagnostics, but it never becomes a journal +identity: the registry refuses such a tiered call with `ToolArgsInvalidError` before its prepare +([tool registry](tool-registry.md#resolution--the-dispatch-lifecycle)), so two different calls +cannot share it and replay one another's results. Two member names of one object that project alike +would leave one member for two argument sets, so such a projection counts as not settled and is +refused the same way. Like any refusal before the prepare, it does not consult a row an earlier +version committed at that slot; the corrected call takes the next slot, as +[§14](#14-known-limitations)'s slot instability already allows ([§4](#4-the-resume-gate)). A recognized structural credential key masks its entire value before traversal. A partial inner text match does not make the remaining value public. Only an anchored Bearer, Basic @@ -382,7 +422,11 @@ or Token scheme survives as its canonical label followed by `[redacted]`; no ori credential remainder or casing is retained. Credential flags in an argument vector mask the whole following element or the complete value in `--token=value` form. Only the ambiguous `-p` spelling retains the existing port-shaped exception; explicitly named -credential flags have none. Unrecognized public fields and ordinary arguments still +credential flags have none. Since 2026-11-04 the credential flags include every flag whose last +word names a credential and curl's `-u`/`--user` pair, and a URL password's `,` or `/` no longer +ends its userinfo ([built-in tools](built-in-tools.md#finite-diagnostic-credential-masking)); as +with the PR95 tightening above, a `v5:` row whose arguments these project differently now refuses +matching prepare, and every other row keeps its digest. Unrecognized public fields and ordinary arguments still contribute their complete surviving values to identity. These projections do not change the arguments dispatched to the host. diff --git a/docs/reference/shared-core/tool-registry.md b/docs/reference/shared-core/tool-registry.md index b00a6230..c3c24049 100644 --- a/docs/reference/shared-core/tool-registry.md +++ b/docs/reference/shared-core/tool-registry.md @@ -256,7 +256,13 @@ host is touched once, in the middle. 8. **Mark the result untrusted** (§Untrusted-data taint) and hand the structured `tool_call` / `tool_result` data + its taint/secret markers to the bus's single translation point ([ADR-0036](../../decisions/0036-run-loop-substrate-event-bus-and-execution-host.md)) for `agent:tool_call` / `agent:tool_result` emission. The durable [effect bracket](effect-journal.md) prepares a tiered call after approval and settles it -after model-facing bounding. Immediately before step 5, after any asynchronous approval and preparation, +after model-facing bounding. Its prepare carries the journal projection of the effective arguments; +when that projection has not settled within three passes (credential-shaped text with no stable +redacted form, [built-in-tools.md](built-in-tools.md#finite-diagnostic-credential-masking)), or when two +member names of one object project alike and would leave one member for two argument sets, the call +is refused with a loop-correctable `ToolArgsInvalidError` before anything is prepared or dispatched. It +names the affected arguments through the same masker, since an open schema lets the model choose an +argument's name. Immediately before step 5, after any asynchronous approval and preparation, the registry invokes the optional trusted `ctx.beforeDispatch(toolId)` synchronously for **every actual dispatch**, including unjournaled tools. No asynchronous work may occur in that hook. It uses the resolved registry id and does not run for a retained-result replay. A refusal proves no dispatch started: a prepared @@ -412,6 +418,12 @@ distinct from the *event* `outputSummary`, which the registry bounding stage pro The ceiling is a **byte/line** bound (no token count — that needs a provider-specific tokenizer, which would break engine purity); concrete defaults and the marker shape are canonical in [built-in-tools.md](built-in-tools.md#tool-result-bounding). `ToolResultLimits` carries them into `ctx`. +The bound counts the inline media a result still holds — a base64 `data:` URI, a canonical base64 +source, a binary buffer — at its full size, although the summary, spill and preview show only a +marker for it: a builtin's media rides the attachment rail ([ADR-0089](../../decisions/0089-media-correctness-four-boundaries.md) §1), +so media left in a dispatch result is text to the model and is settled into the journal. Over the +ceiling, the model gets the media-free preview and the marker reports the full byte count (2026-11-04; +a 30 MiB data URI had been measured as its marker). Observability and journal-argument projections distinguish shared siblings from ancestor cycles. Each walk removes a visited object when its branch finishes, including native/binary/media early @@ -480,7 +492,7 @@ codes by [sse-event-schema.md](../contracts/sse-event-schema.md#error-code-taxon | `UnknownToolError` | id not an exact match | `tool_failed` | fatal (loop-correctable first) | | `ToolPolicyError` | a guardrail / grant denial — `not_granted`, `provider_executed`, `command_not_allowed`, `domain_not_allowed`, `insecure_url`, `gate_required`, `media_scope_denied` (the full `ToolPolicyDenyReason` union; `media_scope_denied` is `read_media`'s scope-set denial, [ADR-0044](../../decisions/0044-media-access-governance-read-media-save-to-cost.md) §1) | `tool_denied` | **fatal** — EXCEPT `media_scope_denied` is `recoverable` (a Step-14 SCOPE denial; see the note below) | | `ToolDeniedByUserError` | an interactive **per-tool approval** denial ([ADR-0057](../../decisions/0057-cli-chat-modes-and-per-tool-approval.md) EA3) — `user_rejected` (rejected by the user / mode policy), `no_approval_hook` (fail-closed: a governed dispatch under an active regime with no confirm hook wired), `approval_error` (fail-closed: the hook threw a non-abort error, so consent could not be obtained) | `tool_denied` | **fatal** (never retried; not loop-correctable — re-asking re-prompts/re-denies, like `ToolPolicyError`) | -| `ToolArgsInvalidError` | effective args fail `parseArgs` / secret-taint | `validation` | fatal (loop-correctable first) | +| `ToolArgsInvalidError` | effective args fail `parseArgs` / secret-taint, or a tiered call's journal projection does not settle or projects two member names alike | `validation` | fatal (loop-correctable first) | | `ToolUnavailableError` | the required `ToolHost` capability is absent (host/config gap, not the model's fault) | `tool_unavailable` | **fatal** (names the tool + the unwired arm actionably — never a bare `internal`; EA1, [ADR-0055](../../decisions/0055-cli-host-capability-seam-tool-environment-factory.md)) | | `ToolExecutionError` | the host capability threw a non-cancel error (cause kept off the message, for logs) | `tool_failed` | retryable (node budget) | | *(AbortSignal abort)* | the run was cancelled mid-tool | `cancelled` | fatal (cancel path, not `tool_failed`) | diff --git a/docs/roadmap/README.md b/docs/roadmap/README.md index c5cb6698..2fa4ddbf 100644 --- a/docs/roadmap/README.md +++ b/docs/roadmap/README.md @@ -149,6 +149,7 @@ to that phase's work breakdown). | **M1 ✅** | **LLM seam proven** *(achieved 2026-06-07, PR #9)* — `@relavium/llm`'s provider-agnostic `LLMProvider` seam with 3 adapters (Anthropic, the OpenAI-compatible adapter serving OpenAI + DeepSeek, Gemini), all passing one shared conformance suite (fixtures on PR, live nightly reserved/pending keys); no vendor SDK type crosses the seam. The `FallbackChain` runner (1.K, PR #13) and the cost tracker (1.B) have since landed, completing 1.m2. (Built **first** per the engine-first order — the engine's `AgentRunner` depends on it.) | 1 | Phase 1 · `@relavium/llm` seam + 3 adapters — conformance green at **1.J** (1.G–1.J; FallbackChain + cost tracker at 1.m2 / 1.B, 1.K) | | **M2 ✅** | **Engine end-to-end (critical-path milestone)** *(achieved 2026-06-16, PR #27)* — `@relavium/core` parses YAML→DAG and runs it (the `WorkflowEngine` + `AgentRunner` over all node types, emitting canonical events through the `RunEventBus`, with checkpoint/resume, retry, the deterministic expression sandbox, and the pre-egress budget governor), with the M1 seam wired in; the 1.U Node harness runs a workflow end-to-end with live streaming, checkpoint + cross-process resume, retry, and a provider fallback. Zero platform-specific imports. | 1 | Phase 1 · `@relavium/core` parser + runner + checkpoint/resume + retry + **expression sandbox (1.AB) + budget governor (1.AC)** + Node harness (1.L–1.U, all landed); the agent-first sub-spine (1.V–1.AA) lands in parallel (1.m5) | | **M3 ✅** | **CLI + engine regression harness** *(achieved 2026-06-23, PR #43)* — the `relavium` CLI drives the engine end-to-end with a live `ink` TUI and a deterministic `--json` CI mode; a small fixture suite runs in CI on every engine change as the agreed regression gate for Phases 2.5–6. (Phase 2 shipped complete as **v0.1.1**, PR #58; the CLI consolidation/authoring work continues in Phases 2.5–2.6 before M4.) | 1 | Phase 2 · 2.D (run wiring), 2.F (`--json` / CI mode), 2.K (regression harness) | +| **M3.5** | **CLI MVP — `relavium` v0.2.0 on npm** *(set 2026-11-04, decision D58)* — a new user installs the CLI, onboards with any of the four providers, chats with an agent that reads, searches, edits and runs commands under approval with diffs, authors workflows conversationally, runs them and sees their outputs, manages runs, sessions, providers and MCP servers from the Home, and reaches the web under the chat egress authority. Ordered as MVP-1…MVP-10 in [current.md](current.md#next-work--the-cli-mvp); it closes 2.6.F's release clause and M2.6-1, and takes slices of Phase 2.6 and the paused Wave 3 of Phase 2.5.5 without closing either phase. | 1 | Phase 2.6 · slices of 2.6.A/B/D/E/G/H/I/J/K/M/Q + the [CLI MVP additions](phases/phase-2.6-conversational-authoring.md#cli-mvp-additions--2026-11-04); Phase 2.5.5 · recorded Wave 3 splits | | **M2.5.5** | **Hardening & remediation review closed** — the 373-finding 2026-07-19 full-project review's verified defects against the Phase 0–2.5 codebase (and the merged 2.6.C/2.6.F slices) are resolved or explicitly re-deferred; the three CRITICAL findings (an unredacted tool-approval preview, a non-atomic chat-turn persister write, unsanitized `relavium run` terminal output) are closed. Runs **concurrently with Phase 2.6** — not a gate on M4/Phase 3, and not itself gated by any 2.6 workstream. | 1 | Phase 2.5.5 · 2.5.5.A–2.5.5.I | | **M4** | **Desktop agent-management center** — a signed, offline, no-account Tauri v2 app: ReactFlow canvas with all node types, live execution theater, gate overlay, local SQLite history (trace/Gantt/replay/retry-from-node), OS keychain; ADR-0010 perf gate + per-platform e2e green. | 1 | Phase 3 · 3.K (packaging), 3.L (e2e + perf gate), 3.P (sign-off) | | **M5** | **Product Phase 1 complete** — the standalone VS Code extension is shipped to the Marketplace: right-click run, live monitoring, human-gate webview, **no desktop app required**, same engine + events + SQLite history as the other surfaces. | 1 | Phase 4 · 4.E (right-click run), 4.H (gate webview), 4.K (Marketplace publish) | diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 4e3cec4d..083e9fd4 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -1,1198 +1,220 @@ # Current state > Status: Living -> Last updated: 2026-10-18 +> Last updated: 2026-11-04 -- **Related**: [README.md](README.md), [phases/phase-2.5-cli-consolidation.md](phases/phase-2.5-cli-consolidation.md), [phases/phase-2.5.5-hardening-and-remediation.md](phases/phase-2.5.5-hardening-and-remediation.md), [phases/phase-2-cli.md](phases/phase-2-cli.md), [deferred-tasks.md](deferred-tasks.md), [../project-structure.md](../project-structure.md), [../tech-stack.md](../tech-stack.md) +- **Related**: [README.md](README.md), [phases/phase-2.6-conversational-authoring.md](phases/phase-2.6-conversational-authoring.md), [phases/phase-2.5.5-hardening-and-remediation.md](phases/phase-2.5.5-hardening-and-remediation.md), [phases/phase-2.6.5-core-reliability-remediation.md](phases/phase-2.6.5-core-reliability-remediation.md), [deferred-tasks.md](deferred-tasks.md), [../project-structure.md](../project-structure.md), [../tech-stack.md](../tech-stack.md) > **Authority (until Phase 2.5.5 and Phase 2.6 close):** This file is canonical for > live progress and execution order; the phase documents stay canonical for scope and acceptance. Ordering > differences between the two are deliberate. `current.md` may override recommended ordering — it may **not** > redefine a security boundary, a scope line, or a milestone acceptance criterion. -This page tracks what is active **right now** and the immediate next concrete actions. -The full phase plan and the global milestone spine are in [README.md](README.md). -**Phase 2.5 (CLI Consolidation) is complete** (milestone **M2.5-4**, PR #69, 2026-07-08) — its -breakdown, now historical, is in -[phases/phase-2.5-cli-consolidation.md](phases/phase-2.5-cli-consolidation.md). -A 2026-07-19 full-project code review then opened **Phase 2.5.5 (Hardening and Remediation)**, an -interstitial phase holding the review's verified findings against the pre-2.6 codebase; it runs -**concurrently with Phase 2.6, not before it** — see -[phases/phase-2.5.5-hardening-and-remediation.md](phases/phase-2.5.5-hardening-and-remediation.md) -and [Active now](#what-is-active-now). -**Phase 2.6 — Conversational Authoring and the First-Class CLI** is **in progress** (re-scoped -2026-07-08); its plan is in -[phases/phase-2.6-conversational-authoring.md](phases/phase-2.6-conversational-authoring.md). -Workstream **2.6.F (platform floor + the full-screen TUI renderer)** is **merged to `main`** -(PR #74, 2026-07-11), and so is **2.6.C** (the reseat transcript-carry + the `/cost` per-model breakdown, -PR #75, 2026-07-13) — see [Active now](#what-is-active-now). **2.6.Q** (dynamic model-catalog -enrichment) is **no longer blocked**: [ADR-0071](../decisions/0071-models-dev-as-the-model-metadata-source.md) -and [ADR-0072](../decisions/0072-model-metadata-in-the-db-behind-a-generated-offline-floor.md) are both -**Accepted**, resolving the six open maintainer questions the phase file had cited, and the P1–P5 -implementation steps are **merged to `main`** (PR #76, 2026-07-26). P6 and the `/settings` → `/models` -visibility toggle remain open; the bundled offline snapshot was **regenerated** in PR #79 after the D3 -price ruling. - -**The unified build order across both phases is in -[Execution order — Phase 2.5.5 + Phase 2.6](#execution-order--phase-255--phase-26-temporary) below** — a -temporary section, deleted when both phases close. Its baseline step is discharged (PRs #76 and #77 merged) -and Wave 0 is merged (PR #82), as is Wave 1 (PR #83). The interlude's `W2` (core reliability, six items) is -merged too (PR #85, 2026-08-28). - - -> **W7 merged to `main`, 2026-10-10 (PR #90, `46a6a701`):** all twelve steps, five genuine overflow records, -> production request ownership and consolidated host lifetimes have independent acceptance. -> All six W7 items are closed; Phase 2.6.5 stood at **47/51 at that merge**. The -> [closing register](phases/phase-2.6.5-core-reliability-remediation.md#w7-closing-register--2026-10-10) -> joins per-item causal controls, both final review rounds and every submitted systematic finding. -> PR #90 merged at 13:57:47 UTC. W8 was still open at that checkpoint; it subsequently merged in PR #92. Approved deferrals remain. -> No implementation, user decision, credential, billing or provider capture is pending for W7. - -> **Post-closure corrections accepted, 2026-10-10:** the [new correction register](phases/phase-2.6.5-core-reliability-remediation.md#post-closure-systematic-review--2026-10-10) -> joins every new report disposition. Core, CLI/DB and tooling/docs each have two NEW complete clean -> cumulative review rounds. Fresh unchanged-production coverage passes 447 files / 9,512 cases / -> eleven existing capability/key-gated LLM skips. Published-head checks are reported in PR #90. -> W7 has no implementation, provider, credential, billing or user-decision prerequisite left. -> The corrections merged with PR #90; W8 subsequently merged in PR #92. Existing approved limits remain. - -> **Publication tool corrections accepted, 2026-10-10:** [two NEW complete clean cumulative rounds](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-10T13-35-14-w7-publication-sonar-round-2-review.md) -> independently verify all four Sonar-label dispositions, preserved UTF-16 ordering and OS-owned Git -> dispatch with PATH-trap causal controls. Shipping production/tests remain unchanged. Final local CI -> and exact published-head GitHub/Sonar checks are reported in PR #90; the prior failed analysis stays recorded. -> W7 has no implementation, capture, credential, billing or user-decision prerequisite left. - -## W8 merged — Phase 2.6.5 complete - -**W8 merged to `main` on 2026-10-11 as [PR #92](https://github.com/HodeTech/Relavium/pull/92), -merge commit `ef0ffa54`. Phase 2.6.5 is complete with approved deferrals: 49/51 items closed.** -The [merge checkpoint](phases/phase-2.6.5-core-reliability-remediation.md#w8-merge-checkpoint--2026-10-11) -joins final publication evidence and all seven phase exit criteria. All seven W8 steps, -CR-81, CR-82 and ADR-0087 §1 are complete on `main`; no W8 implementation, review, -publication, provider, billing, credential or user-decision gate remains. - -1. **Consumption:** explicit run/session modes preserve iterated delivery from construction; - subscribe-only handles retain no primary buffer and reject iteration with a typed error. -2. **Conformance:** strict expected exchanges and the 24-case actual installed-SDK HTTP - matrix detect production-lowering regressions. -3. **Usage uncertainty:** complete/partial/missing observations, pending ACK before invocation, - once-only settlement and late receipts, sticky old-writer uncertainty and qualified - human/JSON totals preserve the distinction between unknown usage and actual zero. - -The [original closing register](phases/phase-2.6.5-core-reliability-remediation.md#w8-closing-register--2026-10-11) -preserves its source acceptance and two NEW complete whole-wave reviews. The -[maintainer correction register](phases/phase-2.6.5-core-reliability-remediation.md#w8-pr-92-systematic-review-corrections--2026-10-11) -joins all report dispositions and two NEW independent complete clean cumulative rounds per -correction increment. Final reviewed head `3d4491df` passed both seven-job PR/push CI -workflows and the SonarCloud quality gate. Published ordinary/enforced-coverage inventories -are 511 files / 11,026 passing tests / 12 established skips; four wire and 23 usage -mechanisms retain 427 permanent cases and 69 joined native children. Fresh full local -CI and bare/enforced coverage at `6139563e` remain separately attributed evidence. -No merge-commit execution, live-provider invoice or unsupported-platform result is inferred. -Two nonblocking Sonar complexity suggestions remain visible. Detailed review records stay -local and ignored; historical citations remain commit-pinned. - -**What remains:** CR-70 (cross-turn tool-history carrying) and CR-93 (tenant-scoped mutable -state) are approved deferrals. The 49/51 count includes CR-95's short-term fail-closed fix; -its long continuation and existing exact-capacity/scoped residuals remain in -[deferred-tasks.md](deferred-tasks.md), with their existing severity, trigger and narrowed -claim. ADR-0087 §1 is outside the denominator. Phase closure does not claim 51 fully -implemented items or an empty debt backlog. Phase 2.5.5 and Phase 2.6 remain in progress. -Earlier dated pending checkpoints in this page and the phase register are historical. - - - - - -## Wave 2 merged — Shut the doors - -**Wave 2 merged to `main` on 2026-10-15 as [PR #94](https://github.com/HodeTech/Relavium/pull/94), -merge commit `ab08209e`. All eight steps and the maintainer corrections are complete.** -The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) -joins the forty item dispositions, four new security sittings plus W4 revalidation, -original whole-wave acceptance and all subsequent correction scopes. Existing incoming -navigation anchors are retained; dated implementation checkpoints remain historical. - -Completed work covers invocation-authorized filesystem boundaries and scratch-policy -composition; bounded secret stdin, early admission and private bridge diagnostics; -faithful-snapshot warnings, structural session privacy and export completion; verified -bounded media/CAS/native lifecycles; and the interim provider-header gate. Maintainer -corrections include runner-level error rejection, valid Opus/Vorbis publication, -FLAC structural consistency, generated-result streaming and finite pin-error provenance. - -Final reviewed correction source `4cdfe011` has two NEW independent complete clean -cumulative rounds covering all 67 correction paths / 188 hunks / 24 complete new files. -These scoped rounds do not claim a new review of every original PR path. Full local -CI, ordinary/enforced coverage and all nine jobs in each PR/push CI workflow pass; -SonarCloud and selected native Windows/Node-floor controls pass. Local coverage records -588 files / 12,983 passing tests / 20 explicit skips; CI records 12,982 / 21 with its -established extra CI-only skip. Complete physical controls are independently inspected. -The merge tree matches the reviewed source; no new merge-commit execution is inferred. - -**Satisfied on `main`:** Phase 2.5.5 exit criteria 1–3 and sub-stream D acceptance. -**Still open:** M2.5.5-2's I half, whole Phase 2.5.5/2.6, M/N production lifecycle, -future live headers and existing approved residuals. Wave 2 has no remaining implementation, -review, publication, ADR approval, provider capture, credential, billing or user-decision gate. -The [deferred register](deferred-tasks.md) retains its existing limits; closure adds no deferral. - -## Next work — Wave 3 — Clear the ground - -Wave 3 implementation is in progress on `development`; Steps 1–8 have scoped -independent acceptance. The initial publication checkpoint is -[PR #95](https://github.com/HodeTech/Relavium/pull/95). A second maintainer systematic -review at `a5a5d45e` reopened corrective work; its -[current correction register](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-second-systematic-review-corrections--2026-10-18) -owns dispositions and remaining gates. Corrective source reviews are complete; -the two later native test-fixture increments have [separate scoped acceptance](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-supplemental-native-fixture-source-acceptance--2026-10-18). -Final pushed-head publication checks remain required, with their live results on -PR #95. Steps 9–19 have not started; check the PR status before extending its scope -with Step 9. See the [Wave 3 queue](#wave-3--clear-the-ground). - -## Historical W7 implementation checkpoints - -Dated W7 checkpoints throughout this page retain their original pending states. The closing checkpoint above -and its later post-closure and merge checkpoints control current W7 status; historical pending/draft -wording describes the checkpoint when it was written. - -> **Prior complete composition review:** [host-closure round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-10T00-38-24-w7-host-closure-round-5-review.md) -> finds no new confirmed actionable issue across all 126 paths / 364 hunks. This is the first -> clean complete cumulative round; one further fresh complete clean round remains required. -> Step 8 and final Step 12 remain open, with no user/provider blocker. -> -> **Complete composition review:** [host-closure round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-10T00-22-00-w7-host-closure-round-4-review.md) -> confirms public stream exit can skip underlying cleanup on retirement failure. Parent adds -> twelve paired permanent cases and guarantees cleanup while preserving the original throw. -> Two fresh complete clean rounds, Step 8 and final Step 12 remain required; no user input is pending. -> -> **Complete composition review:** [host-closure round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T23-56-41-w7-host-closure-round-3-review.md) -> confirms two retirement-boundary Highs: mutable returned output/usage and failed cleanup -> suppressing accounting/acknowledgement. Parent corrections add thirty-two permanent cases. -> Fresh complete review remains required; Step 8/12 remain open, with no user/provider blocker. -> -> **Complete composition review:** [host-closure round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T23-34-00-w7-host-closure-round-2-review.md) -> confirms due-budget departure starvation and provider entry after synchronous cancellation. -> Parent corrections add nineteen permanent cases and fix adjacent canonical-documentation drift. -> Fresh complete review is required; Step 8/12 remain open. No user/provider blocker exists. -> -> **Complete composition review:** [host-closure round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T23-05-00-w7-host-closure-round-1-review.md) -> confirms three lifecycle High findings (post-pause receipt cursor, rejected activation joining, -> post-unmount Ink visibility) and a required shipping-registry acceptance gap. Parent corrections -> add eighteen permanent cases and strengthen native output assertions. Fresh complete review of -> the corrected composition remains required before acceptance; Step 8/12 are still open. - -> **W7 live status, 2026-10-09:** all five immutable provider records and Step 7's four-round -> acceptance remain complete. [Two fresh adapter/chain round-3 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T23-47-31-w7-owned-adapter-chain-round-3-review.md) -> accept all 24 cumulative paths/78 hunks through `0c3c185b`; exact-head required CI, coverage, -> Node 22, Windows, peers and Sonar pass. Core measured-round reuse is implemented at `99e78e55`; -> [both independent round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T00-16-47-w7-owned-core-round-1-review.md) -> clear all 13 paths/47 hunks. [Two fresh round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T01-00-00-w7-owned-core-round-2-review.md) -> accept all 19 cumulative paths/53 hunks through `f1281214`; exact-head CI passes. ADR-0103 -> append/receipt integration is implemented at `a35739a4`; -> [both complete round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T01-27-00-w7-append-receipt-round-1-review.md) clear all 17 paths/57 hunks. -> [Two fresh cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T02-00-00-w7-append-receipt-round-2-review.md) accept all 23 paths/63 hunks -> through `01115050`. Raw provider/iterator/media and native child lifetime integration is -> implemented at `fde3ca0c`; [both complete round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T02-36-10-w7-raw-producer-round-1-review.md) -> verify a public-return High, corrected with two regressions (28 permanent cases total). -> [Two fresh cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T03-25-00-w7-raw-producer-round-2-review.md) accept all 20 paths/41 hunks -> through `1172d247` after complete report/artifact/freeze audits; nested Corepack reviewer -> violations get no proof credit. The HTTP descendant increment now has 23 new permanent -> cases, 16 compiled negative controls and a passing cache-qualified root CI; -> [both complete round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T04-16-24-w7-http-descendants-round-1-review.md) -> clear all 22 paths/38 hunks through `db02b83c` after Parent's complete artifact/freeze audits. -> [Two fresh cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T04-55-00-w7-http-descendants-round-2-review.md) -> accept all 24 paths/40 hunks through `6e620556` after Parent's complete evidence and 645,606-entry freeze audits. -> Invocation-local provider/raw-poll integration adds 57 permanent cases; [complete round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T05-43-00-w7-provider-invocation-round-1-review.md) -> verify a method-receiver defect and separately strengthen a weak Promise test. -> [Both complete round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T06-33-00-w7-provider-invocation-round-2-review.md) verify one getter-acquisition defect. -> Its correction passes root CI (404 files / 9,030 cases / 12 existing skips); -> [both NEW complete cumulative round-3 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T07-06-28-w7-provider-invocation-round-3-review.md) -> and Parent's complete artifact/645,743-entry freeze audits accept all 28 paths/67 hunks through -> `f9a8ff37`, with no new confirmed finding. Exact-head required CI, coverage and Sonar pass. -> MCP transport/handler descendants now have 77 new permanent cases. Rounds 1–3 corrected -> queued-reservation, deadline, disposer, coverage, empty-start stdio and callback failures. -> [Complete round-4 reviews and Parent evidence/freeze audits](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T12-58-01-w7-mcp-lifetime-round-4-review.md) verify -> completed control-lane reference retention. Correction `37220a95` releases only fully -> acknowledged lanes and passes actual root CI; seven new causal regressions and fresh enforced -> private coverage pass (412 files / 9,107 cases / 12 skips; MCP branches 90.68%). -> [Two NEW complete cumulative round-5 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T13-37-00-w7-mcp-lifetime-round-5-review.md) -> and Parent's complete source/artifact audits accept all 38 paths/70 hunks through `99c46bbd`, -> with no new confirmed finding and 268 independently passing cases per reviewer. The full -> shared-tree audit has zero file/link changes and one qualified `.git` directory mtime change. -> Fresh startup registers its complete continuation. [Two complete first-round reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T14-28-09-w7-startup-lifetime-round-1-review.md) -> and Parent audits verify reentrant clock/timer cancellation; three permanent regressions -> cover the correction. [Two NEW complete cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T14-55-12-w7-startup-lifetime-round-2-review.md) -> and Parent's exact-source/physical-artifact/647,041-entry freeze audits accept all ten -> paths/sixteen hunks through `5d44b13b`, with zero new confirmed findings and 2,850 core cases per reviewer. -> [Both complete scheduler readiness round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T15-44-15-w7-scheduler-readiness-round-1-review.md) -> verify a secondary readiness-diagnostic failure. Its correction has twelve causal readiness -> regressions and 2,862 passing private core cases. [Two NEW complete cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T16-14-29-w7-scheduler-readiness-round-2-review.md) -> and Parent's source/artifact/647,245-entry unchanged freeze audit accept all nine paths/ten -> hunks through `95f3923a`, with zero new readiness findings. Both inherited dispatch Highs -> (post-start-append cancellation admission and unsafe detached failure formatting) are -> corrected in `1918690d`, with six new permanent cases and passing exact-input root CI. -> [Both complete dispatch-correction round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T17-28-06-w7-dispatch-corrections-round-1-review.md) -> and Parent's artifact/647,355-entry unchanged freeze audits accept the eight-path/ -> seventeen-hunk scope with zero confirmed findings. [Both complete cumulative second-round reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T17-58-41-w7-dispatch-corrections-round-2-review.md) -> verify an inherited retry-start cancellation High. Its correction and a subsequent -> Parent-discovered synchronous attempt-entry correction add ten permanent cases; -> final-source causal controls pass. NEW complete cumulative reviews are required -> before scoped closure; complete scheduler/dispatch roots stay open. -> [Complete cumulative round-3 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T18-37-30-w7-dispatch-corrections-round-3-review.md) verify a retry/deadline duplicate-terminal High. -> Parent confirms and corrects it, with a new permanent causal regression; fresh cumulative review remains required. -> [Complete cumulative round-4 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T19-10-39-w7-dispatch-corrections-round-4-review.md) verify two paths of one attempt-correlation High. -> Parent corrects explicit failure forwarding and first-start reset with five permanent cases; NEW cumulative review is required. -> [Complete cumulative round-5 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T19-41-03-w7-dispatch-corrections-round-5-review.md) verify a pre-start readiness attempt-correlation High. -> Parent corrects its latest-entered-attempt forwarding with two paired permanent cases; NEW cumulative review is required. -> [Complete cumulative round 6](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T20-08-19-w7-dispatch-corrections-round-6-review.md) finds no new confirmed issue. -> [The NEW complete round 7](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T20-29-02-w7-dispatch-corrections-round-7-review.md) verifies a differing-order grace High: -> an unstarted claim can receive a failure. Parent corrects explicit claim/start tracking, -> with three permanent causal cases; NEW complete cumulative reviews remain required. -> A broader private actor prototype exposed CLI closure/lease-handoff failures in root CI. -> Complete scheduler/dispatch/resume registration must land with shipping host departure; -> original failures and assertions are retained, and that integration stays required in W7. -> Complete MCP startup/all-actor retirement, custom-provider descendants and remaining host departure work -> and Step 8 atomic compaction/recovery remain open. The ownership/lifecycle -> Highs, Step 12, final whole-wave CI/coverage/Sonar and all six W7 register items remain open. -> Draft PR #90 is unmerged; no further paid call, credential or ADR approval is needed. -> Earlier progress statements below describe their publication checkpoints. - -> **W7 implementation, 2026-10-02:** the documentation review is closed. The maintainer approved -> [ADR-0099](../decisions/0099-compaction-has-an-idle-budget-outcome-and-an-unknown-window-policy.md), -> [ADR-0100](../decisions/0100-budget-authorization-is-durable-state-with-a-replay-barrier.md) and the -> [twelve-step W7 plan](phases/phase-2.6.5-core-reliability-remediation.md#w7-pre-implementation-review-and-proposed-execution-plan--2026-10-02). -> Work remains on `development`; each step closes with two fresh independent Codex review rounds and verified -> fixes. Step 1's implementation and first review corrections pass `pnpm run ci` and `pnpm coverage`; -> [round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-04-05-w7-step-1-round-1-review.md) verified seven findings; -> [round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-22-32-w7-step-1-round-2-review.md) verified a cleanup race in its fix. -> [Round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-35-41-w7-step-1-round-3-review.md) verified a passing/starting race; -> cleanup now retains shared parents and deletes only owned probes. All corrections pass the gates; -> [Round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-43-31-w7-step-1-round-4-review.md) cleared probe ownership and found -> a capture pathname-cleanup race. The runner now closes its descriptor without deleting a failed -> destination. [Round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-50-35-w7-step-1-round-5-review.md) cleared the -> complete step and final corrections; step 1 is closed. Step 2's structural store boundary and -> durable effect-turn allocator pass the required checks and root coverage (289 suites, 6,367 passing -> tests, 11 skipped); full CI passed at `1927daa1`. -> [Round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T04-29-12-w7-step-2-round-1-review.md) verified nested-transaction -> key reuse and unsafe session media metadata. Both are corrected; checks and coverage pass -> (289 suites, 6,374 passing tests, 11 skipped), with full CI passing at `c96eb9b5`. -> [Round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T04-41-54-w7-step-2-round-2-review.md) cleared the complete step -> and both fixes, including real lock contention and process-crash checks. Step 2 is closed; -> step 3 implementation now connects structural completed turns, atomic persistence, empty-final -> resume/export and per-call effect ids across the CLI hosts. Required checks and coverage pass; -> [Round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T05-34-44-w7-step-3-round-1-review.md) verified a deep-JSON -> metadata failure and a lost policy-denial classification. Both corrections pass the required -> checks and coverage (291 suites, 6,406 passing tests, 11 skipped); full CI passes at `4045272c`. -> [Round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T05-48-54-w7-step-3-round-2-review.md) verified a cached-command -> durability bypass and a missing tool name on recovered filesystem denial. Both corrections pass -> checks and coverage (291 suites, 6,410 passing tests, 11 skipped); full CI passes at `e95a6010`. -> [Round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T06-10-38-w7-step-3-round-3-review.md) independently verified an -> unjournaled-tool durability bypass and a stale export introduction. Dispatch admission now checks -> the live latch after approval/preparation for every actual tool call and releases a proven -> non-dispatch claim. Both corrections pass checks and coverage (291 suites, 6,415 passing tests, -> 11 skipped); full CI passes at `dd50738c`. -> [Round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T06-34-39-w7-step-3-round-4-review.md) cleared dispatch admission -> and verified loss of nontrailing pre-W7 empty-final user text from resume/reseat and export prompts. -> Legacy context and boundary slots now preserve the old read behaviour without fabricating terminals -> or completed counts. Required checks and coverage pass (291 suites, 6,422 passing tests, -> 11 skipped); full CI passes at `6d5b3f0b`. -> [Round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T06-48-45-w7-step-3-round-5-review.md) cleared the complete step -> and all earlier corrections, including independent mixed-history projection, YAML, real SQLite -> boundary and process-crash probes. **Step 3 is closed; step 4 proceeds automatically** with -> session-result privacy, full-history disclosure snapshots and exact committed-row retention. -> Step 4's [first acceptance round](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T08-05-02-w7-step-4-round-1-review.md) -> independently reproduced pre-mount Ink resume/reseat deletion. The verified correction separates -> opening from committed-mount activation and carries driver activity through disclosure and teardown; -> [Round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T08-48-12-w7-step-4-round-2-review.md) reproduced activation before passive Ink input setup -> and before disclosure was drawn. Activation now awaits usable setup and disclosure render acknowledgement, -> observes renderer exit/error and keeps input gated; actual Ink and Home tests verify the display boundary. -> Required checks and coverage pass (297 suites, 6,487 passing tests, 11 skipped); a third fresh -> corrective acceptance round was required before step 4 closes. -> [Round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T09-24-27-w7-step-4-round-3-review.md) verified incomplete-call attribution and -> flush acknowledgement of unwritable output. Completion now uses the shared structural projector; -> rendered disclosure checks the actual Ink output and its lifetime. Required checks, build and coverage -> pass (297 suites, 6,497 passing tests, 11 skipped); a fourth fresh acceptance round is required. -> [Round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T10-04-04-w7-step-4-round-4-review.md) verified headless stderr enqueueing mistaken for delivery -> and inline disclosure lost during Ctrl-Z suspension. Native stderr callbacks now acknowledge delivery; -> notice publication and render acknowledgement share terminal ownership with suspension. Nine focused suites -> pass 217 tests, including delayed native delivery and independent real STOP/CONT controls. -> Required workspace checks, build and coverage pass (298 suites, 6,514 tests, 11 skipped). Full CI -> passed at `b9d52d76`; round 5 required two more corrections. -> [Round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T10-36-22-w7-step-4-round-5-review.md) independently reproduced blocked native stderr -> holding Ctrl-C teardown and late Home acknowledgement deleting evidence during canceled-session MCP close. -> Headless activation now races an owned interrupt; Home activation requires a running session outside teardown. -> Eight focused suites pass 310 tests; native pipe/FIFO before/after controls and late callbacks after actual -> database close pass. Workspace checks, build and coverage pass (298 suites, 6,520 tests, 11 skipped). -> Full CI passed at `e753beb7`. [Round 6](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T10-57-58-w7-step-4-round-6-review.md) accepted the complete -> step and every correction with no new material findings. Parent native/privacy controls pass; compiled-entry -> process-exit delay under queued stdio also reproduces on the exact pre-step baseline and is explicitly qualified. -> **Step 4 is closed; step 5 proceeds automatically.** All six W7 register items remain open. -> Step 5's memory implementation and canonical docs are checked: -> one pure request projection, frozen policy, completed-turn spans and engine/CLI history refusals; -> actual request, workflow-node and SQLite resume/reseat controls pass. Workspace checks, build and coverage -> pass (299 suites, 6,567 passing tests, 11 skipped). Later measured pre-send/recovery and revised compaction -> outcomes remain staged. Clean-tree full CI passed at `ecdcc395`. -> [Step 5 round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T12-00-17-w7-step-5-round-1-review.md) accepted the implementation: -> both reviewers passed 347 focused tests; parent-rerun controls passed 390 state/history cases and 86 -> actual CLI/SQLite cases. Matched source-head operational controls pass; exact pre-step engine controls -> fail as expected. Adjacent working-context/durable-history wording is corrected. -> [Step 5 round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T12-21-33-w7-step-5-round-2-review.md) accepted the complete step: -> both fresh reviewers passed 347 tests; the parent reproduced 1,536 state/history and 86 CLI/Home/SQLite -> cases, plus exact-source positive/negative controls. **Steps 1–5 are closed; all six W7 items remain open.** -> Step 6's independently reviewed [Accepted ADR-0101](../decisions/0101-configured-output-estimates-apply-only-when-the-wire-is-uncapped.md) -> resolves a newly identified configured-estimate/native-cap precedence gap. The maintainer approved the -> complete revised decision on 2026-10-02; dated forward notes are landed and step 6 proceeds; -> [ADR revision round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T15-09-58-w7-adr-0101-revision-round-1-review.md) records the maintainer's -> verified transport/pricing findings and the corrected measurement-to-attempt handoff. -> [Fresh draft round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T15-18-01-w7-adr-0101-revision-round-2-review.md) accepted the complete -> corrected proposal with no actionable findings; full CI passed at `2b628e1a`. The acceptance and runtime -> obligations remain distinct: runtime acceptance is now complete. -> [Step 6 round 8](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T22-44-01-w7-step-6-round-8-review.md) accepts the complete step and seven corrections. -> Both fresh reviewers reported no verified new findings; parent replays pass 120 contract and 54 -> financial controls. **Steps 1–6 are closed; all six W7 register items remain open.** -> Step 9's quote/debit foundation is implemented, with [candidate preflight and parent causal controls](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T23-40-16-w7-step-9-foundation-preflight-review.md). -> Full CI/coverage pass after the round 3 correction (315 test files, 7,152 passing tests, 11 existing skips). -> [Round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T00-25-08-w7-step-9-round-1-review.md) verified and corrected two caller settlement findings; [Round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T01-10-45-w7-step-9-round-2-review.md) verified and corrected the async consumer and cancellation teardown; [Round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T01-57-14-w7-step-9-round-3-review.md) verified and corrected park/poll host admission loss and the reentrant durable-total race. -> [Fresh round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T02-40-53-w7-step-9-round-4-review.md) accepts the complete foundation with no new findings; both sealed inventories and 70 parent-rerun fresh cases are verified. Its corrected native-loader evidence and prior harness isolation exception are recorded explicitly. -> **Step 9 is closed; approved Step 10 proceeds automatically** with durable workflow activation and replay. -> [Step 10 round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T08-35-49-w7-step-10-round-1-review.md) reproduced and corrected -> passive-resume lease expiry and unbound predecessor CJS/peer/optional edges. Exact-fence renewal and -> an owned, versioned dependency closure pass causal controls, full CI and coverage -> (320 suites, 7,318 passing tests, 11 existing skips), including seventy actual-runner cases, -> twelve real predecessor refusals, two legacy controls and seven permanent graph/runtime controls. -> [Step 10 round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T11-47-48-w7-step-10-round-2-review.md) verified and corrected -> terminal conservative-money ordering, historical budget input compatibility, companion amount symmetry -> and pinned runtime edges. Full CI/coverage pass (320 suites, 7,340 passing tests, 11 existing skips); -> 78 native runner cases, thirteen graph controls and actual historical-input replay pass causal checks. -> [Step 10 round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T12-39-00-w7-step-10-round-3-review.md) verifies and corrects -> authority-witnessed optional-amount duplicate compatibility, preserving strict money conflicts, -> genuine output and unrelated pending work. Eighteen permanent regressions and 300 native selected -> cases pass. Full CI/coverage pass (320 suites, 7,358 tests, 11 existing skips); fresh full-Step-10 -> round 4 precedes closure. All six register items remain open. -> [Step 10 round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T13-33-00-w7-step-10-round-4-review.md) corrects lease cleanup rejection replacing -> primary typed refusals or an already-terminal no-op. Eighteen native regressions cover all six -> guards and successor takeover; full CI/coverage pass (320 suites, 7,376 tests, 11 existing skips). -> Fresh whole-Step-10 round 5 precedes closure; no residual is deferred. -> [Step 10 round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T14-31-39-w7-step-10-round-5-review.md) corrects inherited parallel gate publication handing off -> and then reclaiming a waiting-only lease. Ten permanent regressions cover held pause ACKs, -> actual SQLite fresh-owner resume and immediate human-decision media preparation. Async final-ACK -> limits are explicit; no new clock/timeout policy is chosen. Full CI/coverage pass (320 suites, -> 7,386 tests, 11 existing skips). Fresh whole-Step-10 round 6 precedes closure. -> [Step 10 round 6](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T15-42-00-w7-step-10-round-6-review.md) corrects inherited ordinary -> decision media completion after cancellation, deadline or fenced closure. Twelve permanent -> Core/actual SQLite-CAS regressions prevent late subscriber success and renewed run references; -> guard removal/restoration confirms the failure. Full CI/coverage pass (321 suites, 7,398 tests, -> 11 existing skips). Fresh full-Step-10 round 7 precedes closure. -> [Step 10 round 7](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T16-54-28-w7-step-10-round-7-review.md) corrects inherited media references written before a stale -> event fence is refused. Produced handles now follow append acknowledgement; owned uncertain-terminal -> media remains retained for the outbox. Seven permanent successor/terminal controls and independently -> verified sealed evidence pass, with full CI and sequential coverage (323 files, 7,405 tests, -> 11 existing skips). Fresh full-Step-10 round 8 precedes closure. -> [Step 10 round 8](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T17-50-23-w7-step-10-round-8-review.md) accepts the full 74-file step with no new verified findings. -> Both fresh reviewers pass 857 permanent tests and separate five/six fresh probes; complete reports and -> every sealed inventory entry are parent-verified. **Step 10 is closed; approved Step 11 proceeds automatically** -> with the safe CLI surface, permanent compiled-process smoke coverage and fresh independent reviews. -> Step 11 implementation now passes full CI and sequential coverage (327 files, 7,506 tests, 11 existing skips), -> including permanent compiled CLI refusal/status/abort/MCP-interruption smokes and raw-triple display regressions. -> [Step 11 round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T19-39-26-w7-step-11-round-1-review.md) independently verifies native approval-write cancellation, -> the no-MCP signal guard and stale identified-companion display cleanup. The parent fixes all three, -> adds passive Core cancellation/fence controls and corrects the ownership wording. -> Full CI and sequential coverage pass after the corrections (327 files, 7,519 passing tests, -> 11 existing skips; global line/branch/function 95.88%/92.63%/96.73%). -> [Step 11 round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T21-01-16-w7-step-11-round-2-review.md) verifies two further High findings: -> a successful native busy wait can conceal OS SIGINT until after egress, and terminal write uncertainty -> was misreported as takeover. The CLI host yields across a poll turn before decision acknowledgement; -> terminal uncertainty retains exit 5, while actual successor fencing still refuses without a false -> claim that an earlier approval is absent. Seven new permanent controls pass; both complete sealed -> reports and every inventory entry are parent-verified. Full CI and sequential coverage pass -> (327 files, 7,526 tests, 11 existing skips; 95.88%/92.63%/96.73%). Fresh full-Step-11 round 3 follows -> the correction commits; the step remains open. -> [Fresh Step 11 round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T01-17-23-w7-step-11-round-3-review.md) is clean in both -> complete independent reviews: 452 permanent tests each plus separately reported fresh native, -> passive and compiled-process controls. Both reports and every sealed inventory entry are verified. -> Interrupted earlier round-3 runs contribute no acceptance. **Step 11 is closed.** -> [Independent Step 12 document preflight](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T01-04-15-w7-step-12-document-preflight-review.md) -> verifies and corrects usage/pricing and fixture-coverage prose; connected at-rest/event claims, -> guides and moved follow-up status are reconciled. This independent maintenance proceeds through -> repository checks and fresh post-commit review; Step 12's whole-wave acceptance remains open. -> [Committed maintenance round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T07-00-17-w7-step-12-maintenance-round-1-review.md) -> independently verifies one Medium reservation-existence qualification and one Low inherited -> event-table guarantee. Both are parent-verified and corrected in four text/comment locations. -> Complete reports and all 2,687 sealed entries are verified; 81 fresh reviewer DB tests and -> a real-writer reference-corruption/restoration control pass. Fresh complete maintenance -> round 2 follows the correction commits; whole Step 12 remains open. -> [Committed maintenance round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T07-25-20-w7-step-12-maintenance-round-2-review.md) is clean -> across all 26 paths/37 hunks in both fresh reviews. Both complete reports and all 2,574 sealed -> entries/source pins are verified; fresh 81-test DB/strict checks and real-writer causal controls pass. -> **The independent maintenance subset is closed; whole Step 12 and all six W7 items remain open.** -> [Draft PR #90](https://github.com/HodeTech/Relavium/pull/90) is open on `development`. -> Required Linux CI revealed an actual-Ink test-context mismatch. Further controlled checks found -> undisplayed disclosure under the existing CI opt-outs and Home's inline reseat Static cursor; -> corrections pass two fresh independent review rounds. Prior scoped acceptance does not waive other findings. -> All five required actual overflow records (Anthropic input rejection and native context stop, -> OpenAI, DeepSeek and Gemini) are now preserved in the -> [fixture index](../../packages/llm/src/conformance/fixtures/overflow/README.md). -> The separate Anthropic native context-stop response was captured on 2026-10-08; -> the evidence prerequisite is satisfied. Step 7 classification, Step 8 and whole-wave acceptance remain open. -> Step 1 repaired collection without deleting private artifacts and provides the -> [maintainer-run live overflow capture procedure](../runbooks/capture-provider-overflow.md). -> Classification must replay the preserved live fixtures. The six W7 items stay open. - -> **PR 90 systematic follow-up — 2026-10-04.** Two maintainer reports against `95137c34` register 35 -> deduplicated findings, with [qualified independent triage](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T10-35-00-w7-systematic-review-intake-review.md). -> Step 6's request-ownership guarantee is reopened; adjacent session/CLI, replay-tool and documentation -> obligations require correction and fresh acceptance. The [first committed rendering review](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T10-32-37-w7-rendering-round-1-review.md) -> reproduces notice loss after a supported chat resizes and CI route/ownership drift. Its corrections -> are accepted by [fresh round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T12-00-00-w7-rendering-round-2-review.md). -> The rendering corrective group is closed. Previous step closure records remain historical evidence, -> not current whole-wave acceptance. All ten GitHub workflow checks pass at `d4e4fa9c`; SonarCloud and remaining findings stay -> separately under examination. No finding is deferred at intake. Steps 7–8, whole Step 12 and all six -> W7 register items remain open. - -> **Request-ownership decision preflight — 2026-10-04.** [Proposed ADR-0102](../decisions/0102-a-measured-request-owns-its-inert-data-through-egress.md) -> would close the measured-to-sent alias gap through one owned inert construction and existing candidate -> cap authority. It deliberately narrows non-cap opaque inputs and requires maintainer approval before -> dependent implementation. [Draft round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T12-21-41-w7-adr-0102-round-1-review.md) -> corrects SDK shared-reference parity and incoming measured-plan ordering. [Fresh complete round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T13-02-54-w7-adr-0102-round-2-review.md) -> receives no new material proposal findings; maintainer approval remains required before dependent -> implementation. Independent corrective work continues. The ownership HIGH remains open. - -> **Systematic group 2a, round 1 — 2026-10-04.** [Two complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T13-47-56-w7-systematic-group-2a-round-1-review.md) -> verify the cap/structural repairs but reproduce a new successful-tool observer regression: -> real session usage is lost and the hard turn cap can be bypassed. Corrections retain classified -> engagement/usage and fix the normalizer comment. Forced root checks and build pass; **fresh -> complete round 2 precedes group closure**. Ownership, remaining groups and W7 acceptance stay open. - -> **Systematic group 2a, round 2 — 2026-10-04.** [Both complete reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T14-41-03-w7-systematic-group-2a-round-2-review.md) -> verify the successful observer correction but reproduce an inherited failed-tool observer High. -> `41a3ef9a` retains real usage/engagement for invalid-argument and read-failure notifications, -> including EA7 abort, without broadening registry recovery. Eight new regressions fail before -> repair; forced root checks/build pass afterward. **Fresh complete round 3 precedes closure.** -> Ownership, other groups and whole W7 acceptance remain open. - -## Execution order — Phase 2.5.5 + Phase 2.6 (temporary) - -> **Temporary section — delete it when both phases close.** It owns **ordering only**. Every item's -> definition, size, file list and finding ids live in its phase file — -> [phase-2.5.5](phases/phase-2.5.5-hardening-and-remediation.md) (176 work items, sub-streams A–I) and -> [phase-2.6](phases/phase-2.6-conversational-authoring.md) (15 open workstreams) — and nothing below -> restates them. Items are cited as ` · ()`, which -> greps directly against the phase file. -> -> Derived 2026-07-26 from a full pass over both phase files, their own sequencing sections, and the live -> tree. Where it departs from a phase file's own prose, see [Departures](#departures-from-the-phase-files-own-sequencing). - -### Ordering root — one baseline ✅ **discharged 2026-07-26** - -The plan opened on a divergence neither phase file modelled: Phase 2.5.5's whole backlog is written against -`f88b0e8`, which existed only on `development` behind PR #76, so every `file:line` citation in it resolved -against a tree nobody was branching from. **PR #76 and #77 are both merged** — `f88b0e8` is on `origin/main`, -`origin/development` and `origin/main` held identical trees with no open PRs as of 2026-07-26. Every -citation in this plan now resolves. (PR #79 and PR #80 have opened since.) - -One consequence survives and is worth keeping until the phases close: - -> **Branch from `origin/development`, never from a local `main`.** A long-lived `development` merged into -> `main` by merge commit leaves a stale local `main` behind after every PR, silently and without conflict. -> Run `git fetch` and cut from `origin/development`. - -Also settled while discharging this: `main` now carries branch protection (PR required, `lint · typecheck · -test` required, force-push and deletion blocked) and a `v*` tag ruleset restricting tag creation to the -maintain/admin roles — closing the Phase-0 obligation that made `ci.yml`'s own "REQUIRED status check" -comment untrue. `release.yml` asserts its ref is reachable from `main` before `pack`, so **2.5.5.H's `G26` -is closed ahead of Wave 0**. - -### The three axes this order is built on - -1. **Irreversibility** — a defect whose damage a later fix cannot undo jumps every queue. That is Wave 1: - a credential already written into a durable `--json`-visible row, a billed turn lost after the reply - rendered, model-controlled ANSI inside the consent card, and the states where the cost cap is silently off. -2. **File ownership** — whoever edits a file next inherits everyone else's rebase. Every god-file - decomposition and naming/convention ruling lands *before* the 2.6 workstream that piles onto it, and - eleven items are pulled across the phase line because they edit the exact function another item restructures. -3. **Certification** — a wave that discharges a phase obligation certifies it while the evidence is fresh, - which is why 2.5.5's exit criteria 1–3 close in Wave 2 rather than at the end. - -Waves are readable single-branch sequences. Concurrency is expressed as *"these queues are file-disjoint, -any order"* — never as headcount. +This page tracks what is active **right now** and the order of the next work. The full phase plan and the global +milestone spine are in [README.md](README.md). Finished work keeps its record in the phase registers; this page +links to those records instead of restating them ([History](#history)). + + + +## Now — the CLI MVP comes first + +On **2026-10-26** the maintainer paused Wave 3 of Phase 2.5.5 after Step 9b.2 (accepted at `563402e6`) and moved +the work that a CLI user feels to the front: features and bugfixes in `apps/cli` and `packages/core` that +directly affect an MVP. **Paused means pending, not declined:** every paused item keeps its phase-file home, +its recorded acceptance rules and its security sittings, and the [after-MVP queue](#pending-after-the-mvp-not-declined) +keeps its internal order. On **2026-11-04** the maintainer ruled D58–D72 ([decision table](#wave-gating-decisions)) +from a multi-agent review of the roadmap documents and the code at `5a328bca`; the +[priority reset record](phases/phase-2.5.5-hardening-and-remediation.md#mvp-priority-reset--2026-11-04) holds the +evidence summary, the recorded Wave 3 splits and the process amendments. + +### Status at a glance + +| Area | State | +|---|---| +| Phase 1 — Engine and LLM | Complete (M2, 2026-06-21) | +| Phase 2 — CLI | Feature-complete; `relavium@0.1.1` published on npm 2026-06-28 (M3) | +| Phase 2.5 — CLI Consolidation | Complete (M2.5-4, PR #69, 2026-07-08) | +| Phase 2.6.5 — Core reliability | Complete with approved deferrals, 49/51 (PR #92, 2026-10-11; [merge checkpoint](phases/phase-2.6.5-core-reliability-remediation.md#w8-merge-checkpoint--2026-10-11)). CR-70, CR-93 and CR-95's long half stay deferred in [deferred-tasks.md](deferred-tasks.md) | +| Phase 2.5.5 — Hardening | Waves 0–2 merged (Wave 2: PR #94, 2026-10-15). Wave 3 Steps 1–8 merged (PR #95, 2026-10-18); PR95, 9a.1, 9a.2a, 9a.2b, NI, 9b.1 and 9b.2 accepted on `development`. **Steps 9c–19 paused** (25 of 32 increments). NI's Windows clause closed on 2026-10-26 | +| Phase 2.6 — Conversational authoring | 2.6.C done; 2.6.F done except its release clause; 2.6.Q P1–P5 landed; the other fourteen workstreams have not started in their core. The MVP takes its first slices (below) | +| Phases 3–7 | Not started — desktop, VS Code, then Product Phase 2 (managed inference, cloud) | +| Publication | npm serves `0.1.1`. Nothing newer can be published yet: Step 9a.1's interim guard on `development` refuses every artifact until D59's superseding release ADR is Accepted | +| Branches | `development` is ahead of `main` (`6d2a7372`). `main` lacks the PR95 scrubber fixes, NI and the interim guard; a scoped `development`→`main` checkpoint follows this reset (D61) | + +### What a CLI user meets today + +The engine is not the bottleneck: workflows author, run, gate, resume and retry well. The agent and presentation +layers are. The built-in chat agent can only read (`read_file`, `list_directory`, `git_status`) and starts in +`ask`; there is no edit, search or find tool; `run_command` is refused before any approval unless pre-allowlisted; +`relavium run` never prints the workflow's outputs or the run id; replies render as raw markdown; a resumed chat +opens on an empty viewport; a non-Anthropic first key leads to an opaque credential error. The +[priority reset record](phases/phase-2.5.5-hardening-and-remediation.md#mvp-priority-reset--2026-11-04) cites the +code for each. + +## Next work — the CLI MVP + +**The MVP (D58) is `relavium` v0.2.0 on npm** — new global milestone **M3.5** ([README.md](README.md#global-milestone-spine)): +`apps/cli` plus the core engine (`packages/llm`, `shared`, `db` and `mcp` where they decide user-visible +behaviour). A new user installs it, onboards with any of the four providers, chats with an agent that reads, +searches, edits and runs commands under approval with diffs, authors a workflow conversationally or by export, +runs it and sees its outputs, browses and inspects past runs and sessions from the Home, manages providers and MCP +servers there, reaches the web under the chat egress authority, cancels and resumes chats and gated runs, and reads +docs that describe what ships. English only (the `en`+`tr` bar binds Phase 2.6, not the MVP). There is no npm +pre-release before v0.2.0 (D70). + +**Scope lines kept** (this page may not move them): media input, file-snapshot undo and the workflow-run +`egress`/`os` arms stay out ([phase 2.6 scope](phases/phase-2.6-conversational-authoring.md#explicitly-out-of-scope--phase-3--later)); +the MVP only adds a pre-flight refusal for the latter. Local models stay out by product constraint. + +**How increments are accepted (D64, tiered):** an increment that touches an approval or consent path, egress, +keys or secrets, filesystem writes, a new terminal sink for untrusted content, the lease or resume paths, or +release and publication keeps the 2026-10-23 bar (rigor pass, one NEW complete cumulative round with no blocker, +high or medium finding, and a separate security sitting). Other UX, rendering and command increments are reviewed +per milestone: author self-review, `pnpm run ci` plus `pnpm coverage:enforced`, and one independent scoped round +with no high or medium finding; lows go to [deferred-tasks.md](deferred-tasks.md). + +Each milestone ends green on `development` with its canonical docs updated. Items cite their one home: a phase +2.6 bullet as ` · `, a new item as `MVP-A` in phase 2.6's +[CLI MVP additions](phases/phase-2.6-conversational-authoring.md#cli-mvp-additions--2026-11-04), or a recorded +Wave 3 split as `row ` in the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments). + +0. **Gate 0 — decide and re-order (no product code).** This reset; the `development`→`main` checkpoint (D61); + drafts for the new ADRs: the pre-1.0 release path (D59), the toolbelt plus tool-render/approval-preview contract + (phase 2.6 ADR row 9, split so `ask_user` comes later), the in-house markdown architecture (ADR row 11, part 1; + no dependency), the ADR-0057 amendments (default grant and `default_mode`, D62; denial feedback, D67), the + keymap (D68), the `run_command` approval card (D63) and the ADR-0095 amendment for the CR-71 residual (D67). Maintainer + gates: CI provider keys for the live-nightly lane (D37), provider spend for a manual three-provider turn and a + Windows machine for MVP-10. A stale-entry sweep of [deferred-tasks.md](deferred-tasks.md) and the phase 2.6 + bullets that 2.6.5 and Wave 2–3 work already satisfied. +1. **MVP-1 — Run it and see it.** Human run output and the run id (MVP-A1, MVP-A2); 2.6.G · Non-interactive + parity (`list --runs`, `status `); 2.6.G · **Liveness** (`engine.reconcile()` on Home open and at + `run`/`gate` start; `status` stays read-only); 2.6.K · **Validate the gate decision shape against `gateType` + before resume**; 2.6.A · **Run-path tool pre-flight for the unwired egress/os arms** (and no `web_search` in + the `create` placeholder); `/export` into `.relavium/workflows/` (MVP-A3); the default model follows the stored + key (MVP-A4); the interactive output budget (MVP-A7). +2. **MVP-2 — First run, resume and a readable chat.** Onboarding offered by `relavium chat` (MVP-A5); standard + provider env-var fallback (MVP-A6); 2.6.G's resume-in-place **with the prior transcript repainted** (the + [deferred `chat-resume` entry](deferred-tasks.md)); picker-on-omitted-id and `chat --continue`; excess + positional arguments refused (row 10b split); 2.6.E · **Markdown + code-block rendering** (in-house, part 1 — + highlighting stays after the MVP); full-screen word-wrap (#61, moved from Wave 5a); tab expansion (#65, row 15a + split); a failure mark for failed tool calls (MVP-A8); the persistent keyless nudge (#51, row 16b split); the + `agent run` stdin hint (#24, row 17a split); a minimal `relavium init` (MVP-A18). +3. **MVP-3 — The agent can edit.** ADR row 9 and its mandatory security review; 2.6.M · **`edit_file`**, + **`search_files`**, **`find_files`** (with `list_directory`'s silent 200-entry cap made visible); `read_file` + ranges and the spill store (MVP-A9); 2.6.M · `write_file`'s approval diff (#305) and **Tool-call rendering + v2** (sanitized target kept after the turn; assistant text between tool calls kept); 2.6.M · **Default chat + agent grant review** as amended by D62 (MVP-A14) with `[chat].default_mode` (MVP-A13); the 80×24 guard on + both chat surfaces and the Home strip row budget (#63, row 15c split) — MVP-3 owns the strip's geometry + and its no-hidden-input rule, MVP-8 its content. +4. **MVP-4 — The agent runs commands and keeps its footing.** 2.6.G · **Permission management** as ruled by D63 + (MVP-A15; `!` uses the same card); the default agent gains `run_command`; the keymap, including Ctrl-C at a + pending approval rejecting it (MVP-A12; row 11b split); denials returned to the model (MVP-A10); a failed, + denied or aborted turn kept in context as a content-free interruption marker (the CR-71 residual, + un-deferred, D67; CR-71 itself closed in W7); configurable, finite turn ceilings with conservative + defaults (MVP-A11). +5. **MVP-5 — Context and parity (D71).** Anthropic prompt caching (2.5.5.B · #172); a project instructions file + and environment block (MVP-A16); 2.6.E · **Type-ahead message queue**; `chat --model/--provider` and a one-shot + prompt mode (MVP-A17). +6. **MVP-6 — Conversational authoring.** Accept ADR-0058 and ADR-0060 with their reviews; 2.6.A · + `validateAuthoredWorkflow` back-port and **`nodeCatalogIssue` never resolves the agent's own model**; `$ref` + agent resolution; 2.6.B's wizard and conversational authoring loop, its import gate and the UVP hint + (**Discoverability of the UVP**); 2.6.D `{{ctx.*}}` and `agent run --input`; starter workflows (MVP-A19). +7. **MVP-7 — Web.** G30 (rows 13a–13c, pulled forward; 13c is re-scoped when it starts because Step 12b is + paused); 2.6.M · **`web_search` activation**; web fetch through `http_request` under the chat authority. +8. **MVP-8 — The Home as hub.** Phase 2.6 ADR row 6; 2.6.H's history data layer as far as G needs it; 2.6.K's + shared resume core; 2.6.G's `/agents`, the actionable strip's content (within MVP-3's row budget), + `/workflows` (defined, history, run detail) and the bound-agent view; Home `/help`; `logs --follow/--failed`. +9. **MVP-9 — MCP and provider management.** Phase 2.6 ADR row 7; MCP servers attached to the default agent under + ADR-0084 consent (MVP-A21); 2.6.I · **`/providers`**, `/mcp`, **`relavium mcp list/add/remove/test`** and MCP + header auth; the masked `provider set-key` prompt (MVP-A20). +10. **MVP-10 — Ship v0.2.0.** The D59 release path (superseding ADR, guard change, security sitting); the final + `development`→`main` checkpoint; release prerequisites and checks (MVP-A22, MVP-A23); v0.2.0 closes 2.6.F's + release clause and M2.6-1. + +**After v0.2.0, first:** v0.2.1 — cross-turn tool memory (CR-70, on its recorded trigger) designed together with +ADR-0096 §5's in-turn overflow continuation (D67). + +**Recorded risks (maintainer rulings):** the published package grants no usage right under `LICENSE` §2 and +stays so (D60); the published `0.1.1` still prints unsanitized model output in `relavium run` (#56, fixed on +`development`) and is left as is because the product has no users yet (D72). + + + +## Pending after the MVP (not declined) + +> **Temporary section — delete it when Phase 2.5.5 and Phase 2.6 close.** It owns **ordering only**; every +> item's definition, size, file list and finding ids live in its phase file, and items are cited as +> ` · ()`. The waves below keep the order derived on +> 2026-07-26; since 2026-11-04 they run **after** the MVP above. The original plan's baseline, axes, Waves 0–1 +> and the PR #81 closing list (the only definition of `#W15-1`…`#W15-24`, which code comments cite) are preserved +> verbatim in the [Phase 2.5.5 register](phases/phase-2.5.5-hardening-and-remediation.md#execution-record-moved-from-currentmd--2026-11-04), +> and so, since the [PR #96 review correction](phases/phase-2.5.5-hardening-and-remediation.md#moved-by-the-pr-96-review-correction--2026-11-04), +> are this page's other records: the Wave 2 and Wave 3 sections, its status blocks, and the W7, W8 and Phase 2.6.5 +> records in [Phase 2.6.5's](phases/phase-2.6.5-core-reliability-remediation.md#execution-record-moved-from-currentmd--2026-11-04). ```mermaid flowchart TD - W0["Wave 0 — One true baseline
baseline ✅ · CI truth · numbers"] - W1["Wave 1 — Stop the bleeding ✅
3 CRITICALs · cost cap · ADR-0074"] - LEDGER["#W15-1 — realized-cost ledger ✅
ADR-0076 + ADR-0077"] - P265["Phase 2.6.5 — Core reliability
51 CR items · 8 P0 blockers behind ADR-0078–0084
W0–W8 complete · W3 blocker closed in W8
W4 merged — the hostile-MCP class, ADR-0088
W5 merged — media correctness, ADR-0089+0090
W6 merged — authoring correctness, ADR-0091–0094
W7 merged — PR #90 · W8 merged — PR #92
49/51 closed · approved deferrals retained"] - W2["Wave 2 — Shut the doors ✅
PR #94 merged · fs jail · secrets · config trust
2.5.5 EXIT 1–3 satisfied"] - W3["Wave 3 — Clear the ground
god-file decomposition · CLI net"] - W4a["Wave 4a — The spine
2.6.A/D/H/K + 2 ADRs"] - W4b["Wave 4b — Money floor & close-out
2.5.5.B/C/F/H drain · v0.2.0"] - W5a["Wave 5a — Paint before you build
render-v2 · theme · config"] - W5b["Wave 5b — Home becomes the product
2.6.G/I/J · then i18n sweep"] - W6["Wave 6 — Hands, voice, lineage
2.6.M/B/E + 2.6.N foundation"] - W7["Wave 7 — Orchestration & the gate
2.6.O/P · go/no-go → Phase 3"] - W0 --> W1 --> LEDGER --> P265 --> W2 --> W3 --> W4a --> W5a --> W5b --> W6 --> W7 + G0["Gate 0 — decisions, re-order, ADR drafts"] + MVP["MVP-1…MVP-10
run+see · first run · edit · commands · parity
authoring · web · Home hub · MCP · ship v0.2.0"] + R021["v0.2.1 — CR-70 + ADR-0096 §5"] + W3["Wave 3 remainder (paused)
9c–19b minus the recorded splits"] + W4a["Wave 4a — The spine (what the MVP left)"] + W4b["Wave 4b — Money floor & close-out"] + W5a["Wave 5a — Paint before you build"] + W5b["Wave 5b — Home becomes the product"] + W6["Wave 6 — Hands, voice, lineage"] + W7["Wave 7 — Orchestration & the gate"] + G0 --> MVP --> R021 --> W3 --> W4a --> W5a --> W5b --> W6 --> W7 W3 --> W4b - W4b -.->|v0.2.0 release| W7 W4a -.->|file-disjoint lanes| W4b ``` -### Wave 0 — One true baseline - -Collapse the divergence, make the required gate execute the artifact it ships, and fix the review -checklists before ~30 security-gated PRs are reviewed against them. - -1. ✅ **Merge PR #76** (2.6.Q P1–P5, ADR-0071/0072, the three picker bugs, the param self-heal) — **done - 2026-07-26**, along with #77 (this plan + the Phase-2.5.5 opening + `release.yml`'s `G26` ancestry gate). -2. ✅ **Reserve numbers** — **done**, recorded here: migrations end at `0012` and ADRs at `0072` (verified), - so `0013` = **ADR-0074 §4's conservative money columns** (`agent_sessions.total_conservative_microcents` + `session_costs.conservative_microcents`) — the number was freed when D8 ruled against the approval-preview scrub and is RE-USED here rather than left as a hole · `0014` = 2.5.5.C's enum CHECKs (#101) · `0015` = 2.6.H · - `0016` = 2.6.G's pins · `0017` = 2.6.N lineage; **ADR-0075+** for the nine unwritten 2.6 ADRs (`0073` = the history.db migration lock, `0074` = durable conservative budget commitments — both landed in Wave 1). -3. ✅ **One `ci.yml`/`turbo.json`/`tsup.config.ts` PR** — **done 2026-07-29 (PR #80)**, all eight items: - 2.5.5.H · the required gate never runs the compiled binary + undeclared `drizzle` output (#294, #315) → - local `pnpm run ci` vs `ci.yml` divergence (#312 — and `pnpm ci` is shadowed by a pnpm builtin, so the - script was unreachable by the name everyone types) → the coverage-floor **ruling and its implementation** - (#296, #152) → the `(advisory)` labels (#320) → `THIRD_PARTY_EXTERNAL` (G27, #248) → bundle-closure - single-chunk assert (#314) → `sync:models:check` (#317). *(`release.yml`'s ancestry check, `G26`, landed in - #77; its tag-protection half is configured.)* The PR also closed the four blockers from the 2026-07-29 - roadmap review, and its own CI caught two real defects in it — a TS4111 that `turbo run typecheck` does not - cover, and `pnpm ci` being unreachable because pnpm reserves `ci` as a builtin. -4. ✅ **2.5.5.F · three skills' hardcoded foreign-project paths** (#162) — **done 2026-07-29.** Filed as docs, - actually a functional bug: they pointed at `/Users/dev/…/Agent-Organizer/`, so `add-package`'s `mkdir -p` - would have created a tree outside the repo. All three now anchor on `$(git rev-parse --show-toplevel)`. -5. ✅ **One markdown PR** — **done 2026-07-29**: `packages/mcp` added to all five inventory sites (#128, #129, - #153, #163, #254), the reviewer agent's secrets item extended to the shipped CLI floor (#164), the - security-review skill's SSRF step now requires reusing the shared guard rather than re-deriving one (#167), - and the i18n clause settled by the D5 ruling (#74, #260, #134). -6. ✅ **The locale-bar amendment** (D5, `en`+`tr`) and the **snapshot regen** (D3) — **done**, propagated to - all seven locale sites and merged as PR #79 (two price rises taken, nine retirements accepted). - -> ✅ **Wave 0 is COMPLETE (2026-07-29).** The required gate now executes the binary it ships, -> `apps/cli/drizzle/**` is a declared turbo output, and `lint · typecheck · test` + -> `engine coverage floor (llm, mcp)` are both required checks under branch protection. In phase terms this -> closes **8 of 2.5.5.H's 14** tasks and **3 of 2.5.5.F's 20**. **Wave 1 followed and is also complete -> (2026-07-30, PR #81) — see below; Wave 2 is next.** - -### Wave 1 — Stop the bleeding - -All three CRITICALs, plus the states where `max_cost_microcents` is silently not a cap. Four file-disjoint -lanes, any order between them, plus the money hole they left open. - -**✅ Wave 1 is COMPLETE — lanes (a)–(d) on 2026-07-30 (PR #81), lane (e) on 2026-08-09 (`#W15-3`).** - -> **Superseded 2026-08-09 — kept because the correction is the record.** Lane (e) closed with `#W15-3`: -> `/cost --release` makes §1's escape hatch reachable, and `/cost` renders the durability state. The -> retraction below was accurate when written and is no longer; it stays so the history of the false -> completion claim is not quietly erased. -> -> **Retraction (2026-07-30).** This section previously said Wave 1 was complete and ADR-0074 fully implemented. -> That was wrong on lane (e) and I am correcting it rather than leaving it: **§1's release escape hatch is -> exposed on `GovernorWiring` but reachable from no command, REPL action or Home affordance**, and `/cost` never -> renders `conservativeDurabilityBroken`. §1 names this exact state as forbidden — "a single usage-less response -> would permanently shrink a long-lived chat's cap with no way out" — and this PR REMOVED the accidental escape -> a restart used to provide. A section whose compensating bound is unreachable is not implemented, and marking -> it done made four other documents inherit the error. Two independent reviews flagged it; both were right. -> -> Lane (e)'s remaining work, and two defects it exposed that ADR-0074 does not cover, are tracked in -> **[PR #81's closing list](#pr-81s-closing-list-and-the-two-items-that-need-an-adr)** below. Every lane below is verified against the shipped code, not -against its own task text — the check for each is stated inline. Two gaps are recorded rather than closed; both -are named in their lane and marked in the source, and neither is a shipped defect: - -1. the engine-level composition test for ADR-0074 §3's abort-breakable media-job hold (`engine.ts`, at the abort - listener) — the governor's own state machine is mutation-verified, the composition with `#countRunning`'s - termination gate is not; -2. the CLI-layer test that `restoreConservativeCost` is wired through `buildSessionRuntime` → `GovernorWiring` - (`session-host.test.ts`) — `AgentSession.resume` calling the hook IS mutation-verified in `packages/core`; - only the spread that supplies it is unproven. - -- ✅ **(a) `registry.ts`** — 2.5.5.A · **redact tool-approval previews like `sanitizeInput()` already does** - (#91). *(The originally-planned migration 0013 scrubbing already-persisted previews was ruled OUT — see D8.)* *Closed: `previewFor()` now scrubs the DISPLAY copy while `rawPreviewFor()` keeps an - `unredactedPreview` for in-process classification — splitting the two uses, because a whole-string scrub - flipped the protected-path decision and a per-segment one reopened the leak. Verified: `registry.ts` and - `tools/types.ts` both carry `unredactedPreview`.* -- ✅ **(b) `packages/db` chain, strict** — 2.5.5.C · `SQLITE_BUSY_SNAPSHOT` (#100) → async backoff replacing - the blocking `Atomics.wait` (#226) → **the chat-persister safety net** (#228, CRITICAL) → - serialize `runMigrations` (#99) co-landed with the unconditional 0600 self-heal (#28, #33) → - `createClient` typed error + PRAGMA docstring (#104, #105). `deprecationDate` crash guard (G1) is a - standalone one-liner, droppable anywhere. -- ✅ **(c) render/signals** — 2.5.5.I · **terminal-control sanitization to the four boundaries it never - reached** (G34, G44, #56, #57, CRITICAL — one PR at the shared helper, its own security review) → - `SIGTSTP`/`SIGCONT` + onboarding signal-registration ordering (G0, #50). *Closed at all four boundaries the register names, verified by call count: - `RunApp.tsx` (#56) 9, `final-summary.ts` (#57) 6, `render-error.ts` (G34) 4, `clack-prompter.ts` (G44) 4. G0's - run/gate `SIGTSTP`/`SIGCONT` is `render/run-job-control.ts`; #50 is closed by ordering — - `drive-home.tsx` subscribes signals at `:808`, before `runOnboardingWizard` at `:855`.* -- ✅ **(d) money, split across two PRs** — `budget-governor.ts` only: 2.5.5.A · reservation ledger under - `max_parallel` (G38) → re-armed `budget:warning` + projected `thresholdPct` (G39, G47, G49) → **2.6.Q's - strict-cap half**. Then `fallback-chain.ts`/`cost-tracker.ts` only: 2.5.5.B · `maxRetries: 0` + jitter + - `Retry-After` (#276, #279) → narrowed `#emitSuccess` catch + `CostTracker` bounds (#194, #198) + - **2.6.Q's realized-cost half**. - -- 🔶 **(e) ADR-0074 — durable conservative budget commitments (§2–§5 shipped; §1 INCOMPLETE)** · the money hole the other four lanes leave open: - a bounded estimate retained when a provider may already have billed but returned no trustworthy usage was - kept only in memory, so a crash or a resume reopened a strict cap against money that may already be owed. - §5 (a forward-compatible stored-event READ, the precondition for emitting anything new) → §2 (the - `budget:estimate_committed` contract, the governor's durability barrier, the checkpoint fold and the resume - restore) → §3 (freeze an async media job's accepted units + price) → §4 (the session's conservative - aggregate + per-model attribution + migration + transactional persister write). - -> 2.6.Q's cost-cap item is **split**: its task text spans both `budget-governor.ts` and `fallback-chain.ts`'s -> `#emitSuccess` — the same catch 2.5.5.B narrows. The two halves join two different PRs, not one. -> **Closes M2.5.5-1.** -> -> **Outstanding inside lane (e) — restated at close, because §4 changed the facts.** ADR-0074 §1 ties a -> commitment's release to the surface that renders it. -> -> **Rendering: done on all three surfaces.** The run TUI and `relavium logs` shipped with §2; `/cost` shipped -> with §4's fold and reads *estimated, possibly billed*, with `(no completed call)` for a model that has one -> without a completed call. All three are mutation-verified. -> -> **Release: reachable, as of `#W15-3` (2026-08-09).** `/cost --release` on the chat surface — the one that -> renders the amount, which is what §1 ties it to — clears the durable per-model and aggregate columns first -> and the live governor second, so a released commitment does not return on the next `chat-resume`. `/cost` -> also renders the durability state. This note previously said no affordance called it; that was true and no -> longer is. -> -> The reserved `budget:estimate_released` event stays **reserved and unemitted**, and that is a decision, not -> an omission: the session path restores its conservative total from COLUMNS rather than by replaying events, -> so zeroing them is what makes the release durable there. The event is needed only by a workflow-run release -> surface, which does not exist — a run is not long-lived, and `on_exceed: pause_for_approval`'s per-node -> bypass or raising the YAML cap are §1's own analogy. §1's dated note in -> [ADR-0074](../decisions/0074-durable-conservative-budget-commitments.md) carries the same statement. - -### PR #81's closing list, and the two items that need an ADR - -Two independent adversarial reviews of PR #81 (2026-07-30) found that the cap's core safety claim does not yet -hold across a process boundary. - -**Almost all of this belongs to PR #81 itself, not to a later wave** — a first framing filed it as one, which -was wrong. The test is simple: this PR either ADDED the code a finding is in (`background-failure.ts`), or -claimed to have closed the area it sits in (the terminal-safety boundaries, ADR-0074 §1), or REMOVED a -protection without replacing it (§1's escape hatch, which a restart used to provide). Shipping a regression and -deferring its fix is not a schedule decision. - -**Only §A is genuinely separable.** Both of its items need an append-only ADR before any code — one changes what -"spent" means across a crash, the other supersedes a decision ADR-0074 §5 already recorded. Those earn their own -PRs. Everything from §B down is #81's to close. - -Ordered by whether the repo currently states something untrue, then by blast radius. - -**Every item names the check that would close it, not just the defect.** - -> **Status — 24 of 24 closed and MERGED, across two PRs.** Everything marked ✅ below is fixed and -> break-verified with the mutation confirmed applied; the two PRs are recorded separately because the closure -> dates differ: -> -> - **23 items merged to `main` via PR #81** (2026-08-09). -> - **`#W15-1`, the last one, closed 2026-08-10** behind ADR-0076 + ADR-0077 (see §A) and **merged to `main` -> via PR #82** (2026-08-11), together with the first Phase 2.6.5 batch. -> -> §E's six coverage gaps are all closed — `#W15-16`'s composition test fails by TIMING OUT when the abort -> listener is removed, which is the unkillable run reproduced exactly rather than an assertion standing in -> for it. -> -> **`#W15-2` closed 2026-08-09** as [ADR-0075](../decisions/0075-fail-closed-resume-on-an-unreadable-event-log.md), -> amending ADR-0074 §5: a read that feeds a REPLAY refuses when any row was skipped, a read that feeds a -> DISPLAY stays tolerant. It landed first on purpose — `#W15-1` adds a new durable event type, and an older -> binary reading it falls straight into §5's skip path, so this decision governs how that event degrades. -> -> **One gap this closing deliberately left open**, named here rather than implied: ADR-0075's refusal is -> pinned at `gate.ts`'s own catch (where the blocker was) and at `toUserFacing`, but NOT through -> `dispatch.ts` / `specs.ts` to the process exit. A re-wrapping `try/catch` added in either layer reproduces -> the same blocker and leaves both tests green. Closing it needs a `run(argv, io)` drive with a redirected -> HOME and a seeded on-disk `history.db`; no such harness exists for `gate` yet, and the marker sits at the -> test it concerns. -> -> **`#W15-1` is what remains, and its DECISION is now made**: -> [ADR-0076](../decisions/0076-durable-per-attempt-realized-cost-ledger.md) (Accepted 2026-08-09) — an -> additive durable run event recording one settled provider attempt, emitted through ADR-0074 §2's barrier and -> folded into a `run_costs` row at persist time. It landed after ADR-0075 on purpose: a new durable event type -> is exactly the input that decision governs. -> -> **The implementation is staged, and starts here.** In order, because each step is a precondition for the -> next: -> -> 1. `packages/shared/src/run-event.ts` — `cost:attempt_settled`'s schema, its union arm, and -> `parseStoredRunEvent`. Until this lands nothing else compiles. -> 2. `docs/reference/contracts/sse-event-schema.md` — the canonical shape + which of the two cost events is -> authoritative (ADR-0076's last Negative names that drift risk). -> 3. `packages/db/src/run-history-store.ts` — the `applyDerived` arm writing the `run_costs` row in the SAME -> transaction as its event. The `node:completed` delta then telescopes to zero on its own; a test must pin -> that `SUM(run_costs) == runs.total_cost_microcents` still holds, since that is the invariant carrying the -> no-double-count claim. -> 4. The attempt boundary and its barriers — **corrected by -> [ADR-0077](../decisions/0077-realized-cost-ledger-uses-the-conservative-commitment-barrier.md)**. ADR-0076 -> §1 asked for an `await` at the settle point; there is no `await` to be had there. The seam's observer is -> `onAttempt?: (record) => void` — synchronous, void — and BOTH money events are emitted from that one -> callback a few lines apart, which `budget-governor.ts` already states outright ("which cannot await"). So -> the ledger takes ADR-0074 §2's shape: **start** the write at the settle instant on a chained in-flight -> promise, **join** it at three barriers — before the next egress admission, **before tool dispatch** -> (`agent-turn.ts`'s `await dispatchToolCalls(...)`, the barrier this adds beyond §2), and at the turn/node -> terminal. Every barrier awaits AND observes the failure, because `#emitDurable` is total for store faults -> and resolves. -> 5. `packages/core/src/engine/checkpoint.ts` — **corrected during implementation**. The instruction here was -> "fold it as a SUM of deltas", borrowing ADR-0074 §2's reasoning, and the wrong half of that cost two -> rewrites. Summing into the realized total double-counts (a node terminal's snapshot already contains its -> attempts); summing into a separate accumulator and maxing the two families UNDER-counts (a media node -> writes a snapshot and emits no attempt row, so every attempt after the last boundary vanishes). What is -> correct is the fold already two arms above it: `Math.max` over every durable ABSOLUTE total. §2's -> rejection of `Math.max` does not carry — it was rejected there only because a release can DECREASE a -> conservative total, and realized spend is monotonic. -> -> **STATUS: all five steps are landed** (`8d7ffcf` … `50c60bd`), each with an Opus and a Sonnet round folded. -> The break-verify that matters most was step 4's ordering, and it now holds: with the tests draining to -> quiescence before asserting, deleting B2 reddens the tool-dispatch test and deleting B3 reddens the node- -> terminal test. B1 is masked by B2 on the current fixture (the second egress only comes after the tool), so -> it is covered by construction rather than by a red — named rather than implied. -> -> **This is now the prerequisite of Phase 2.6.5, not a parallel track.** Its five steps touch the same four -> files `CR-10` (ordered append tail) and `CR-12` (effect journal) restructure, so it lands first — see the -> 2.6.5 section below. -> -> **What ADR-0076 explicitly does NOT close**, so it is not read as more than it is: the cost of a TOOL EFFECT. -> A resumed run can still re-execute an `http_request` POST, a `run_command` or an MCP mutation — the -> duplicate EFFECT is the harm, and no cost bookkeeping prevents it. That needs a durable effect journal with -> a prepare/receipt pair, which is a different decision about a different failure and earns its own ADR. It is -> the natural next one after this chain (forward-compatibility → conservative commitment → realized cost). -> -> Two decisions inside the closed set are worth surfacing here rather than leaving in a commit message. -> `#W15-4` **changed a previously-tested behaviour**: #228 pinned that a session survives a failed durable -> write and the next turn is clean; it now stops instead, because continuing to spend above a transcript that -> has fallen behind is the defect. And `#W15-23` is scoped to `run-history-store` only — the other -> `packages/db` stores share the same outer-`db`-inside-a-transaction convention, which remains open, is -> stated here rather than implied closed, and has its own PR queued after this one. - -#### A. Needs an ADR first — the only part that is NOT PR #81's to close - -- ✅ **`#W15-1` (blocker) — a workflow's REALIZED cost is not durable at the provider-attempt boundary.** - *Closed 2026-08-10 by ADR-0076 + ADR-0077 and the five staged steps; the barrier is a chain-and-join, not - the inline await ADR-0076 §1 first specified. See the staged plan below for what each step landed and the - three carried-forward gaps.* - `engine.ts` folds `cost:updated` into memory and streams it; the store documents that it is never persisted, - and the checkpoint can only recover cost from a LATER `node:completed`/gate snapshot. So: a paid call - succeeds → the model asks for a tool → the process dies during the tool → the realized cost is gone and the - resumed node re-runs the same paid call, against a cap that has forgotten the first one. ADR-0074 makes only - the CONSERVATIVE commitment durable; a call that returned trustworthy usage has no equivalent barrier. - *Needs an ADR: an idempotent per-attempt durable ledger, written before the next tool side-effect, the next - egress and the turn/node terminal. Not a local fix — it changes what "spent" means across a crash.* -- ✅ **`#W15-2` (blocker) — a tolerant read is fail-OPEN on the resume path.** ADR-0074 §5 drops an event whose - `type` an older binary does not know, and `checkpointer.ts` builds resumable state from what remains. An - older binary cannot know whether the dropped row was a node terminal, a job submission, a gate decision or a - cost commitment — so it may re-run completed or already-submitted work. Preserving the sequence high-water - mark prevents a UNIQUE collision; it does not restore lost state. Tolerance is right for `logs`/`status` - (read-only); it is wrong for a replay that changes the world. - *Needs a superseding ADR (§5 is append-only): fail closed on the resume path with "a newer Relavium is - required" when any row was skipped, while read-only surfaces stay tolerant — or classify events as - state-bearing vs not, and tolerate only the latter.* - -#### B. PR #81 — ADR-0074 §1, the half that makes the rest safe - -- ✅ **`#W15-3`** (blocker) — the release is exposed but unreachable, and nothing renders `durabilityBroken`. - `GovernorWiring.releaseConservativeCommitments` and `conservativeState()` exist and are tested; no command, - REPL action or Home affordance calls either. §1 requires the surface that RENDERS the amount to also expose - clearing it, and this PR removed the accidental escape a restart used to provide. - *Closed by `/cost --release`: `/cost` renders the durability state, and the release clears the per-model AND - aggregate columns before the live governor, so it survives `chat-resume`. The durable - `budget:estimate_released` named here was NOT needed — the session path restores from columns, not from an - event replay — so it stays reserved; see the lane-(e) note above.* - -#### C. PR #81 — data integrity, each with a concrete failure - -- ✅ **`#W15-4`** (blocker) — session persistence failures are invisible to the producer. `RunEventBus` isolates - listener errors by design, so a persister that cannot write still lets the turn complete. Two consequences: - `persister.ts` advances the in-memory cost total BEFORE the DB write, so a failed `recordSessionCost` leaves - the durable total low and a resume restores a cap that has forgotten real spend; and a failed `commitTurn` - leaves the staged turn uncleared, so the next `beginUserTurn` overwrites the previous user message and the - durable transcript loses an exchange the in-memory session still has. - *Closes when: cost and transcript persistence move to an awaited durability port (UI subscribers may stay - passive), and a persistent write failure puts the session in a durability-degraded state that refuses new - egress rather than reporting success.* -- ✅ **`#W15-5`** (high) — stored payloads are not checked against the authoritative columns. `readEventLog` - selects `seq`/`event_type` alongside the payload but never compares them to the parsed event's own `runId`, - `sequenceNumber` and `type`. A row with `seq = 2` carrying `sequenceNumber: 1` is accepted, leaving the - high-water mark wrong and the resume colliding on the next write. - *Closes when: every denormalized projection is compared after parse and any mismatch is a - `CorruptRunEventError`.* -- ✅ **`#W15-6`** (high) — terminal cost snapshots on failed/cancelled runs are dropped by the read models. - `node:failed`, `run:failed` and `run:cancelled` all carry cost in the shared schema, but the store folds - none of them, the checkpoint folds none, and the run TUI's summary ignores the terminal figure. A sibling's - paid media cleanup after a root failure exists ONLY on `run:failed.cumulativeCostMicrocents`, so replay and - history lose it. Pre-dates this PR, but the PR touches all three files and claims cost safety. -- ✅ **`#W15-7`** (medium) — `media_job:submitted` accepts a half-frozen basis. `units` and - `acceptedCostMicrocents` are independently optional, so a row with a frozen cost and no frozen units resumes - with the old reservation and a re-derived volume. The invariant is not "both or neither" (the approved-bypass - case legitimately has units only) but **`acceptedCostMicrocents` present ⇒ `units` required**. - -#### D. PR #81 — security and correctness - -- ✅ **`#W15-8`** (high) — the new failure-reporting paths can leak credentials. `background-failure.ts` prints an - arbitrary rejection message and, under `RELAVIUM_DEBUG`, a raw stack; `sanitize.ts` strips ANSI/control/bidi - but performs no secret redaction. Same class in the session listener notice, `render-error.ts`'s verbose - stack and `index.ts`'s fatal debug stack. The rejection can be an MCP error, a provider response body or a - tool result — all of which carry keys, `Bearer` headers or credentialed URLs. Existing tests plant ANSI, never - a secret. - *Closes when: the shared secret-shape redactor runs BEFORE terminal sanitization on every one of those paths - (stacks included), with tests planting `sk-*`, `Bearer`, `Basic`, credentialed URLs, query tokens and PEM.* -- ✅ **`#W15-9`** (high) — the stream path can throw out of a contract that says it never throws. - `#runStreamAttempt` calls `#emitSuccess` OUTSIDE its try/catch (the `generate()` path has it inside), so a - provider returning non-integer usage makes `assertAccountableUsage` throw raw out of the async generator — - and `streamOneTurn`'s `for await` has no try/catch, so it crashes the turn unclassified, AFTER the content was - produced and paid for. -- ✅ **`#W15-10`** (high) — `--json` output is not sanitized. `status --json` / `gate list --json` carry - model-controlled text; `JSON.stringify` does not escape Unicode bidi overrides (CVE-2021-42574). The human - render path is sanitized; the JSON branch is not. `createJsonRenderer` is the same gap on the NDJSON path. -- ✅ **`#W15-11`** (high) — `createPlainRenderer` sanitizes AFTER splitting on newlines, so an embedded newline - still forges a line. Every other boundary this PR touched sanitizes first; `renderer.ts` is the exception. -- ✅ **`#W15-12`** (high) — `isBlankPreview` treats a fully-redacted preview as informative (`chat-mode.ts`), so a - user pressing "always" on `Approve write to [redacted]?` auto-approves every `write_file`/`run_command`/ - `http_request` for the session, having been shown nothing. -- ✅ **`#W15-13`** (medium) — the config reader chmods PROJECT configs to 0600. `reassertOwnerOnly` runs on every - layer, but `config-spec.md` says project files are committed and shared. Reading a config mutates repo file - permissions, and `statSync`/`chmodSync` follow symlinks. Self-heal belongs to the canonical global config - only. -- ✅ **`#W15-14`** (medium) — a provider `Retry-After` sleep is not abort-aware. `fallback-chain.ts` passes the - delay straight to an injected `sleep` with no signal, and the real hosts use a plain `setTimeout`. A cancel - during a 60 s clamped wait hangs until the timer fires. The existing abort test resolves `sleep` immediately, - so the window is untested. The docblock also says an oversized value becomes `undefined`; the code clamps. -- ✅ **`#W15-15`** (medium) — corruption is reported to a multi-run view as "no data". `readPerRunOrDegrade` - returns only the fallback, so `gate list` can print "No pending human gates." for a run whose gates were lost - to corruption. It must return the degradation alongside the value, with a warning in human output and a - machine-readable marker under `--json`. - -#### E. PR #81 — coverage gaps it shipped knowingly - -All six are now CLOSED and their in-source markers removed. None had been a shipped defect — each had -mutation-verified coverage one layer down — but closing them surfaced one that was: `onLegacyMediaJobHold` -reached `WorkflowEngine` from a real `gate.ts` sentence and the constructor never read it, so ADR-0074 §3's -hold notice was dead and a `relavium gate` resume of a legacy parked media job stalled silently. Found by -the review of `#W15-16`, fixed with it, and now the signal that test drives. - -- ✅ **`#W15-16`** — the engine-level composition test for §3's abort-breakable media-job hold; deleting the abort - listener leaves the whole suite green. -- ✅ **`#W15-17`** — the CLI-layer test that `restoreConservativeCost` is wired through `buildSessionRuntime`. -- ✅ **`#W15-18`** — the `coalesce` backfill of `session_costs.model_catalog_id` (needs a `model_catalog` FK row - the block does not seed). -- ✅ **`#W15-19`** — that a chat turn's terminal is emitted only AFTER `flushBudgetCommitments` resolves. -- ✅ **`#W15-20`** — the H3 approved-bypass `acceptedCostMicrocents` omission; reverting it leaves the suite green. -- ✅ **`#W15-21`** — `agent-run.ts`'s `attachConservativeWriter`; deleting it silently breaks every capped - `agent run`. - -#### F. PR #81 — standards and documentation drift - -- ✅ **`#W15-22`** (low) — the PR's "no unsafe `as`" conformance claim is false. Two assertions in new production - code: `shared.ts`'s `readRetryAfter` casts an arbitrary `headers.get`, and `openai.ts`'s `mapOpenAiApiError` - casts a `headers` field back onto a parameter type that lacks it. A structural `headers?:` plus runtime - narrowing removes both. -- ✅ **`#W15-23`** (medium) — `persistEvent`'s `applyDerived` uses the outer `db`, not the transaction handle. - Harmless under `better-sqlite3`, but it breaks ADR-0005's Postgres-portability promise on the highest-volume - money write. -- ✅ **`#W15-24`** (low) — canonical docs describe code that was not shipped. `tool-registry.md` says path - redaction is per-segment (it is whole-string, and the same file says so correctly three paragraphs later); - `llm-provider-seam.md` still claims model-discovery/media-poll keep a small SDK retry; ADR-0028's amendment - note still says ADR-0074 §2–§5 have not landed. Each could send a future reader back toward a #91-class hole. - -### Phase 2.6.5 — Core reliability remediation (between Wave 1 and Wave 2) - -Three independent core reviews of the post-Wave-1 tree converged — independently — on the **same seven blocking -gaps**: effect journal, stdio MCP consent-before-spawn, run lease, compaction trust elevation, stream grammar, -input admission, event-log ordering. Three separate reviews landing on the same seven points is not opinion. - -The full, self-contained work list is -[phase-2.6.5-core-reliability-remediation.md](phases/phase-2.6.5-core-reliability-remediation.md) — **51 items** (`CR-01`…`CR-98`; `CR-96` and `CR-97` were opened 2026-09-14 from the `W7` ADR reviews, `CR-98` on 2026-09-18) with evidence, fix, acceptance criteria and a decision/ADR/gate register, written so the work -can be done from that document alone. An adversarial plan review on 2026-08-10 corrected the phase boundary, -the exit rule and the execution order, and added two items (`CR-17` resume identity, `CR-63` `input_schema` -docs-only); `CR-64` came from the Batch 1 triage, `CR-21b` from ADR-0082 §10 and `CR-21c` from the `W2` -document review on 2026-08-25, which is why the total has moved since the list was first written. - -> **Historical phase checkpoint — 41 of 51 closed (updated 2026-09-18; at W6's merge it was 39 of 48 — see the register for why not 43: W5 was counted twice). `W0`–`W2` merged clean; `W3` merged 2026-08-30 (PR #86) with one reproduced BLOCKER and nine verified findings open — see the `W3` residuals in [deferred-tasks.md](deferred-tasks.md). `W4` MERGED 2026-09-01 (PR #87) behind [ADR-0088](../decisions/0088-the-mcp-boundary-is-hostile.md): a systematic review of the branch found five merge blockers — an orphaned MCP child on a signal, an optional validated `fetch`, a bypassable transport byte bound, discovery paging past its budget, and two cloud-metadata endpoints reachable through the local opt-in — all reproduced and fixed before it merged. `W5` MERGED 2026-09-02 (PR #88) behind -> [ADR-0089](../decisions/0089-media-correctness-four-boundaries.md) + -> [ADR-0090](../decisions/0090-a-continuation-token-rides-the-part-it-belongs-to.md): all six of -> `CR-50`–`CR-55` closed, two of them for one half of a two-part obligation and saying so in their own -> headings. A systematic maintainer review of the branch returned **six merge blockers** — the async -> media-job result bypassing the first-resolution pin, a run-scoped re-host ceiling that reset on resume, a -> capability gate keyed on the model id alone (breaking custom `base_url`), a negative DB media rate buying -> strict-cap headroom, a load-bearing migration backfill with no upgrade test, and a drifted canonical -> AgentRunner spec — all reproduced and fixed before it merged. -> -> **`W6` — authoring correctness — MERGED 2026-09-04 (PR #89)** — `CR-60`–`CR-64` behind -> [ADR-0091](../decisions/0091-first-means-first-declared-not-first-to-finish.md)–[ADR-0094](../decisions/0094-a-tool-grant-is-checked-when-the-plan-is-built.md), -> 32 commits — 27 of work over seven internal review rounds, then five folding four maintainer rounds on -> PR #89: `first` means first DECLARED and the plan stops claiming otherwise; a -> tool grant is refused when the plan is built rather than mid-run; an expression sees only its transitive -> closure and a literal out-of-closure read is refused at parse; an `output_schema` is compiled at parse in an -> allowlist-strict mode and enforced at run time — with **no new dependency**, because the compiler the -> deferral said we needed already existed one package away. -> -> **What the wave is actually worth recording for is that most of what the review rounds found were defects in -> the FIXES, not in the code they repaired.** The branch-order search was wrong three times in three rounds, -> each time in the mirror of the previous error. The expression scan's central soundness claim — that it never -> produces a false refusal, which is the entire justification for a regex over a parser — was asserted in three -> documents and false in EIGHT distinct ways — the eighth, a Unicode identifier escape, found after the -> wave was declared complete. One stand-down gate, added to prevent a false refusal, silently -> removed the whole check it was protecting. Every one is a dated correction inside its own ADR rather than a -> quiet rewrite, because the shape of the error is the reusable part. -> -> **The maintainer review then found two merge blockers, and both broke a headline claim of the wave.** Deep -> validation did not enforce `required` when the property's schema constrained nothing — `{ properties: { -> status: {} }, required: ['status'] }` accepted `{}`, because an unconstrained property compiles to -> `z.unknown()`, which is optional inside a `z.object`, and presence was checked only for names ABSENT from -> `properties`. And the expression scanner's regex was quadratic and synchronous: 514 ms at 32 K of -> whitespace, on a parser that accepts 2 MiB, with plan build synchronous so no deadline or cancel could -> interrupt it. Both reproduced and fixed before the merge, with eight further High findings — among them a -> crash on `{ type: 'string', format: 'email', maxLength: 254 }`, which is the exact shape the new canonical -> subset document tells authors to write, caused by an `as unknown as` cast that CLAUDE.md rule 1 forbids -> precisely because it turns a type error into a runtime one. -> -> **Eleven residuals** are recorded in [deferred-tasks.md](deferred-tasks.md) — ten under the four `W6` -> residual headings plus ADR-0091's zero-branch-merge question, filed with the fan-in items. Two await a -> maintainer ruling. -> -> **The `W4`-waits-for-the-`W3`-blocker gate was lifted deliberately on 2026-09-01, by the maintainer, and it is recorded here rather than left to be inferred from the commit order.** The reasoning: the blocker is a `BoundedEventStream` drop on a stream nobody iterates, its fix is ADR-0087 §1 — a decision that was still **Proposed** and unapproved — and `W4` touches a disjoint surface (the MCP boundary). Blocking a whole wave on an unapproved ADR would have stalled the phase rather than protected it. **The blocker stays open and stays first in the `W3` residuals**; nothing here closes it or reduces its severity. **Updated 2026-09-04:** ADR-0087 is now **Accepted**, with nine dated corrections written into it after a review refused acceptance. That removes the "unapproved ADR" half of this gate — §1 is now a settled decision — but §1 remains **unimplemented** and the blocker is unchanged. Its Correction 1 records what §1 must settle before it can be built: the two factories §1 names are not symmetric — a host calls `createSessionHandle` directly, but a `RunHandle` is only ever produced inside `WorkflowEngine.start()` / `resumeFromCheckpoint()`, so the run half needs the mode plumbed onto an engine entry point before it can be passed at all. A `subscribe-only` handle must also not present a silently-empty iterable. -> -> - **Batch 1, merged 2026-08-11 (PR #82)** — the prerequisite (`#W15-1`), the oracle (`CR-90`, `CR-91`) and -> all of `W0` (`CR-01`–`CR-03`). `CR-64` was added in the same batch and shipped 2026-09-02 with `W6`. -> - **Batch 2, merged 2026-08-24 (PR #83)** — all of `W1`: the eight P0 blockers plus `CR-92`, behind -> ADR-0078…ADR-0084. A comprehensive review of the assembled PR found seven further defects, six of them in -> the W1 code itself; all seven were fixed and mutation-verified before merge. -> -> - **Batch 3, merged 2026-08-28 (PR #85)** — `W2`, liveness and deadlines, behind -> [ADR-0085](../decisions/0085-the-node-executor-owes-liveness-and-the-engine-enforces-it.md). All six -> items closed; the wave's own closing register is in the phase document. The assembled PR drew a -> **Request changes** verdict with fourteen findings, five of them blockers, plus a later round of eight; -> all were verified against the code before being acted on, and the register records which were fixed and -> which were declined with the reason. The five blockers had one root cause — an ownership decision taken -> at a point in time, guarding work that spans an await. -> Its document review moved the count twice, and the two moves **cancel**: `CR-21` turns out to have -> shipped **with `CR-14`** in PR #83 (ADR-0082 decides both, and the execution graph above already -> scheduled them together) and only its heading was stale, while `CR-21c` — an unbounded individual -> `pollMediaJob` call — was found during the same review and added. `W2` was therefore still **five** open -> items out of six, not four; the phase total moved 47 → 48, not the wave. All five then closed on -> 2026-08-27 — `CR-22`, `CR-21b`, `CR-21c`, and finally `CR-20` and `CR-23` together, which -> [ADR-0085](../decisions/0085-the-node-executor-owes-liveness-and-the-engine-enforces-it.md) decides as -> one because they are one mechanism at one place. -> -> **Not to be confused with the 2.5.5 remediation's own `W2`** ("Shut the doors", in the graph above). Same -> label, different phase: this one is 2.6.5's liveness wave. -> -> Per-item history, the seven post-review findings and the carried-forward gaps live in the phase document. - -> **Pre-`W7` close-out, 2026-09-06.** *(Superseded in part on 2026-09-14: `CR-96` and `CR-97` reopened exit criterion 1, -> the count is 41 of 51, and `CR-94` is decided and scheduled into `W7` — see the next block.)* The last two non-deferrable items closed, so **exit criterion 1 is met** -> and the phase count is **41 of 48** (40 ✅ headings plus `CR-95`'s short-term half; the arithmetic is shown -> because it has been wrong twice here). `CR-73`: `invoke_agent` was advertised to every model granted it while -> its delegate is wired nowhere in the tree, so the advertise-filter now reads a tool's declared -> `requiresDelegate` and not only its `ToolHost` arm — `read_media` is dropped for the same reason, which is -> `CR-50`'s own argument for leaving it unwired. `CR-80`: a rejected custom `base_url` was caught and the -> DEFAULT adapter left standing, which is the official API; it now installs a refusing adapter that names the -> URL's shape and never its credentials. **Criteria 5 and 7 also closed**: the three missing security sittings -> (prompt/trust provenance, hostile MCP, provider/config trust) and the `W5` and `W6` closing registers are -> written, each stating that it was written after its wave merged rather than with it. **Criterion 6** passes -> on both halves, read from the exit code — `pnpm run ci | tail` reports `tail`'s status and produced one false -> green in this session. Still open: criterion 2 (`CR-94` has no deferral record, and that is a maintainer's -> call) and criterion 4 (the `W3` live blocker keeps the gap-free stream promise wider than the code). -> *(Criteria 2 and 4 have both moved on: `CR-94` was decided 2026-09-13 and its remainder now has a deferral -> record, and ADR-0087 §1 — criterion 4's blocker — is scheduled into `W8`. Criterion 5, called closed above, -> REOPENED on 2026-09-18 with a fifth sitting, `history.db` at rest.)* - -> **`W7` decisions made and accepted, 2026-09-14.** -> -> **What was decided.** The maintainer settled `CR-70`–`CR-72` and `CR-94` on 2026-09-12 and 2026-09-13. Four ADRs -> were **accepted** on 2026-09-14, with their implementation staged in `W7`: -> -> - [ADR-0095](../decisions/0095-what-an-agent-session-remembers-across-turns.md) decides that tool history is -> persisted as structure, never content; that carrying it into the model's context is deferred; and that `memory` -> is implemented. `memory` stays inert until `W7` lands. -> - [ADR-0096](../decisions/0096-a-request-is-measured-before-it-is-sent.md) decides that a request is measured -> before it is sent, that a context overflow is classified and recovered only before any tool runs, and that input -> is priced. Until `W7` lands, the estimate still prices output only. -> - [ADR-0097](../decisions/0097-a-budget-approval-is-an-allowance-not-an-exemption.md) decides that a budget -> approval is a dispatch-owned allowance, shown before it is granted. Until `W7` lands, an approved step still runs -> uncapped. -> - [ADR-0098](../decisions/0098-a-session-effect-row-holds-no-result-and-never-replays.md) decides that a session's -> effect row holds no result and never replays. -> -> **What the reviews found.** The first six review rounds shaped the ADRs; a seventh, on 2026-09-18, reviewed them -> against the tree before any code (see the block below). The six surfaced two shipping defects, now -> register items `CR-96` and `CR-97`. Both meet the phase's non-deferrable definition, so exit criterion 1 is open -> again, and the count is **41 of 51**. -> -> **What `W7` now covers:** `CR-71`, `CR-72`, `CR-98` (ADR-0096's prerequisites, opened as an item 2026-09-18), -> `CR-94`, `CR-96` and `CR-97`, plus `relavium budget resume`, which moves here from 2.6.K because ADR-0097's -> shown-amount guarantee otherwise has no non-TTY surface. Five canonical claims that were already false are -> narrowed ahead of the code: -> -> - the session spec's tool history and export `tools` union; -> - `memory`'s default; -> - the uncapped approved budget step; -> - the SSE schema's "user/assistant/tool messages are persisted" sentence (2026-09-18); -> - the chat reference's "full transcript" in the export (2026-09-18). -> -> *(This `W7` is Phase 2.6.5's seventh wave. It is not the remediation's "Wave 7 — Orchestration & the gate" in -> the graph at the top of this file, exactly as `W2` is not Wave 2.)* - -> **`W7` document review, 2026-09-18 — before any code.** Eight dimensions reviewed the four ADRs, the register, -> the deferrals and every canonical document they touch against the tree; each finding was verified by an -> independent adversarial pass. 113 findings held, and the maintainer settled seventeen decisions the same day. -> Every ADR carries a dated note; the register carries a per-document landing checklist. Three document defects -> mattered most, and all three would have been discovered mid-implementation: -> -> - **ADR-0098's join does not exist.** It says the effect journal already records the tool-call id; every session -> surface writes a wiring constant instead, so `W7` must thread an engine-assigned id through the dispatch port. -> - **ADR-0098's monotonic turn key could not be derived** from the journal, because the sweep deletes the rows the -> high-water mark would be read from. It becomes a durable column on the session row. -> - **ADR-0096 §1's per-part floor** would have charged an inline image megabytes of tokens, contradicting the -> per-modality media ceiling in the same section. -> -> Two of the seventeen decisions change what ships beyond those ADRs: `history.db` opens with -> `PRAGMA secure_delete = ON`, so "no tool result at rest" is true of the file's bytes and not only of the row; and -> a pre-content HTTP 4xx now releases its admission instead of committing an input-sized estimate, which keeps a -> rate-limit burst from exhausting a session's cap once input is priced. ADR-0087 §1 — the `W3` live blocker that -> belonged to no wave — is scheduled into `W8`. - -**This is the corrected execution order, and it is what the graph above shows:** - -1. **`#W15-1` first.** Its five staged steps touch `run-event.ts`, `engine.ts`, `checkpoint.ts` and - `run-history-store.ts` — the same four files `CR-10` and `CR-12` restructure. Landing it after the durability - spine means writing its barrier against a persistence path that is about to change. -2. **Then Phase 2.6.5, closing before Wave 2 opens.** Which is only true if the work Wave 2 would block on - lives there — so **the hostile-MCP threat class moves into 2.6.5**: `CR-16` (consent before a stdio spawn) - plus `CR-40`–`CR-42` (transport cancellation, DNS/redirect SSRF, ingress bounds). Wave 2's queue item 2 is - **executed as 2.6.5's `W4`**; the 2.5.5 finding ids keep their home in the 2.5.5 phase doc and certify from - there. Wave 2 keeps the MCP name-collision/tool-poisoning trust items, the fs jail, the secrets layer, - persistence-security and the certification. -3. **Within 2.6.5, the oracle comes before the spine.** `CR-90`/`CR-91` (crash + durable-truth harness) land - first because they are the instrument `CR-10 → CR-11 → CR-92 → CR-12` is proven with; `CR-92` moved out of - the harness group into the spine, since an API returning `completed` while history says `failed` is a - runtime defect, not a test concern. - -Two rules that changed with the plan review: - -- **Nineteen items are non-deferrable** — every W0/W1 item plus `CR-50`, `CR-55`, `CR-73`, `CR-80`, `CR-92`, `CR-95`'s short-term fix, and (added 2026-09-14) `CR-96` and `CR-97`. Each has a cheap fail-closed option (refuse, remove, narrow the claim), so "too big - to fix now" argues for the cheap option, never for deferral. The previous exit criterion permitted deferring - all 43 items and declaring the phase complete. -- **The gate is `pnpm run ci` AND `pnpm coverage`.** `coverage` is a separate required CI check and is not - inside the `ci` script; this phase edits `packages/core` heavily, and the CI coverage job does not block on - `core`. - -`CR-01`–`CR-03` were half-closed by Wave 1 and finished right after the oracle — **all three are closed as of -2026-08-11** (PR #82). `CR-03` was a propagation gap from `#W15-10`'s own fix: the finding named three `--json` -paths that never got the safe serializer, and closing it found **five**, which is why the call-site half is now -an ESLint selector rather than a list. +**Pulled forward into the MVP (the waves below keep the rest):** the `engine.reconcile()` wiring (#229), the +gate-decision/`gateType` check (#0) and the run-path tool pre-flight (#37/G29) (Wave 4a → MVP-1); 2.6.A's +back-port and catalog pre-flights, 2.6.B and 2.6.D (Wave 4a/6 → MVP-6); the toolbelt's edit/search/find tools, render v2 slices, the default grant and +`web_search` (Wave 6 → MVP-3/7); 2.6.G, 2.6.H and 2.6.K as far as the Home hub needs them (Waves 4a/5b → MVP-8); +2.6.I (Wave 5b → MVP-9); #61 word-wrap (Wave 5a → MVP-2); #172 prompt caching (Wave 4b → MVP-5); the v0.2.0 +release (Wave 4b → MVP-10). After the MVP, a wave takes up whatever of its items the MVP did not finish. + +**Remaining product lanes after v0.2.1 (phase 2.6, ordered):** sub-agents and the rest of orchestration +(2.6.N → 2.6.O/2.6.P, Waves 6–7); `/rewind`/`/fork` and persisted input history (2.6.E); `todo` and `ask_user` +(2.6.M, ADR row 9 part 2); syntax highlighting (ADR row 11 part 2); settings, themes and `en`+`tr` (2.6.L, +Wave 5a/5b); onboarding v2's account stub (2.6.J); history retention and delete (D33, Wave 4b); scope-line items +only after a phase amendment and an ADR. + + + + + + ### Wave 2 — Shut the doors -**Complete on `main`, 2026-10-15, merged as PR #94 (`ab08209e`).** The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) joins all forty queue IDs, the exit/D sign-off and final correction evidence. Hostile-MCP work is reused from W4/PR #87; the remaining filesystem, secrets/config, persistence and media work is complete within its recorded limits. The queue below preserves scope and traceability. - -1. ✅ **Completed in W4/PR #87:** 2.5.5.D · project-layer MCP name collision redirecting a global secret - (G19) and MCP tool-definition poisoning (#202); see the - [W4 closing register](phases/phase-2.6.5-core-reliability-remediation.md#w4-closing-register). -2. **MCP queue, strict — EXECUTED AS PHASE 2.6.5 `W4`, and COMPLETE there (2026-09-01, [ADR-0088](../decisions/0088-the-mcp-boundary-is-hostile.md)).** The queue below landed in five steps, plus `#202` and `G19` from step 1 of this wave and the MCP half of `#21` from 2.5.5.E — all the same threat class and the same security sitting, so splitting them would have booked the reviewer twice. Certify the 2.5.5 ids from the `W4` closing register in [phases/phase-2.6.5-core-reliability-remediation.md](phases/phase-2.6.5-core-reliability-remediation.md). - *The original queue, for the record:* Connect-phase timeouts on every transport - (#35, G32, #205) → transport/discovery/result size bounds against a hostile server (G33, #201, #209, #288) → - structured `serverId`/`reason` discriminants (#203, #204) → the two fail-loud violations (#206, #207) → - per-file transport-adapter tests (#297) → `CLIENT_INFO.version` (#208). These finding ids keep their home in - [phases/phase-2.5.5-hardening-and-remediation.md](phases/phase-2.5.5-hardening-and-remediation.md) and - certify from there in step 6 below; the *work* happens in 2.6.5 alongside `CR-16` (consent before a stdio - spawn) and `CR-41` (the DNS/redirect SSRF hole), because they are the same code and the same security - sitting. Splitting them across two phases would book the hostile-MCP reviewer twice. -3. ✅ **The fs jail, ONE reviewed change set — complete on `main`, PR #94** — 2.5.5.D · sensitive-credential floor for `.kube`/`.azure`/gcloud - (#36, #39) → extract `deepestExistingReal` to `packages/shared` (#235) → **2.6.M's `project`-tier - `extraRoots`** and **2.6.N's fs-floor home-anchoring**, both *pulled forward*. `~/.relavium/tmp/` (G21) - is resolved under the same general predicate (decision D12) — otherwise `fs.ts` takes two security reviews. -4. ✅ `discoverCatalog` realpath containment (#1, accepted in Step 2); the secrets-layer test pair — OS-keychain bridge (#27) and - `readSecretFromStdin` + stdin cap (#31, #293); secret-typed `run --input` rejection (#25); the unredacted - workflow-snapshot doc + lint (#30); 2.5.5.B · `scrubSecrets` param-name broadening (#196). -5. ✅ Persistence-security: 2.5.5.C · `assertRelativePath`'s Windows drive-relative form (#113), media-store - hardening (#111, #108, #112), media `mimeType` verification (#107, #109, #110), and certification of - W7's structural session/tool-content controls (#115, Wave D15). 2.5.5.A · chat-export continuity/slug - completion and existing tool-content exclusion (#80, #81, #83, D14), plus `analyzeSecretTaint`'s - context sites (G22). 2.5.5.D · the dormant `default_headers` admission/custody gate (G20, D13). -6. ✅ **Phase 2.5.5 exit criteria 1, 2 and 3 and sub-stream D acceptance are satisfied on `main` by PR #94.** The D half of M2.5.5-2 is complete; the I half remains open. - -> Book **four security-review sittings by threat class** (fs/path-jail · secrets-at-rest · secrets-input · -> config-trust), not ~30 per-PR passes. The **hostile-MCP** sitting moved to Phase 2.6.5 with the queue-2 work; -> it covers `CR-16` and `CR-40`–`CR-42` there. Reviewer availability, not code, is this wave's critical path. +✅ **Merged to `main` on 2026-10-15 as [PR #94](https://github.com/HodeTech/Relavium/pull/94) (`ab08209e`).** +The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) joins +the forty item dispositions, the security sittings and the final evidence. Phase 2.5.5 exit criteria 1–3 and +sub-stream D are satisfied on `main`; M2.5.5-2's I half stays open. ### Wave 3 — Clear the ground -**Implementation in progress — 2026-10-18.** ADR-0110–0113, their linked -contracts and the nineteen-increment plan were approved on 2026-10-16. -[Steps 1–8 are independently accepted](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) -on `development`, with whole Step 8 reviewed at `c83c8575`. The phase register -owns each increment's source, tests, cumulative review rounds, security sittings -and platform/publication evidence, including historical NONCLEAN R1–R24 records. -This live page links that history instead of repeating its superseded pending states. - -The maintainer authorized the initial completed-Steps-1–8 publication checkpoint, -[PR #95](https://github.com/HodeTech/Relavium/pull/95). The -[previous corrective review completion](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-corrective-review-completion--2026-10-18) -remains attributed to `ba14b1bd`; it does not certify later changes. The second -maintainer systematic report reviews `a5a5d45e` and opens further corrections. -The [second correction register](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-second-systematic-review-corrections--2026-10-18) -is canonical for that work. The -[second corrective source-review completion](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-second-corrective-source-review-completion--2026-10-18) -accepts corrections at `ec0f3e10` after two NEW complete cumulative clean rounds -and a NEW separate clean security sitting. Current source-bound PR/push CI, -coverage, SonarCloud, same-archive consumers and physical control artifacts remain -required publication criteria; PR #95 owns their live result. Source review does -not imply merge or whole-wave acceptance. The later -[native Windows admission preparation](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-native-windows-admission-preparation--2026-10-18) -retains the failed advisory attempt and separate test-preparation review scope; -production runtime is unchanged. The later [supplemental fixture acceptance](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-supplemental-native-fixture-source-acceptance--2026-10-18) -records both owned-preparation and native-concurrency review scopes, including -both failed Windows attempts. PR #95 owns final pushed-head publication results. -Steps 9–19 have **NOT STARTED** and whole-wave acceptance remains Step 19. Check PR #95 before later implementation commits, -which extend its review scope. No new decision, credential, billing or provider -capture gate is introduced. - -Decompose, guard and normalize every hot file a 2.6 workstream is about to land on. -**The multi-file queue is Step 1 and runs first.** The approved -[nineteen-increment execution plan](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) -owns the full order and review gates: finish each increment and its fresh reviews -before advancing; no parallel edits of shared hot files. - -The groups below retain the original scope inventory and within-file dependency -arrows, not independent scheduling permission or current completion claims. The -[source-corrected preflight dispositions](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-preflight-and-proposed-execution-plan--2026-10-15) -own whether/how each original finding needs implementation or preservation. - -Two orderings here are not this plan's invention — they are the intra-phase serializations -[2.5.5 records under Positioning](phases/phase-2.5.5-hardening-and-remediation.md#positioning): **E before I** -on `home-controller.ts`/`chat-ink.tsx` (the decomposition moves already-guarded code), and **C before E** on -`packages/db/src/client.ts` (C types the error path in Wave 1; E's `openLocalDb` rewrite lands last, here). - -- ✅ **Multi-file Step 1 accepted — 2026-10-16** ([evidence](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-1-acceptance--2026-10-16)) — 2.5.5.E · `closeQuietly()` to its four siblings (#22) → `displayPath` - discipline to `resolve.ts`/`catalog.ts`/`chat-export.ts` (#4, G11) → `const exhaustive: never` across the - six sites (#300, #301, G3, which also makes `relavium run` render `agent:reasoning`). -- **`chat.ts` (strict)** — 2.5.5.G · the four money/token reimplementations (#236, #237, #242, #307) → - `swapAgentModel` → `reseatAgentModel` (#218) → 2.5.5.E · SIGINT actually calling `session.abort()` (#21, #23) - → `budgetWarningText` → **2.5.5.I · decompose `chat.ts`** (#220, #244). Then **2.6.M's `allowedDomains` - chat gap (G30)**, pulled forward into `session-host.ts`. -- **`home-controller.ts` (strict)** — 2.5.5.E · `selectGatePrompter` test (G37) → the swallowed palette - rejection (#281) → **2.5.5.I · decompose `createHomeController` + the overlay discriminated union** - (#215, #243) → 2.5.5.G · the `Capability`/`Port` vocabulary split (#217, #222). - Settle #125/D26 before decomposition: the approved preflight certifies the planned scope of the real - in-Home-chat cost path and retaining chat-only scope. If a new bare-Home cost - surface is selected, wire its explicit session target before decomposition. -- **`chat-ink.tsx`** — 1,906 lines at preflight (the original queue recorded 1,795), decomposed by **neither** phase doc. - **Recommendation: decline** a new decomposition (decision D18); take only the exhaustiveness guard and the - min-terminal-size guard (#63), and hand restructuring to 2.6.M's render-v2 in Wave 5a. -- **Viewport (strict)** — 2.5.5.I · share the ADR-0069 grapheme segmenter (#58, #62, G40, G51) → - 2.5.5.G · tab expansion before width measurement (#65) → per-cluster width cache + resize debounce - (#64, #66), refined to frame coalescing in the proposed plan. Alt-screen word-wrap - (#61) retains the approved Wave 5a ownership; it is not an unapproved Wave 3 prerequisite. -- **`packages/shared` (strict)** — the `.code` ruling (G50, D10) → `findDuplicates`/`superRefine` implementation or preservation at the - schema dispositions (#89, #90, G16, G17, G18, #210, #211, #212) → `BaseEventSchema` disposition (#213) → - the `RelaviumError` base class (#219). *The ruling must precede the base class, or it is baked in permanently.* -- **`registry.ts` / `builtins.ts` / `engine.ts`** — declared-source context repair and honest unsupported-mapping disposition (#92/Accepted ADR-0113) → `ctx.fsScope` (#93) → - `ok`/`success` (#216, #221) → identifier-sort `localeCompare` pinning (#258, #259) → the `git_commit` - denial copy (#308); then the four schema length caps (#95), the `llmVisibleParams` rewrite (#96, #98), - stale doc comments (#78, #97, #214); then `#armRunTimeout` (#75), `#dispatch` dead branch (#79), - `AbortSignal` into `resolvePrompt` + the human-gate recheck (#76, G9). -- **Process/CLI boundary (strict head)** — the revised plan serializes pure flag/descriptor/owner substrate → lazy guarded runtime graph → native crash integration → map/archive custody → command behaviour. It covers 2.5.5.E · the five crash/signal net gaps (#40, #41, #42, #224, - #269), the `--help`/`--version` historical ~400 ms tax (#43, #44, #271), and the flag-system ruling (#45, #264, - #265, #268), the exit-code and `--help --json` rulings (G35, #13), durable-only logs preservation (G15 refuted), the `--input` - namespace ruling (#18, #310). **`openLocalDb` wrap-and-redact (#278, #280) lands LAST** — it touches 8+ - call sites the other queues are still moving. -- The numbered plan assigns the remaining 2.5.5.A/E/G/I tails and keeps the central - DB refactor last before whole-wave acceptance. Step 19 owns sub-stream **A, E - and G sign-offs**, the reconciled **EXIT:4 reading** (D22), and **M2.5.5-3** - closure. Full I acceptance remains in Wave 5a. +**Paused on 2026-10-26 after Step 9b.2; 25 of 32 increments pending behind the MVP.** Steps 1–8 merged to `main` +as [PR #95](https://github.com/HodeTech/Relavium/pull/95) on 2026-10-18 ([merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18)); +PR95, 9a.1, 9a.2a, 9a.2b, NI, 9b.1 and 9b.2 are accepted on `development`, each with its acceptance record in the +[register](phases/phase-2.5.5-hardening-and-remediation.md). The +[increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) owns the +remaining increments' scope and order. Whole-wave acceptance stays Step 19. + +**Recorded splits moved into the MVP (D65, ID coverage unchanged):** row 10b's excess-argument refusal and +row 15a's #65 (MVP-2), row 15c's #63 (MVP-3), row 11b's Ctrl-C at a pending approval (MVP-4, now under the +keymap ADR), row 16b's #51 and row 17a's #24 (MVP-2), and G30's rows 13a–13c (MVP-7). Each keeps its recorded ADR +amendment and security sitting; the [priority reset record](phases/phase-2.5.5-hardening-and-remediation.md#mvp-priority-reset--2026-11-04) +records the within-row order amendments. 9c and 9d (with Wave 4b's production recipient) stay here under release +path B (D59), until 1.0 or until they land. + +Two serializations survive the pause: **E before I** on `home-controller.ts`/`chat-ink.tsx` and **C before E** on +`packages/db/src/client.ts`, as [2.5.5 records under Positioning](phases/phase-2.5.5-hardening-and-remediation.md#positioning). +Experience code from the MVP lands on the undecomposed `chat.ts` and `home-controller.ts`; Steps 12 and 14 rebase +over it when they resume (accepted cost, D65). ### Wave 4a — The spine @@ -1230,7 +252,8 @@ adapter/catalog money floor, cut the release 2.6.Q P6 is gated on. seeded-then-migrated harness #291; …) · 2.5.5.H's remainder (engine-deps allowlist #313/#318/#285; the docs-style checker #316; the logging-standard ruling #266/#267/#270; `fast-check` pilot #299) · 2.5.5.F's 20-site not-yet-built hedge sweep (#122 …) **rebased onto Wave 0's edits, not reverting them**. -- **Sub-stream B, C and H acceptance sign-offs**, then **cut and publish v0.2.0**. **Closes M2.5.5-4.** +- **Sub-stream B, C and H acceptance sign-offs. Closes M2.5.5-4.** *(Moved 2026-11-04: v0.2.0 is cut by the CLI + MVP's MVP-10, not here; M2.5.5-4's acceptance never contained it.)* ### Wave 5a — Paint before you build @@ -1311,7 +334,7 @@ unanswered**; the remainder sit inline in their own phase-file bullet. | D3 | 0 | Accept the upstream price changes the ADR-0071 §9 guard is refusing? | ✅ **Ruled + executed 2026-07-26: take current prices.** Verified: `gemini-flash-latest` moved $0.30→$1.50 in / $2.50→$9.00 out (the shipped floor under-priced by 5×) and `gemini-flash-lite-latest` $0.10→$0.25 / $0.40→$1.50. The nine upstream retirements were accepted in the same ruling; snapshot regenerated and merged in **PR #79**. Nothing outstanding | | D4 | 0 | Confirm the number reservation (`0013`–`0017`, ADR-0073+) | As listed — one item per number, in landing order | | D5 | 0 | The binding locale bar | ✅ **Ruled + propagated 2026-07-26: ship `en` + `tr`**, catalog architected for n locales, `es`/`fr`/`de` staged. Amended at all seven sites | -| D7 | 0 | Publish v0.1.1 as-is, or supersede with v0.2.0? | v0.2.0 — ADR-0067's Node `>=22` bump is breaking for 0.x | +| D7 | 0 | Publish v0.1.1 as-is, or supersede with v0.2.0? | v0.2.0 — ADR-0067's Node `>=22` bump is breaking for 0.x · **Refined 2026-11-04 by D58/D59** | | D8 | 1 | Do already-persisted approval previews need a scrub? | ✅ **Ruled 2026-07-30: NO scrub migration.** The recommendation had been "yes, migration 0013, same PR"; the ruling went the other way, so `0013` was left free by this ruling and is now taken by ADR-0074 §4's conservative money columns; the reservation ledger above shifts nothing — `0014` (#101) keeps its number. The forward fix shipped (#91: `previewFor` scrubs the display copy, `unredactedPreview` carries the in-process classification copy), so no NEW row can carry an unscrubbed preview; historical rows in a local `history.db` keep theirs. Recorded here because the shipped state contradicted this table for four days | | D10 | 1 | `BudgetExceededError`/`BudgetPauseError`: adopt `.code`? | ✅ **Ruled + executed 2026-07-30: adopt.** Both carry a stable `code` (`budget_exceeded` / `budget_paused`), with exported `isBudgetExceededError` / `isBudgetPauseError` guards — because `instanceof` is unsafe from a surface (the CLI bundles the engine, so two realizations of a class coexist and `instanceof` answers `false` at the boundary that must catch it). Same shape as `isCorruptRunEventError`. Precedes Wave 3's `RelaviumError` migration as required | | D12 | 2 | `~/.relavium/tmp/`: `tmp`-specific rule or the general home-anchored predicate? | General predicate accepted in [ADR-0105](../decisions/0105-filesystem-roots-are-invocation-grants-not-project-authority.md), with invocation authority and removal of unused tmp provisioning; approved 2026-10-12; implementation accepted in [Step 2](phases/phase-2.5.5-hardening-and-remediation.md#step-2-acceptance--2026-10-12). Full child/scratch lifecycle remains with Phase 2.6.N. | @@ -1329,13 +352,28 @@ unanswered**; the remainder sit inline in their own phase-file bullet. | D33 | 4b | Retention policy (the phase's largest single item) | Default bound 50 + `--limit`/`--tail`; ship `chat delete` writing the dead `deleted_at` columns; never auto-hard-delete | | D34 | 4b | Logging standard: build the injected-logger seam, or amend the standard? | **Amend** — building it adds a port to `packages/core`, needs its own ADR, breaks 2.5.5 exit criterion 8, and collides with 2.6.N's artifact-store port | | D37 | 4b | Provision CI provider keys for the live-nightly lane? | Yes, nightly-only, wired **before** the parity sweep | -| D50 | 6 | ADR row 11: adopt `ink-syntax-highlight`, or build in-house? | **Decline the dependency** per CLAUDE.md rule 2 — the markdown renderer is already in-house and untrimmed `highlight.js` is the phase's own named bundle risk | +| D50 | 6 | ADR row 11: adopt `ink-syntax-highlight`, or build in-house? | **Decline the dependency** per CLAUDE.md rule 2 — the markdown renderer is already in-house and untrimmed `highlight.js` is the phase's own named bundle risk · **Corrected 2026-11-04:** no markdown renderer exists yet; MVP-2 builds a minimal in-house one (ADR row 11, part 1) | | D52 | 6 | Draft ADR row 12 in Wave 6, not Wave 7 | Yes — settle the lineage event shape once, covering both 2.6.N and 2.6.P | | D57 | 7 | Dynamic runtime sub-workflow selection? | **Defer past the phase** — it is exactly the unauthored composition the *authored surface == executable surface* invariant exists to prevent | +| D58 | MVP | MVP scope | ✅ **Ruled 2026-11-04 — the larger MVP:** v0.2.0 on npm (milestone M3.5) after MVP-1…MVP-10, including conversational authoring, web, the Home hub and MCP/provider management; refines D7 | +| D59 | MVP | Release path under the interim guard | ✅ **Ruled 2026-11-04 — option B:** a superseding pre-1.0 release ADR publishes the smoke-verified package-test tarball from a trusted `v*` tag on `main` with a green `ci` at that commit; no archived encrypted maps for those releases; custody becomes mandatory again at 1.0 or when 9c/9d land. The guard stays in force until that ADR is Accepted and its guard change passes a security sitting | +| D60 | MVP | `LICENSE` §2 grants no usage right without a written agreement | ✅ **Ruled 2026-11-04 — unchanged.** Recorded risk for the published package | +| D61 | Gate 0 | `development`→`main` checkpoint before the MVP | ✅ **Ruled 2026-11-04 — yes,** a scoped checkpoint PR right after this reset; merging is the maintainer's action | +| D62 | MVP-3 | Default chat agent posture | ✅ **Ruled 2026-11-04:** read/search/find plus `write_file`/`edit_file` (hidden in ask/plan, per-call diff approval in accept-edits) plus `run_command` under D63; egress stays opt-in; the default mode stays `ask` with a `[chat].default_mode` setting and a one-time hint. Needs an ADR-0057 amendment and a security review | +| D63 | MVP-4 | `run_command` approval semantics | ✅ **Ruled 2026-11-04:** a typed card for any non-allowlisted command — once, session or remember; remembered grants live outside the repository in operator-owned config keyed by project root; accept-edits keeps prompting; headless stays fail-closed. New ADR (amends ADR-0029(a)/0061/0063) and a security sitting | +| D64 | MVP | Acceptance bar for MVP increments | ✅ **Ruled 2026-11-04 — tiered** (see [Next work](#next-work--the-cli-mvp)) | +| D65 | MVP | Paused Wave 3 increments' user-visible pieces | ✅ **Ruled 2026-11-04:** six recorded splits (10b, 15a, 15c, 11b, 16b, 17a) and #61 move into the MVP; Steps 12/14 stay paused | +| D66 | Gate 0 | `current.md`'s history | ✅ **Ruled 2026-11-04:** relocate the unique records, delete the duplicated history, keep every inbound anchor | +| D67 | MVP-4 | Context fidelity and "a user deny is final" | ✅ **Ruled 2026-11-04:** the CR-71 residual un-deferred into MVP-4 (content-free marker; CR-71 itself closed in W7), denials returned to the model, finite configurable ceilings; CR-70 with ADR-0096 §5 in v0.2.1 | +| D68 | MVP-1/2/4 | First-run defaults and the keymap | ✅ **Ruled 2026-11-04:** env-var key fallback, the interactive output budget, the default model following the stored key, a minimal `init`, and one keymap ADR (Ctrl+C interrupts the turn, a second press ends the session, Ctrl+C at a pending approval rejects it first) superseding the 2026-10-19 `/cancel` wording | +| D69 | MVP | Order of the added lanes | ✅ **Ruled 2026-11-04:** authoring → web → Home → MCP | +| D70 | MVP | npm pre-release before v0.2.0 | ✅ **Ruled 2026-11-04 — none** | +| D71 | MVP-5 | Parity items (caching, project instructions, type-ahead, `--model`, one-shot) | ✅ **Ruled 2026-11-04 — in the MVP** | +| D72 | — | The published `0.1.1`'s unsanitized `relavium run` output (#56) | ✅ **Ruled 2026-11-04 — no action;** the product has no users yet | ### Departures from the phase files' own sequencing -This ordering is compatible with both phase files' stated dependencies. It departs from their prose in six +This ordering is compatible with both phase files' stated dependencies. It departs from their prose in seven places, each deliberate: 1. **A baseline step neither file has.** Both treated `f88b0e8` as the working tree while it was not yet on @@ -1350,727 +388,41 @@ places, each deliberate: 5. **2.5.5's own eight exit criteria are certified nowhere in either file.** Criteria 1–3 are attached to Wave 2 and 4–8 to Wave 7; per-sub-stream acceptance sign-offs are attached to the last wave touching each. 6. **`chat-ink.tsx` is decomposed by neither file** despite being the largest TUI file. Raised as decision D18. - -## Where we are - -**Phase 1 — Engine and LLM is COMPLETE** (milestone **M2** reached, PR #27, 2026-06-16; -all workstreams 1.A–1.AH merged through PR #38, 2026-06-21). The engine runs end-to-end: -YAML parse → DAG → run loop → node execution (agent + the six non-agent handlers) → -checkpoint/resume → node retry → provider failover, with per-attempt cost tracking and a -gap-free `sequenceNumber`. The `@relavium/llm` seam is frozen and proven behind a -provider-agnostic `LLMProvider` interface with all three adapters (Anthropic, -OpenAI/DeepSeek, Gemini) and the `FallbackChain` runner. Two additive sub-spines also -completed: the agent-first entry point (`AgentSession` multi-turn + session persistence + -export-to-workflow, Lane C / 1.m5, 1.V–1.AA) and multimodal I/O (media input + engine -plumbing + inline & async output generation + generative adapters, 1.m6, 1.AD–1.AH). See -[Phase 1 detail](phases/phase-1-engine-and-llm.md), the [decision index](../decisions/), -and the [reference specs](../reference/). - -> **Repository checkpoint (recorded 2026-07-26; publication status corrected 2026-10-13):** -> (1) The CI `ci` job became a required check under `main` branch protection, alongside a -> `v*` tag ruleset limiting tag creation to maintain/admin. The optional `TURBO_TOKEN`/ -> `TURBO_TEAM` secrets for cross-runner remote cache were recorded as unset at that checkpoint; -> this does not assert their current configuration. (2) The former v0.1.1 publish-pending note -> was stale: the [npm registry](https://registry.npmjs.org/relavium) records `relavium@0.1.1` -> published on 2026-06-28. This resolves the package-publication status, without asserting the -> current `NPM_TOKEN`, npm 2FA or GitHub release-workflow state. The release procedure remains -> [ADR-0051](../decisions/0051-cli-distribution-thin-bundle-private-engine.md) / -> [release-a-surface.md](../runbooks/release-a-surface.md). - -## What is active now - -### 2026-07-19 full-project code review — triaged, Phase 2.5.5 opened - -A multi-agent code review of the whole tree at HEAD `f88b0e8` ran while the toolchain was green -throughout (lint, typecheck, and all 226 test files passing) — every finding is a defect a green -build cannot catch. 377 findings went through two-lens adversarial verification (one agent trying -to refute each finding against the code, one auditing its materiality); 373 survived and were -triaged into 205 work items. The review's central conclusion: the gap to first-class is a gap in -**propagation**, not competence — the correct pattern almost always already exists in the repo and -fails to reach the next site (a redaction helper not called twenty lines away, terminal -sanitization built for chat and never carried to `relavium run`, `withBusyRetry` applied to every -writer but the three oldest, cancel-wins implemented in four node handlers and omitted in the -fifth). That makes the remediation cheap and low-risk: the reference implementation is already -in-tree. - -Disposition: 176 work items opened the new **[Phase 2.5.5 — Hardening and Remediation](phases/phase-2.5.5-hardening-and-remediation.md)** -(engine/LLM/persistence hardening, MCP/secrets, the CLI operator-safety net, TUI terminal safety, -docs accuracy, CI/test-infrastructure, and hygiene), running **concurrently with Phase 2.6, not -before it** (327 findings); 27 findings (16 work items) were folded directly into the charter of an -already-open 2.6 workstream; 16 findings (10 work items) were already tracked in -[deferred-tasks.md](deferred-tasks.md) and were re-confirmed rather than duplicated; 3 needed no -action per the finding's own argument. Severity of the 373 triaged findings: 3 critical, 48 high, 134 -medium, 155 low, 33 polish. - -The three critical findings, visible here so they aren't buried in the phase file: - -- **A tool-approval preview that is not secret-redacted** (`packages/core/src/tools/registry.ts`) - — the persisted, `--json`-visible `ToolActionPreview` echoes a command/path verbatim instead of - going through the same redaction its sibling input-sanitizer already applies. -- **A transient `history.db` write failure silently drops a chat turn, then crashes the session** - (`apps/cli/src/chat/persister.ts`) — an async write failure surfaces after the reply was already - shown to the user, as an unhandled rejection. -- **Missing terminal-output sanitization on the `relavium run` TUI** (`apps/cli/src/render/tui/RunApp.tsx`) - — `stripTerminalControls`/`sanitizeInline` is applied at nearly every other dynamic-text display - boundary but never reaches this one, leaving an ANSI/OSC/Trojan-Source injection hole. - -### Phase 2.6.F — platform floor + the full-screen TUI renderer (merged to `main`, PR #74, 2026-07-11) - -The first 2.6 workstream. Behind [ADR-0067](../decisions/0067-node-supported-floor-22-reaffirm-better-sqlite3.md) -(Node `>=22` published floor, `>=22.13.0` dev-install floor; `better-sqlite3` re-affirmed), -[ADR-0068](../decisions/0068-full-screen-tui-renderer-ink7-harness.md) (`ink` 7 + a hand-built alternate-screen -renderer + the `ink-testing-library` harness), and -[ADR-0069](../decisions/0069-string-width-for-the-cli-renderer.md) (`string-width` for display width — **Accepted** -on the PR #74 merge, the CLAUDE.md rule-2 approval it gated). - -Shipped: `ink` 6→7 and the component-render harness; the alt-screen lifecycle with restore nets on every -termination path; a hand-built transcript **viewport** (grapheme-aware wrapping, windowing, scroll + auto-follow, a -per-entry wrap cache) and the inter-session alt-buffer **hoist**; the **`/scrollback`**, **`/edit`** and **`/copy`** -copy-and-search hatches over a `suspendFullScreen` primitive; **in-app mouse text selection with copy-on-select over -OSC 52** on both surfaces, with edge auto-scroll, a frozen auto-follow and a transient **`✓ Copied` toast** so the -otherwise-silent copy is confirmed; `--no-mouse` / -`[preferences].{alt_screen,mouse,copy_on_select,show_banner}`; a branded Home banner; and DEC-2026 synchronized -output — which `ink` 7 turned out to already emit, so the step **pins** it rather than building the planned -`stdout` Proxy. - -The full-screen renderer is now the **default on a TTY**; `--no-alt-screen` and a machine / non-TTY / `--json` / CI -path stay on the byte-identical inline renderer. - -Two adversarial review rounds (one per step, one over the whole phase) drove the last third of the work. The -whole-phase round found that **the caps-lift ADR-0068 exists to deliver was never implemented** — a name collision -with the Step-4b-3 wrap cache — so a >4000-character answer was still clipped in the very viewport built to scroll -it; that `displayWidth` under-counted 8 539 code points against `ink`'s own width function; and that a keyboard -Ctrl-C during a Home hatch stranded mouse reporting on the user's shell. All folded, each with a break-verified -regression test. - -### Phase 2.6.C — reseat transcript-carry + the `/cost` per-model breakdown (merged to `main`, PR #75, 2026-07-13) - -Driven by two maintainer manual-test findings. **F1:** a mid-session `/models` reseat **blanked the alt-screen -viewport** — the reseat builds a fresh view store, and on the full-screen renderer (the TTY default since 2.6.F, with -no native scrollback behind it) that store *is* the scrollback, so the whole conversation vanished the moment a user -switched models. The durable transcript and the new model's context were never at risk; the loss was the view. The -reseat now carries the rendered conversation, gated to full-screen (a seeded **inline** store would re-print the -conversation over the copy ink's `` already put in the terminal's scrollback), and the switch notice lands -**beneath** it as an inline `⇄ model changed → ` marker, keeping ADR-0059's bound disclosure clause. - -**`/cost`** now shows where the money went, behind a new -[ADR-0070](../decisions/0070-durable-per-model-session-cost-attribution.md): a durable `session_costs` table with the -invariant **`SUM(rows) == agent_sessions.total_cost_microcents`**, true by construction. ADR-0059 had planned to -derive the breakdown from the per-message `model_id`, which **cannot work** — one row holds one model, but a turn -whose tool loop fails over bills two. Along the way it closed a **live bug**: `total_cost_microcents` was being SET -blindly by five writers, so two `chat-resume` processes could permanently corrupt a session's total. It now has a -single owner. `session_messages`' never-written token/cost columns were dropped rather than left as a second, wrong, -zero-valued source. - -Four adversarial review rounds ran over the work and each found something real — including that a tripwire I shipped -was **armed in production** and **swallowed by ink** (so it could never fail), that a perf harness measured a -**pointer assignment** while its ADR note claimed it verified an O(n) bound, and that the `/cost` DB read — the -step's central change — had **zero test coverage** at any layer. All folded, each with a break-verified regression. - -**Open obligations carried out of 2.6.C:** - -- **`chat-resume` opens on an empty viewport** — it has never repainted prior turns, in any mode or version (nothing - projects `session_messages` into rendered entries). Not a 2.6.F regression; tracked in - [deferred-tasks.md](deferred-tasks.md). -- **Reported spend is a systematic under-estimate of the provider's invoice** — an egress that streamed content but - ended without a usage chunk, and a mid-stream failure, are recorded as 0 on *both* sides of the invariant. That is - a usage-capture gap in the seam, filed against **2.6.Q** (ADR-0070 §3). -- **Cross-turn tool-call memory** — a "the reseat forgets tool calls" report turned out to be an *engine* property, - not a reseat one: tool pairs never cross **any** turn boundary (ADR-0062 §6). Deferred behind a default-off toggle, - but **three already-live bugs gate it** (no pre-egress context-window check; overflow is a fatal, non-retryable - `bad_request`; the budget gate prices only output tokens) — see [deferred-tasks.md](deferred-tasks.md). -- **The CLI e2e suite migrates the developer's real `~/.relavium/history.db`** — verified, and it is what let a - broken migration reach real data while staying invisible to CI. Tracked in [deferred-tasks.md](deferred-tasks.md). - -**Open obligations carried out of 2.6.F:** - -- **The Home landing can overflow a short terminal** — a populated "Attention required" section pushes the top off, - and the alt buffer has no scrollback to recover it. Predates 2.6.F (the strip is 2.5.B); **2.6.G**'s management - browsers replace the strip wholesale, which is the right place to fix it. -- **`relavium run`'s TUI stays inline** (a deliberate 2.6.F scope cut); its retained-scrollable-history is tracked. -- **Themes** (beyond colour / `NO_COLOR`) are deferred to **2.6.L**, per the maintainer's Step-5 decision. -- Real-TTY signal paths (double-Ctrl-C, `kill -TERM`/`-HUP`/`-QUIT`, a hatch under `tmux`) remain a **manual PR-time - check** — the unit tests pin the orchestration around injected seams, not the kernel. - -**Phase 2 — CLI (milestone M3) is feature-complete** (every in-phase workstream 2.A–2.S merged; published as -**v0.1.1**). The CLI is the first real -`@relavium/core` consumer and doubles as the engine's regression harness — validating the -engine API ergonomics before the desktop and VS Code surfaces. **Landed:** the CLI skeleton + -process contract (**2.A**) and the two-level config-resolution loader (**2.B**), both ✅ Done -(PR #40, 2026-06-22) behind [ADR-0047](../decisions/0047-cli-framework-commander-ink-clack.md) -(commander/ink/@clack) and [ADR-0048](../decisions/0048-toml-config-parser.md) (smol-toml); and -**2.D** (`relavium run` wired to `@relavium/core` — the M3 keystone and first real engine -consumer), ✅ Done (PR #41, 2026-06-22), which also adds the `defaultProviders()` seam registry; -and **2.F** (the `--json` CI machine-output contract — pure-NDJSON stdout, diagnostics → stderr), -✅ Done (PR #42, 2026-06-22) behind [ADR-0049](../decisions/0049-cli-machine-output-contract.md); -and **2.K** (the engine regression harness, now the engine's CI regression gate), ✅ Done -(PR #43, 2026-06-23) — **reaching global milestone M3**; and **2.H** (durable local run history via -`@relavium/db` — the `RunStore` writer + read API the gate-resume/list/logs/status surfaces consume), -✅ Done (PR #44, 2026-06-23) behind [ADR-0050](../decisions/0050-cli-history-db-at-rest-posture.md); -and **2.C** (the `relavium provider` commands — a provider registry + API keys in the OS keychain via -`@napi-rs/keyring`, resolved keychain → `RELAVIUM__API_KEY` env var → error), ✅ Done -(PR #45, 2026-06-23) behind [ADR-0019](../decisions/0019-cli-node-keychain-library.md) + -[ADR-0006](../decisions/0006-os-keychain-for-api-keys.md) (no new ADR — `secrets.enc` deferred past v1.0); -and **2.E** (the `ink` streaming TUI — the third `RunRenderer` over the one event bus: live per-node status -+ spinners, the active node's streaming tokens, a running cost footer, a persistent final summary; cooperative -Ctrl-C cancel), ✅ Done (PR #46, 2026-06-24) behind [ADR-0047](../decisions/0047-cli-framework-commander-ink-clack.md) -(ink + React 19, confined to `apps/cli`; no new ADR); -and **2.G** (the interactive human-gate prompt — a `@clack/prompts` card during `run` — plus the out-of-band -`relavium gate ` cross-process resume: reload snapshot → reconstruct checkpoint → `resumeFromCheckpoint`, -idempotent, secret-input fail-closed), ✅ Done (PR #47, 2026-06-24) behind -[ADR-0047](../decisions/0047-cli-framework-commander-ink-clack.md) (`@clack/prompts`; Node floor 20.11→20.12; -no new ADR) — **fully closing 2.K's deferred gate-resume half**; -and **2.I** (the read commands `list` / `logs` / `status` / `gate list` over durable history — go/no-go #2, the -read side; surfaces the pending `gateId`s the 2.G `gate` command points at), ✅ Done (PR #48, 2026-06-24) -(no new ADR — an additive workflow-agnostic `@relavium/db` read seam + a `@relavium/core` `parseAgent`); -and **2.L** (packaging, distribution & install verification — go/no-go #7, the last gate-closing spine PR: the -`tsup` engine-inlined ESM bundle, the bundle-closure drift guard, and the tag-triggered cross-OS install-smoke -`Release CLI` workflow), ✅ Done (PR #49, 2026-06-24) behind -[ADR-0051](../decisions/0051-cli-distribution-thin-bundle-private-engine.md) — **closing go/no-go #7, so the -Phase-2 spine is complete and all seven Phase-3 exit criteria now hold (Phase 3 may start)**. -**Also landed — the first additive lane:** **2.S** (media host-wiring — the surface half of the multimodal -sub-spine: the `model_catalog` reader → `resolveMediaSurface` routing + the D15 catalog load-check shared by -`run`/`gate`, the content-addressed `MediaStore` de-inline to a `media://` handle, the SSRF-validated -`EgressCapability.fetch` egress, the containment-checked `save_to` write port, durable fail-cost on the terminal -events, the produced-media render surface, and the best-effort run-end host media GC), ✅ Done (PR #52, 2026-06-25) -behind [ADR-0042](../decisions/0042-engine-media-storage-substrate-mediastore-deinline-retention.md)–[ADR-0046](../decisions/0046-inline-media-out-via-generate-streaming-triad-deferred.md) -(no new ADR). -**Also landed — the first user-facing `AgentSession` surface:** **2.M** (`relavium chat` — the agent-first -interactive REPL over `AgentSession`: streaming tokens, tool-call annotations, the FS-scope tier + `allowedCommands` -allowlist honored, `git_commit` denied; `/exit` / `/cancel` / an input-stream EOF / raw-mode Ctrl-C all end the -session with **exit code 4** — over ONE framework-free command core driving both an `ink` TTY app and a plain -non-TTY line loop; a built-in default agent over `[chat].default_model` for a zero-config first run; durable -per-turn persistence to the shared `history.db` that round-trips via `reconstructSessionState`; the ADR-0028 -cost cap wired; model output + pasted input sanitized of terminal control sequences at the display boundary), -✅ Done (PR #54, 2026-06-26) — **no new ADR** (covered by [ADR-0024](../decisions/0024-agent-first-entry-point-agentsession.md), -[ADR-0047](../decisions/0047-cli-framework-commander-ink-clack.md), [ADR-0028](../decisions/0028-workflow-resource-governance.md), -[ADR-0050](../decisions/0050-cli-history-db-at-rest-posture.md), [ADR-0029](../decisions/0029-tool-policy-hardening.md)). -`read_media` **input** access (D12) — which 2.S had pointed at 2.M — was **split into a dedicated, -security-reviewed follow-up** (maintainer-approved); the 2.M REPL shipped without it (tracked in -[deferred-tasks.md](deferred-tasks.md)). -**Also landed — the rest of the agent-first chat family:** **2.N** (`relavium chat-resume` — reload + continue a -persisted session over a shared REPL), **2.O** (`chat-list` — over a new additive `SessionStore.listSessions` -read seam), **2.P** (`chat-export` + the in-REPL `/export` — session → `.relavium.yaml` scaffold, [ADR-0026](../decisions/0026-session-export-to-workflow.md)), -and **2.Q** (`chat --json` — a headless `SessionEvent` NDJSON driver — + the one-shot `relavium agent run` with a -minimal in-house `--fixture` cassette for deterministic offline replay), all ✅ **Done (PR #55, 2026-06-26)** — -**no new ADR** — completing the agent-first CLI lane. (`agent run --input` is reserved/rejected until session -`{{ctx.*}}` prompt interpolation lands — a tracked engine follow-up in [deferred-tasks.md](deferred-tasks.md).) -**Also landed — 2.R (the inbound MCP client):** agents now consume external MCP servers' tools across `chat`, -`run`, and one-shot `agent run`. The **`@relavium/mcp`** foundation (the SDK-fenced package, the dependency-free -JSON-Schema→Zod compiler, the fail-loud connect-all manager, the `mcp_{server}_{tool}` namespacing) is ✅ **Done -(PR #56, 2026-06-26)**, and the **host wiring** (chat/run/agent-run), the **network transports** (`http`/`sse`/ -`websocket`) behind the SSRF pre-connect floor + the per-server `allow_local_endpoint` opt-in, **named secrets** -via the isolated `mcp-secret:*` keychain namespace, the by-name `ref` registration form, and the **real-spawn -e2e** are ✅ **Done (PR #57, 2026-06-27)** — behind [ADR-0034](../decisions/0034-mcp-client-sdk-dependency.md), -[ADR-0052](../decisions/0052-inbound-mcp-client-package-lifecycle-registration.md), and -[ADR-0053](../decisions/0053-mcp-network-transport-egress-security.md). It was off the M3 critical path and the -Phase-3 go/no-go (capability without gating). Residual MCP hardening — the connect-by-validated-IP dialer, -network header-auth, tool-list caching, mid-call abort propagation — is tracked in -[deferred-tasks.md](deferred-tasks.md). The **stdio spawn is gated** as of 2.6.5 `CR-16` -([ADR-0084](../decisions/0084-consent-before-a-local-mcp-spawn.md)): consent per machine and per resolved -declaration, on every path that can open an MCP-bearing artifact. The "import-trust" framing is retired — a -`git pull` changes a committed artifact with no import step — and only the `npx` version/integrity pinning half -remains deferred. -**Also landed — 2.J (the YAML-authoring lifecycle), the last in-phase lane:** `relavium create` (a -`@clack/prompts` wizard scaffolding an agent **or** a minimal single-agent workflow, validated against the -kind-appropriate `@relavium/shared` schema before write, dual-TTY-gated), `relavium import ` (schema- -validated copy-in with **project-global** id uniqueness), and `relavium export ` (a portable copy -**re-serialized from the validated AST** — canonical, comment-free, no provider key by construction), sharing one -`assertSlugAvailable` cross-catalog guard, ✅ **Done (PR #58, 2026-06-28)** — **no new ADR** (covered by -[ADR-0026](../decisions/0026-session-export-to-workflow.md)/[ADR-0047](../decisions/0047-cli-framework-commander-ink-clack.md)). - -**Phase 2 — CLI is feature-complete.** Every in-phase workstream (2.A–2.S) is merged and the published CLI is cut -as **v0.1.1**; M3 was reached at 2.K and the Phase-3 go/no-go held from 2.L. See the -[Phase 2 workstreams](phases/phase-2-cli.md) and the -[sequencing plan](phases/phase-2-cli.md#sequencing--parallelization). The full status-aware history is the -[Remaining build order](phases/phase-2-cli.md#remaining-build-order) section (its queue is now empty). - -**Phase 2.5 — CLI Consolidation & Conversational Home has started.** **2.5.A** (the spine's secure base — a -shared `assembleToolEnv({ profile, fsScopeTier, workspaceDir })` factory wired into **both** the chat and -workflow-run paths, the host-side `fs` (`realpath`+`commonpath` jail, read-only chat fail-close, single-fd -`O_NOFOLLOW`/`O_NONBLOCK` reads) and `process` arms, the advertise-filter, **EA1** `tool_unavailable`, and **EA2** -real failed-turn usage) is ✅ **Done (PR #60, 2026-06-28)**, behind -[ADR-0055](../decisions/0055-cli-host-capability-seam-tool-environment-factory.md) — **reaching milestone -M2.5-1 (secure base)**. The `egress`/`os` arms, the `project`-tier `extraRoots` allowlist, and a write-capable -chat are deferred to **2.5.E**/[ADR-0057](../decisions/0057-cli-chat-modes-and-per-tool-approval.md) (tracked in -[deferred-tasks.md](deferred-tasks.md)). **2.5.B** (the bare-invocation Home) is ✅ **Done -(PR #61, 2026-06-29)**, behind [ADR-0054](../decisions/0054-cli-bare-invocation-interactive-home.md) (Accepted): the -TTY-gated bare `relavium` opens a read-only management strip (recent sessions/runs/agents + an "Attention -required" section of pending human gates / failed runs) over a bounded, indexed `history.db` read seam, sitting -above a live prompt that graduates into an in-process chat; rendered as a single ink tree (one `useInput` owner) -with one SIGINT/SIGTERM lifecycle (clean Home exit `0`; an external signal → the conventional `128+signo`, 130/143; -the in-Home chat's exit-`4` consumed, never leaked) and bracketed paste (DECSET 2004), all while every -non-interactive path keeps the byte-for-byte help + exit-`0` meta-op ([ADR-0049](../decisions/0049-cli-machine-output-contract.md)). -Canonically homed in [home.md](../reference/cli/home.md). **2.5.C** (the in-app command system) is ✅ **Done -(PR #62, 2026-06-30)**, behind [ADR-0056](../decisions/0056-cli-in-app-slash-command-system-and-manifest.md) -(Accepted): a curated **two-registry** model (the shell `COMMAND_MANIFEST` driving `commander` + `--help --json` -+ the `executeCommand` dispatch, vs the in-REPL `REPL_COMMANDS` driving a filterable `/` palette + slash commands -in **both** the chat and the bare Home — no command in both); `/help`, the `notice` output channel, `/workflows`, -`/cost`, and `/doctor` (fast tier: keychain/config/wired-tools; `--deep`: a **redacted** provider-key probe + a -**read-only** MCP-status report — a security-review decision: it reports the live session's already-connected -servers, never a fresh connect/spawn); plus the `name + args` slash dispatch and a context-aware footer hint-bar -surfacing `/ for commands`. Canonically homed in [commands.md](../reference/cli/commands.md) + -[chat-session.md](../reference/cli/chat-session.md). **2.5.E** (chat modes + per-tool approval + mid-turn abort) -is ✅ **Done (PR #63, 2026-07-03)**, behind -[ADR-0057](../decisions/0057-cli-chat-modes-and-per-tool-approval.md) (**Accepted** after the mandatory -security review): the reseat-less mode system (ask / plan / accept-edits / auto on `Shift+Tab` + `/mode`), the -fail-closed per-tool `confirmAction` floor (`[y]/[a]/[n]` + a session once/always cache), the `Esc` mid-turn -abort (EA7), and the host arms closing the 2.5.A deferral — a write-capable `fs` tier + **protected paths** -(refused in every mode incl. `auto`), the SSRF-hardened `egress` arm (shared with media), and the `os` arm (now -a governed action class). Wired live into `relavium chat`, one-shot `agent run`, and the Home (each activates -the regime before its first turn). Every step passed the mandated opus + Sonnet 5 loop plus the dedicated -holistic security review (~50 findings fixed, 4 HIGH). A same-PR chat-UX follow-up also landed: a host tool -EXECUTION failure on the interactive surface (a file-not-found READ) is fed back to the model to recover -(`recoverToolFailures`, scoped to IDEMPOTENT tools via a stamped `ToolExecutionError.recoverable`; a governed / -side-effecting failure stays fail-fast) plus a static secret-free `tool_failed` hint. **With 2.5.E the CLI -Consolidation spine (2.5.A → C, E) is complete.** **2.5.D** (chat input ergonomics) is ✅ **Done (PR #64, merged -2026-07-03)** — the first experience-arm workstream: the pure-ergonomics half (`Ctrl+J` multiline, -`↑/↓` history + `Ctrl+R` reverse-search, readline motions, a shared cursor-bearing `EditorState`) plus the two -data-moving affordances behind [ADR-0061](../decisions/0061-cli-input-layer-file-injection-and-shell-escape.md) -(**Accepted** after a two-round maintainer security review): **`@`-mention** (dir-navigable file completion that -reads through the SAME `FsCapability` `read_file` uses — jail + the expanded sensitive-read confidentiality floor -+ listing-gate + binary/size guards — and injects as UNTRUSTED, nonce-fenced, byte+line-bounded context) and -**`!`-shell** (the additive `AgentSession.runUserCommand` — EA8 — routing `!command` through the one `run_command` -boundary: `enforcePolicy([chat].allowed_commands)` BEFORE the mode-aware `confirmAction` → `spawn`/`shell:false`; -**empty-default allowlist ⇒ `!` inert**, secure-by-default, with an actionable deny hint). Each of the 5 steps -passed the mandated opus + sonnet adversarial-review loop, and the ADR-0061 mandatory security pass ran inside the -step-4/5 loops (14 findings fixed across the two `@`-mention rounds; on the `!`-shell the opus + security pass -confirmed 0 defects after adversarial verification, and the sonnet second pass caught a HIGH — a `!`-command in -flight left no host-visible busy signal, so a message typed mid-command could crash the session — now fixed with a -`shellBusy` input gate, plus a LOW type-hygiene fix). A post-implementation comprehensive review then refined the -`@`/`!` presentation to a **pending-attachment (chip) model** (an inline `@path` marker + a read-only `!`-output -preview, expanded into the SAME UNTRUSTED nonce-fenced frame only at submit — byte-identical model context, a -clean prompt), and two follow-up review passes hardened the `[chat]` allowlist resolution (exact + glob arrays are -now a **coupled unit**) and fixed a Backspace regression (ink reports the Unix physical Backspace as `key.delete`); -all recorded in the ADR-0061 "Refined at implementation" append. **2.5.F** (the ADR-0062 context commands) is ✅ -**Done (PR #65, merged 2026-07-05)** behind [ADR-0062](../decisions/0062-context-compaction-and-cli-history-commands.md): -`/clear` (a host-level fresh-session swap across `chat`/`chat-resume`/Home, TTY-interactive only), the -`session:compacting` "Summarizing…" moment event (amending ADR-0036), and the footer context-fullness indicator — -completing compaction alongside the earlier `/compact` + `/trim` + automatic compaction. **2.5.G** (onboarding -wizard + Home `/models` + the live model catalog) is ✅ **Done (PR #66, 2026-07-07)** — its scope **expanded to -Option A**: a **live** model catalog (per-key provider discovery + a DB cache + refresh + a static/live merge) -and a complete model-pricing story (user-supplied pricing that governs cost), behind three ADRs -([ADR-0063](../decisions/0063-cli-config-write-contract.md) config-write · -[ADR-0064](../decisions/0064-live-model-catalog.md) live catalog · -[ADR-0065](../decisions/0065-provider-economics-and-extensibility.md) provider economics), across 12 reviewed -steps (all landed); with it **milestone M2.5-2** is reached. The additive lane **2.5.H** (reasoning render + live-turn feedback + an actionable error taxonomy) is ✅ -**Done (PR #67, 2026-07-07)** — behind **EA6** (a new dual-envelope `agent:reasoning` stream event that *amends* -[ADR-0036](../decisions/0036-run-loop-substrate-event-bus-and-execution-host.md); no new top-level ADR): a -host-emit of the reasoning the `@relavium/llm` seam already carries (ADR-0030), a collapsible "thinking" panel -(`/thinking` / `Ctrl+T`), the `Thinking…/Working… {elapsed}s · Esc to stop` timer, a visible `…` elision marker -(fixing the silent live-buffer head-drop), per-attempt `via {model}` failover attribution, and a static, -secret-free per-`ErrorCode` recovery hint (session-survives; a context-overflow message heuristic → `/compact`· -`/trim`). Four steps, each opus + Sonnet reviewed (3 HIGH fixed: a run-path silent-drop, a frozen Home timer, a -scrollback elision loss; + a one-shot `agent run` hint-leak). **With 2.5.E this reaches milestone M2.5-3.** The -consolidation lanes **2.5.I** (regression harness + DB concurrency hardening — `loadFull` read-txn snapshot, -`BEGIN IMMEDIATE` writes with a deterministic `SQLITE_BUSY` retry, the concurrent chat+run + cassette-chain + -perf-budget e2es, an advisory Windows CI lane) and **2.5.J** (docs-debt: the accurate unencrypted-history -posture per ADR-0050, and `NO_COLOR`/`FORCE_COLOR`/`--color` resolution) are ✅ **Done (2.5-close-out, PR #69, -2026-07-08)** — **reaching milestone M2.5-4, so Phase 2.5 is complete, merged to `main`** — landed alongside the doable-now -Batch A–E backlog (test-hardening; 2.5.H TUI polish; `AgentParseError` line/col; the ADR-0057 approval/security -batch — `[c]` reject-with-reason, non-TTY policy, SCOPE-denial recovery, Ctrl+T-in-approval, the Trojan-Source -bidi floor, behind an append-only ADR-0057 amendment; the profile-aware advertise-filter + the in-house -`.gitignore` matcher), each implement → Opus → Sonnet with a security-review pass on the approval batch. Two -`gate.ts`-resume items (`relavium budget resume` + secret re-provide) are deferred to a focused follow-up, and -the session `{{ctx.*}}` interpolation stays with the Proposed ADR-0060 (Phase-2.6); all tracked in -[deferred-tasks.md](deferred-tasks.md). See the -[Phase 2.5 workstreams](phases/phase-2.5-cli-consolidation.md). A **post-2.5.G model-UX follow-up** (from six -maintainer questions on model/provider/wizard behavior) then landed as a sequenced plan: `/models` key-awareness, -the onboarding-wizard live key-validation + retry UX, wizard-dynamic provider docs, **mid-session model switching** -(the `/models` reseat across `relavium chat` + the in-Home chat — [ADR-0059](../decisions/0059-cli-mid-session-model-reseat.md), -now Accepted), and **normalized reasoning-effort control** ([ADR-0066](../decisions/0066-normalized-reasoning-effort-control.md), -now Accepted, **merged 2026-07-07**): a provider-agnostic effort tier (`off`/`low`/`medium`/`high`/`max`) authored in -agent YAML or the `[chat].reasoning_effort` config default, each adapter mapping it to its provider's **native** tier — -**all four providers**, DeepSeek-v4's `thinking` param included (doc-verified against api-docs.deepseek.com) — gated -per-model by a host-injected capability resolver (plus a conservative id heuristic for a live-discovered model). It is -changed **interactively**: `/effort` opens a keyboard-owning tier-selector **overlay** in `relavium chat` + the in-Home -chat (§6), and the `/models` picker's **effort sub-step** either applies a **per-turn session override** in a live chat -(no reseat, §5) or — in the **bare Home** — **writes** the model + effort as the next session's config defaults (a new -global `[preferences].reasoning_effort` key), the active tier shown in the footer. Two multi-agent review passes -hardened it: the first caught the picker (wrongly) routing effort through a full model reseat — the P0 fix rebuilt it -as the ADR-mandated session-level setter; the second caught the bare-Home effort default not being re-read for a -same-process next chat — fixed to mirror the model's fresh per-chat re-read. - -Carry-over hardening is tracked in [deferred-tasks.md](deferred-tasks.md) — Phase 2 picks -items up as it first touches each file. Notable inheritances: 1.AH's host-wiring half -(distributed across Phases 2–6), the media-egress host-side SSRF mechanism, and the -keychain no-raw-key IPC test. - -## Not started yet - -The rest of **Phase 2.6 — Conversational Authoring and the First-Class CLI** -([phase-2.6-conversational-authoring.md](phases/phase-2.6-conversational-authoring.md), in progress — -**2.6.F is done**, -unblocked by the 2.5 close and **re-scoped 2026-07-08** from maintainer UX findings + a competitor -research pass + the deferred-tasks triage): a full-screen, Home-managed CLI (browsers for -workflows/runs/agents, provider + MCP + settings management, onboarding v2 with the Relavium-account -stub), competitor-breadth tools under the governance floor, settings/theming/`en`+`tr` localization, -the run-ops resume follow-up — plus the original spine: the shared `@relavium/authoring` package and -a chat that **authors** standards-valid `.relavium.yaml`. Then the surfaces and the cloud — -everything after the engine critical path: the desktop app (Phase 3) and the VS Code extension (Phase 4), then **Product Phase 2** — first **managed -inference** ([phase-5-managed-inference.md](phases/phase-5-managed-inference.md), the opt-in -`managed` gateway, engine still local), then the **cloud execution layer and web portal** -([phase-6-cloud-execution-portal.md](phases/phase-6-cloud-execution-portal.md)), the two -decoupled per Option B. See the [phase index](README.md#phase-index) and the -[milestone spine](README.md#global-milestone-spine) (M3 onward). - -**Systematic group 2a, round 3 — 2026-10-04.** [Two complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T15-44-35-w7-systematic-group-2a-round-3-review.md) -verify all four tool notifications, then reproduce an inherited raw-callback usage/hard-cap gap and -skipped realized recording. `e7973031` preserves original error identity while carrying canonical -accounting and starts mandatory recording after authoritative cost aggregation despite delivery failure. -Original baseline: 40 failing/13 passing; corrected root: 7,625 passing/12 skipped across 331 files, -23 forced tasks and six forced builds. Every sealed report/entry is verified before Original resumes. -**Fresh complete round 4 is required; group 2a, ownership, remaining groups, Steps 7–8/12, Sonar and PR remain open.** - -**Systematic group 2a, round 4 — 2026-10-04.** [Both fresh complete reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T16-34-57-w7-systematic-group-2a-round-4-review.md) -reproduce frozen classified errors defeating the carrier, proven pre-provider refusals consuming -slots without a governor, and classified success-flush errors losing usage or counting twice. -`ecf0b209` separates canonical accounting from throwable mutation/classification and records -actual chain-owned provider invocation; untyped chain diagnostics are fixed. Original baseline: -34 failing/243 passing; corrected focused run: 451 passing. Forced root checks pass all 23 tasks, -332 files, 7,665 tests/12 skips, six builds and test isolation. Both full reports and every one of -3,524 sealed entries/pins are independently verified before Original resumes. **Fresh complete -round 5 is required; group 2a, ownership, remaining groups, Steps 7–8/12, Sonar and PR remain open.** - -**Systematic group 2a, round 5 — 2026-10-04.** [Both fresh complete reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T20-43-04-w7-systematic-group-2a-round-5-review.md) -reproduce hostile error presentation replacing the original throwable/suppressing its terminal, -method lookup counted as invocation, and valid generated usage lost on pricing failure. -`902737eb` guards presentation, resolves receiver-bound methods before invocation and retains -safe-integer validated usage without inventing a cost. Original baseline: 19 failing/287 passing; -corrected focused run: 417 passing. Final forced root checks pass 23 tasks, 332 files, -7,694 tests/12 skips, six builds and test isolation. Both complete reports and every one of -3,589 sealed entries/pins are independently verified before Original resumes. **Fresh complete -round 6 is required; group 2a, ownership, remaining groups, Steps 7–8/12, Sonar and PR remain open.** - -**Systematic group 2a, round 6 — 2026-10-04.** [Both fresh complete reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T21-48-21-w7-systematic-group-2a-round-6-review.md) -verify existing protections and reproduce six overlapping price/quantity/invocation/diagnostic families. -`b4134e99` owns validated returned usage, marks unknown prices unpriced, guards nested/cause inspection, -keeps admission ownership through discharge and rechecks lookup cancellation. Exact-head baseline: -25 failing/19 passing; corrected focus: 461 passing. Final forced root passes 23 tasks, 333 files, -7,738 tests/12 skips, six builds, formatting and test isolation. Both complete reports and all -3,922 sealed entries/source/link/bootstrap pins are independently verified before Original resumes. -**Fresh complete round 7 is required; group 2a, ownership, remaining groups, Steps 7–8/12, Sonar and PR remain open.** - -**Systematic group 2a, round 7 — 2026-10-04.** [The complete contracts review and incomplete runtime observations](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T23-06-22-w7-systematic-group-2a-round-7-review.md) -reproduce generated observer errors causing paid retries/private public diagnostics and second tool-round -reflection replacing callback failures. `dd9d5a7d` retains exact attempt-observer provenance and guards -budget inspection. Original permanent baseline: 7 failing/24 passing; corrected focus: 200 passing. -Final forced root passes 23 tasks, 335 files, 7,769 tests/12 skips, six builds and test isolation. -Both entire reports and all 4,140 sealed entries/source/link/bootstrap pins are independently verified -before Original resumes. Runtime's platform-interrupted review is explicitly incomplete, with no -restored runtime certification. **Fresh complete round 8 is required; group 2a, ADR-0102 approval, -remaining groups, Steps 7–8/12, Sonar and PR remain open.** - -**Systematic group 2a, round 8 — 2026-10-05.** [Both complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-05T00-56-00-w7-systematic-group-2a-round-8-review.md) -verify held terminal mutation before confirming EOF, mutable attempt diagnostics and typed paid -observer failures granting unintended workflow retry/budget authority. Corrections own the held -terminal, freeze diagnostic copies and carry exact external observer origin separately from internal -admission faults. Session classified delivery and actual money failure ownership remain intact. -The 51 new terminal controls reproduce 31 failures before repair; final forced Original checks pass -23 tasks, 336 files, 7,831 tests / 12 existing skips, six builds and test isolation. Both complete -reports and all 4,175 physical entries/pins/links are verified before Original resumes, with prior -outer-launcher gaps qualified. **Fresh complete round 9 is required; group 2a, ADR-0102 approval, -remaining systematic groups, Steps 7–8/12, current Sonar and PR remain open.** - -**Systematic group 2a, round 9 — 2026-10-05.** [Both complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-05T01-57-06-w7-systematic-group-2a-round-9-review.md) -verify unsafe cause authority, generated result access after paid observation, compaction/later-tool -callback origin and false money-writer attribution. Verified corrections `1b23ff82` / `22d9df14` and -119 additional permanent controls pass forced full lint/typecheck/test (23 tasks, 339 files, -7,950 tests / 12 existing skips), forced build and isolation. Both complete reports and all 3,784 -physical entries/source/link/bootstrap pins verify before Original resumes; actual-price/unpriced -and early startup limits remain explicit. **Fresh complete round 10 is required; group 2a, -ADR-0102 approval, remaining groups, Steps 7–8/12, Sonar, W7 and PR #90 remain open.** - -**Systematic group 2a, round 10 — 2026-10-05.** [Both complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-05T02-56-00-w7-systematic-group-2a-round-10-review.md) -verify lifecycle setup/provenance and nested generated-content authority gaps. Corrections -`639d1ace` / `c6749597` add 94 controls and pass forced full lint/typecheck/test (23 tasks, -343 files, 8,044 tests / 12 existing skips), six builds and isolation. Both complete reports and -all 3,741 actual physical entries/source/link/bootstrap pins are audited; Contracts' three post-audit -cache files and one directory discrepancy explicitly qualify its seal claim. **Fresh complete -round 11 is required; group 2a, ADR-0102 approval, remaining groups, Steps 7–8/12, Sonar, W7 and -PR #90 remain open.** - -**Systematic group 2a, round 11 — 2026-10-05.** [Both complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-05T07-03-00-w7-systematic-group-2a-round-11-review.md) -verify terminal cancellation after successful durability, custom controller reentrancy and deadline -cleanup losing known price or acquiring control authority. Scoped repairs preserve canonical usage, -EA7 late abort and genuine money controls; 101 new core controls plus three shared disposal controls -pass. Forced Original checks pass 23 tasks, 351 files, 8,148 tests / 12 existing skips, build six and -isolation. Both exact final seals are independently audited; early evidence gaps remain qualified. -**Fresh complete round 12 is required; group 2a, ADR-0102 approval, remaining groups, Steps 7–8/12, -Sonar, W7 and draft PR #90 remain open. No additional paid generation is authorised.** - -**Systematic group 2a, round 12 — 2026-10-06.** [Qualified recovery and corrections](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T16-02-49-w7-systematic-group-2a-round-12-review.md) -record two interrupted reviews without full-scope acceptance. Cold reconstructed compaction preserves -terminal cancellation; generated typed output is owned before host pricing while known quantities -remain chargeable on projection failure. Thirty-one controls pass; forced Original lint/typecheck/test -passes 23 tasks, 355 files, 8,179 tests / 12 existing skips, six builds and isolation. Both complete -recovery reports and all 3,534 physical entries are audited. **Fresh complete round 13 is required; -group 2a, remaining groups, Steps 7–8/12, Sonar, W7 and draft PR #90 remain open.** - -**ADR-0102 maintainer clarifications — 2026-10-06.** [Two fresh qualified static reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T16-02-49-w7-adr-0102-maintainer-clarifications-review.md) -verify all six submitted clarifications at `1f7c353e`; no new material draft finding. SDK HTTP acceptance -is still required during implementation. **ADR-0102 stays Proposed, pending maintainer approval. -No dependent ownership implementation or additional paid generation is authorized.** - -**Systematic group 2a, round 13 — 2026-10-06.** [Both complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T16-53-54-w7-systematic-group-2a-round-13-review.md) -reproduce a cold provider resolver starting a real nested turn, after which compaction overwrites -its controller. `9f35173a` rechecks existing idle ownership before arming compaction; cold cancel, -idle abort and warm memoization remain intact. The Original regression fails before correction; -final forced lint/typecheck/test passes 23 tasks, 357 files, 8,195 tests / 12 existing skips, -six build tasks and isolation. Both reports, the complete Runtime static supplement and all -3,526 physical entries are audited, with early launch/read/preparation gaps retained. -**Fresh complete round 14 is required; group 2a, Proposed ADR-0102 approval/implementation, -remaining groups, live-fixture-gated Steps 7–8, final Step 12, Sonar, W7 and draft PR #90 remain open.** - -**Systematic group 2a, round 14 — 2026-10-06.** [Both complete independent qualified reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T17-49-28-w7-systematic-group-2a-round-14-review.md) -accept the 66-path scope at `207b2716`: cold operation ownership, generated projection/pricing -precedence and actual money durability joins retain their guarantees. Contracts passes 955 tests; -Runtime passes 934, with fresh builds and strict fixtures. Both reports, the stopped static supplement, -all 3,553 actual physical entries and 153 external artifact tuples are independently audited before -Original resumes; initial launch/read/preparation gaps stay qualified. **Group 2a is accepted within -this scope. Proposed ADR-0102 approval/implementation, remaining Groups 3–6, live-fixture-gated -Steps 7–8, final Step 12, current Sonar, whole W7 and draft PR #90 remain open.** - -**Systematic group 3, round 1 — 2026-10-06.** [Both complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T18-43-01-w7-systematic-group-3-round-1-review.md) -verify two High acknowledgement races: late timeout refusal overrides a claimed budget decision; -invalid inline approval closes native SQLite before the aggregate pause append is acknowledged. -`a78f859a` preserves the synchronous decision claim, and `7ceae889` drains the actual durable pause -or terminal before resource release. Nineteen independent regressions are promoted; the old-code -baseline has seven expected failures among 24 cases, and the correction passes all 24. Forced -Original checks pass 23 tasks, 362 files, 8,239 tests and 12 existing skips, plus six builds and -isolation. Both complete reports, all 3,802 actual entries and 362 external artifact tuples are -independently audited with timestamp, inherited-read and exercised-guard qualifications retained. -**Fresh complete round 2 is required; Group 3, Proposed ADR-0102, remaining Groups 4–6, Steps 7–8/12, -current Sonar, W7 and draft PR #90 remain open.** - -**Systematic group 3, round 2 — 2026-10-06.** [Both complete fresh reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T19-25-33-w7-systematic-group-3-round-2-review.md) -verify one High: after an invalid inline approval, SIGINT during a held native pause append loses -cancellation to the paused exit. `350f7152` retains cancellation through the acknowledged terminal; -eleven permanent native controls include held cancellation writes and actual authorization claims. -The independent 25-case baseline has three expected failures; its one-guard correction and all -prepared controls pass. Original checks pass 23 forced tasks, 364 files, 8,250 tests and 12 existing -skips, with six forced builds, isolation and format. Both complete reports, all 3,635 actual entries -and 862 external artifact tuples are audited with inherited-read, environment and exercised-guard -qualifications retained. **Fresh complete round 3 is required; Group 3, Proposed ADR-0102, -remaining Groups 4–6, Steps 7–8/12, current Sonar, W7 and draft PR #90 remain open.** - -**Systematic group 3, round 3 — 2026-10-06.** [Both complete fresh reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T20-13-20-w7-systematic-group-3-round-3-review.md) -verify two Highs: cancellation re-enters queued cards, and cancellation during paused renderer -finalization loses its subscription, native terminal write and truthful Ink summary. `94948cd4` -latches cancellation, suppresses later cards and retains the iterator through a summary settlement -barrier and fallback resource drain. Eleven promoted regressions have seven causal old-code -failures; the corrected independent combined 61-case focus passes. Original passes 23 forced -tasks, 366 files, 8,261 tests / 12 existing skips, six builds, isolation and format. All 3,797 -actual sealed entries and 3,349 external artifact tuples are independently audited with complete -reports, changed-case reads and preparation/guard/platform qualifications retained. **Fresh -complete round 4 is required; Group 3, Proposed ADR-0102, Groups 4–6, Steps 7–8/12, current Sonar, -W7 and draft PR #90 remain open.** - -**Systematic group 3, round 4 — 2026-10-06.** [Both complete fresh reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T20-57-47-w7-systematic-group-3-round-4-review.md) -verify two High UI-lifetime defects: genuine budget rejection opens a surplus queued card; -cancellation or acknowledged terminal during suspension or an open card retains stale input. -`9a62d665` follows emitted rejection/terminal authority, aborts scoped Clack input and preserves -the primary terminal/durability stream. A losing idempotent rejection leaves genuine later gates -usable; ordinary rejection still continues. Twenty-five permanent controls have 16 causal -old-code failures, then all pass; the corrected combined 135-case focus passes. Original passes -23 forced tasks, 369 files, 8,286 tests / 12 existing skips, six builds, isolation and format. -Both complete reports and all 3,745 actual sealed entries / 3,260 external regular tuples are -independently audited, with helper-guard, timer-seam, SDK/platform and preparation qualifications -retained. **Fresh complete round 5 is required; Group 3, Proposed ADR-0102, Groups 4–6, -Steps 7–8/12, current Sonar, W7 and draft PR #90 remain open.** - -**Systematic group 3, round 5 — 2026-10-06.** [Both complete fresh reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T21-43-24-w7-systematic-group-3-round-5-review.md) -retain one verified High: authored gate timeout/autoapproval during voluntary paused finalization -can misreport the terminal, and an unpublished node/terminal append can outlive SQLite closure. -Parent independently reproduces four failures among five real native controls; a stable pause -passes. ADR-0103 is being drafted/reviewed and no dependent lifecycle implementation is approved. -`fa884a92` separately corrects the CI completion-phase oracle and the safe status JSON contract; -26 lifecycle/Clack and 22 JSON/status cases pass in independent parent preparation. Original passes -23 forced tasks, 369 files, 8,292 tests / 12 existing skips, isolation, format and diff checks. -Both complete reports/child supplements and all 3,565 sealed entries / 3,077 regular tuples are -independently audited before Original resumes. **Group 3 remains changes requested. Fresh narrow -correction reviews and the High's approved implementation are required; ADR-0102, Groups 4–6, -Steps 7–8/12, current Sonar, W7 and draft PR #90 remain open.** Independent Groups 4–6 may advance -through their own checks and review while the new lifecycle decision is pending. - -### W7 group 3 round-5 correction accepted in narrow scope — 2026-10-06 - -Two fresh independent static/runtime rounds accept the nine-path test/documentation correction -through `8e241377`. Runtime focuses pass 98 and 96 cases, with fresh ordered builds, strict inclusion -and causal cleanup/JSON controls. Parent independently audits every recorded actual artifact field -and all 1,154 source/Original receipts before releasing both freezes. The [qualified review record](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T22-16-35-w7-systematic-group-3-round-5-correction-review.md) -retains all preparation/read/guard/platform bounds. Full Group 3 remains changes requested: the -voluntary paused-departure High is open, ADR-0103 remains an unapproved draft and ADR-0102 remains -Proposed. Independent Groups 4–6 may proceed under the existing plan; no W7/merge acceptance is granted. - -### W7 systematic group 4, round 1 — 2026-10-07 - -Both [complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-07T06-17-17-w7-systematic-group-4-round-1-review.md) -verify one High: acknowledged activation can delete isolated corrupt identity evidence, then reissue -an old effect-turn key. The correction preserves the canonical legacy floor inside the same deletion -transaction, or refuses deletion and allocation with evidence intact. Parent focus passes 585 tests; -removing only the preservation call fails seven controls, and exact restoration passes all 51 cases. -Original passes 23 forced tasks, 371 files, 8,316 tests / 12 existing skips and six forced builds. -Both reports and all 3,067 actual sealed entries / 2,704 regular artifacts are independently audited -before Original resumes, with preparation, register and native-helper qualifications retained. -**Group 4 remains changes requested; fresh complete round 2 is required. Group 3's High, -Proposed ADR-0102, unapproved ADR-0103, Groups 5–6, Steps 7–8/12, current CI/Sonar, W7 and draft -PR #90 remain open. No additional paid generation is authorised.** - -### W7 systematic group 4, round 2 — 2026-10-07 - -Both [complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-07T07-00-05-w7-systematic-group-4-round-2-review.md) verify a High: a known SQLite suspension column -with an unknown payload discriminator could be skipped, letting reconciliation invent an interruption -terminal. The correction checks raw type equality before tolerant parsing and refuses without leases, -terminal writes or changes to either run. Thirteen new native regressions pass; removing only the guard -fails eight required-refusal controls while five genuine/matching-unknown controls pass. Original passes -23 forced tasks, 373 files, 8,329 tests / 12 existing skips, six forced builds and isolation. Parent audits -all 3,082 actual sealed entries / 2,719 regular artifacts and actual native-child proof closure before -Original resumes, preserving read/preparation/checkJs limits. **Group 4 remains changes requested; -fresh complete round 3 is required. Group 3, Proposed ADR-0102, unapproved ADR-0103, Groups 5–6, -Steps 7–8/12, final CI/coverage/Sonar, W7 and draft PR #90 remain open.** - -### W7 systematic group 4, round 3 — 2026-10-07 - -Both [independent reports](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-07T07-43-01-w7-systematic-group-4-round-3-review.md) verify a High: discovery and the state reader could trust a -streaming SQL label before validating a genuine suspension payload. Actual native reconciliation then -invented an internal failure terminal. The correction validates every stored row through the existing -parser before excluding genuine streaming events from the returned fold; persisted-history validation -includes their parsing cost. The two permanent native files pass 81 cases; with existing run-history -tests, 164 pass. Separate causal removals fail 19 discovery and 11 state-read controls; exact restoration -passes. Original passes 23 forced tasks, 373 files, 8,397 tests / 12 existing skips, six forced builds and isolation. Parent audits all 3,126 sealed physical entries / 2,763 regular artifacts, -full actual self bindings, registers and child provenance before Original resumes. The runtime report -is interrupted and its wrong store/reader API attempt is not counted as state-refusal evidence; the -complete static report and independent Parent actual-reader regressions establish that surface. -**Group 4 remains changes requested; fresh complete round 4 is required. Group 3, Proposed ADR-0102, -unapproved ADR-0103, Groups 5–6, Steps 7–8/12, final CI/coverage/Sonar, W7 and draft PR #90 remain open.** - -### W7 systematic group 4, round 4 — 2026-10-07 - -[Both fresh reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-07T09-10-29-w7-systematic-group-4-round-4-review.md) cover all 33 corrected paths / 61 hunks and ten complete test files / 5,108 lines. -No new material implementation finding remains. Three literal newline escapes in the preceding -review prose are corrected without changing its historical result. Independent runtime passes -29 files / 753 tests, five fresh builds and strict fixture/setup/config inclusion. Discovery-only -and state-only removals fail 43 and 31 controls; exact restoration passes. Parent audits both -complete reports, all 3,145 physical entries / 2,781 regular files, source receipts, self bindings, -registers and eight actual native-child proof closures before Original resumes. Read ordering, -18 historical Own-evidence revisions, capture/format repairs and increased parsing cost remain -explicit qualifications. **Group 4 is accepted within scope. Group 3's High, Proposed ADR-0102, -unapproved ADR-0103, Groups 5–6, Steps 7–8/12, final CI/coverage/Sonar, W7 and draft PR #90 remain -open. No additional paid capture or ADR-dependent implementation is authorised.** - -### W7 systematic group 5, implementation — 2026-10-07 - -The [budget replay harness](../../tools/budget-replay-compat/README.md) now runs in the mandatory GitHub CI job. Its predecessor -portable bytes are shipped independently of live installed dependencies, and all 143 source files -are checked against the actual pinned Git baseline. Native SQLite remains the documented, -invocation-hashed ABI exception. Completed evidence is bounded to three invocations; active, -interrupted, foreign and redirected evidence is preserved. Seven provenance, 17 closure and eight -retention controls pass, with seven additional finalization cases in three result groups. The -isolation probe uses explicit file-marker readiness raced against child closure instead of elapsed-time assumptions. Turbo is pinned -exactly to the cooled 2.11.4 release, with its installed local schema and unchanged task definitions. - -Forced Original `pnpm run ci` passes: 23 lint/typecheck/test tasks, 373 test files, 8,397 passing -cases / 12 existing skips, six builds, formatting, migration sync, fences and all three offline -smokes. Initial tool-lint and formatting failures are retained; neither is presented as a passing -CI run. `typecheck:tools` covers its configured TypeScript programme, not every MJS helper. -**Group 5 is implemented and awaits fresh independent reviews; it is not accepted. Group 3's -High, Proposed ADR-0102, unapproved ADR-0103, Group 6, Steps 7–8/12, final coverage/Sonar, W7 -and draft PR #90 remain open. No additional paid capture or ADR-dependent implementation is -authorised.** - -### W7 Group 5 — review corrections implemented, fresh acceptance pending — 2026-10-08 - -The [first independent systematic Group 5 review](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T04-58-12-w7-systematic-group-5-round-1-review.md) corroborates -secondary-evidence failure masking, asynchronous closed-stderr errors, native failed-spawn -completion before close, a same-user stale retirement pathname, and two IPC wording mistakes. -Actual callers now preserve failure precedence, join real close and write fixed diagnostics -synchronously. Retirement claims and validates physical identity plus exact records, preserving -unexpected claims for inspection. Fourteen permanent caller controls and five causal removals -pass their intended positive/negative outcomes. Forced full CI passes 23 tasks, 373 files / -8,397 passing tests / 12 skips and six builds after an initial retained DB-timeout failure and a -clean isolated 522-test DB run. This is an implementation correction; fresh Group 5 round 2, -Group 6, ADR approvals, provider captures and final whole-wave acceptance remain open. - -### W7 systematic group 5, round 2 — 2026-10-08 - -[Two fresh complete reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T11-44-55-w7-systematic-group-5-round-2-review.md) and independent Parent audits accept all 22 Group 5 paths / 47 textual hunks and the immutable archive at `ab8c5138`. The actual harness closes all three workers and passes twelve predecessor refusals / two positive controls; fourteen caller controls, five grouped causal removals with exact restoration, 33 independent writer cases, six preparation cases and the real 45-second watchdog pass. Both full reports, physical inventories, source/Git pins and command/read/native evidence are verified before the Root freeze is released. Retirement compares directory identity and complete parsed records by semantic equality; it does not attest exact raw record bytes or record inodes. Application-isolation, transport/read-capture, historical release-date and initial CI-failure qualifications remain in the review record. **Group 5 is accepted within scope. Group 6, Group 3's High, Proposed ADR-0102, unapproved ADR-0103, provider captures, Steps 7–8/12, final CI/coverage/Sonar, W7 and draft PR #90 remain open.** - -### W7 Proposed ADR-0103 published for maintainer review — 2026-10-08 - -[ADR-0103](../decisions/0103-a-paused-run-hands-off-its-local-producers-before-its-host-closes.md) is published with the exact eighth reviewed candidate bytes, after [two complete static reviews and qualified Parent audits](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T11-44-55-w7-adr-0103-eighth-proposal-review.md). It proposes engine-owned safe paused-host departure, narrow late-receipt authority, truthful money/effect disposition and CLI exit 8. **It remains Proposed; no dependent implementation is authorised before maintainer approval. ADR-0102 also remains Proposed.** Independent Group 6 work continues; the lifecycle High, provider captures, Steps 7–8/12 and final W7/PR acceptance remain open. - -### W7 systematic group 6, narrow corrections implemented — 2026-10-08 - -The remaining review-document discrepancies are corrected: 51-item diagram, five recorded security sittings, MCP consent flags, expression anchor, disclosure-reader docblock and review EOF whitespace. Behaviour-preserving leaf changes make streaming membership a Set after full persisted-row validation, flatten the replay capture-cut selector, preserve exact generated loader bytes with raw literals, spell the bounded overflow close observation inside the loop, extract the budget detail label and retain primitive-only cold-request assertion formatting. The native signal fixture also receives its explicit string parameter annotation. - -Forced full Root `pnpm run ci` passes **373 files / 8,397 tests / 12 skips**, 23 initial tasks including build dependencies, six package build targets, seven build/format tasks and all offline smokes. Explicit strict/checkJs inclusion of the changed native fixture passes separately. Sixty-three synthetic loader cases generate identical bytes and parse; 32 event/state cases keep the exact capture cuts. **These are implementation checks, not independent Group 6 acceptance. Fresh independent rounds and individual adjudication of the current 180 Sonar labels remain required. ADR-0102/0103, provider captures, Steps 7–8/12 and final W7/PR acceptance remain open. No additional paid call or ADR-dependent implementation occurred.** - -### W7 systematic Group 6, round 1 — 2026-10-08 - -Two complete independent reviews of `c6c55b3b..bbe01091` verify one new MCP-consent documentation discrepancy and two pre-existing overflow-smoke error/close-join failures. After both full Parent physical and semantic audits, the narrow correction observes native errors immediately, joins close on every failure and limits the documented no-MCP guarantee to `budget abort`. Actual offline overflow smoke, tools lint and configured tools typecheck pass. Review/capture/guard qualifications and the historical 180/current 170 Sonar boundary remain explicit in the [round-1 record](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T13-20-40-w7-systematic-group-6-round-1-review.md). Fresh round 2 is required; Group 6, both Proposed decisions, live captures, Steps 7–8/12 and final W7 acceptance remain open. - -### W7 systematic Group 6, recovered round 2 — 2026-10-08 - -[Two fresh independent reviews and complete qualified Parent audits](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T14-50-57-w7-systematic-group-6-round-2-review.md) verify missing permanent native failure regressions and the unregistered `budget abort` spelling. The actual smoke now shares its private runner with five native failure controls; separately reverting error observation or final close joining fails the mandatory smoke, and exact restoration passes. The canonical command is corrected to `budget resume --abort`. Interrupted prior evidence is not counted as a completed round. Tools lint, configured typecheck and offline smoke pass; required CI at `7c29c462` passes separately. **Fresh round 3 is required. Group 6, Proposed ADR-0102/0103 approval and dependent implementation, Group 3's High, provider evidence, Steps 7–8/12, Sonar and final W7/PR acceptance remain open.** - -### W7 systematic Group 6, round 3 — 2026-10-08 - -[Two complete cumulative reviews and a fresh current Sonar adjudication](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T15-45-57-w7-systematic-group-6-round-3-review.md) cover all 17 paths / 27 hunks through `82acfcac` and all 175 current keys. Parent verifies all three complete reports, physical self bindings and source/native evidence before releasing the freeze. The actual mandatory smoke passes at head/restoration and under `CI=true`; separate error-observation and close-join removals fail its permanent controls. Generated loader/body bytes are unchanged. Root full CI passes 373 files / 8,398 tests / eleven skips; exact-head GitHub required CI and coverage pass separately. No new material implementation finding remains. **Group 6 is accepted within scope. Sonar remains failed pending three caller-scoped security dispositions; the complete keyed registry and proposed texts are linked from the review. Proposed ADR-0102/0103 approval and dependent implementation, Group 3's lifecycle High, Gemini/Anthropic missing provider evidence, Steps 7–8/12 and final W7/PR acceptance remain open. No additional paid call or ADR-dependent implementation occurred.** - -### W7 ADR-0102/0103 maintainer follow-up — 2026-10-08 - -[Round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T18-21-45-w7-adr-0102-0103-maintainer-round-1-review.md) checks the eight maintainer items and verifies a reachable Gemini SDK prototype-key merge conflict. The corrected Proposed ADR-0102 explicitly narrows non-cap data to refuse own `__proto__` keys; it preserves the cap exception and unchanged measured subset. Proposed ADR-0103 now specifies real primary publication/delivery observation, gap/abandonment refusal and the terminal-free local-stream qualification. [Two fresh complete round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T18-25-00-w7-adr-0102-0103-maintainer-round-2-review.md) find no further material decision-level issue. **Both ADRs remain Proposed; the additional compatibility narrowing and lifecycle/CLI decision require maintainer approval before dependent code.** No runtime acceptance or new provider result is claimed. The supplied Gemini 3.1 Flash Lite Free Tier quota is 250K input TPM, below its 1,048,576-token window; the capture access/authorization gate remains open. Ownership/lifecycle implementation Highs, Steps 7–8/12, Sonar and W7/PR acceptance remain open. - - -### W7 latest decisions approved and completion sequence — 2026-10-08 - -The maintainer approves the latest [ADR-0102](../decisions/0102-a-measured-request-owns-its-inert-data-through-egress.md) -and [ADR-0103](../decisions/0103-a-paused-run-hands-off-its-local-producers-before-its-host-closes.md), -including their 2026-10-08 clarifications. Both are Accepted. Dependent implementation is -authorised on `development`; earlier approval restrictions remain historical records. - -Complete the remaining work in these reviewable increments: - -1. Own request data and candidate cap projections at invocation; prove actual SDK generate/stream - parity and typed refusals, then reuse the measured construction through core/media handoffs. -2. Implement engine departure, primary delivery observation, structured receipt authority and - retained exact-fence writer lifetimes; preserve bounded terminal publication. -3. Derive parked-node deadlines and integrate acknowledged CLI input/host departure, final - receipt disposition, exit 8 and the canonical diagnostic contract. -4. Obtain the two missing genuine provider records, complete Step 7 classification and Step 8 - compaction/recovery, and close Step 12 through whole-wave acceptance and document updates. - -Each increment receives an implementation commit, fresh independent review, confirmed fixes -and another fresh review before acceptance. These are implementation subdivisions of the -approved work, not replacement W7 step numbers. The maintainer authorises paid Gemini access -and necessary new provider work with a 2 USD test-call limit. Billing setup requiring a separate -legal agreement or a larger minimum prepayment remains an explicit account-setup action; no -new provider request has been made under this authorisation yet. Implementation, provider -evidence, final CI/coverage/Sonar and all six remaining W7 register items remain open until -their actual acceptance evidence lands. - - -### W7 authorised provider evidence and internal ownership foundation — 2026-10-08 - -The maintainer completed Gemini billing; the selected Gemini 3.5 Flash-Lite model has paid -Tier 1 access and a 4M input-TPM quota. The existing OS-keychain account was used through stdin, -without exposing its key. One authorised Gemini request returned the genuine HTTP 400 -`INVALID_ARGUMENT` input-overflow body; its exact v1 artifact is preserved in the -[fixture index](../../packages/llm/src/conformance/fixtures/overflow/README.md). **Four of five genuine records now exist. Only Anthropic's native -`model_context_window_exceeded` response is missing.** The earlier approval sequence's -no-new-call/two-missing statements describe its publication checkpoint, not the present state. - -One free Anthropic token-count request calibrated 199,762 input tokens. One accepted Haiku 4.5 -probe ended with `end_turn`, so it is retained privately and does not establish native overflow. -Its reported 199,762 input / 1,505 output tokens imply 0.207287 USD at the published rates; this -is usage-priced evidence, not an invoice. The Gemini overflow was an unsuccessful 400 request; -no successful token usage or invoice was returned. The bounded next Anthropic probe, available -only through explicit capture purpose selection, remains subject to the maintainer's 2 USD -limit and independent capture-tool review. No automated paid retry is authorised or introduced. - -[Fresh ownership foundation round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T19-29-00-w7-inert-graph-round-2-review.md) -accepts only the internal inert-graph helper after descriptor/native-brand fixes and separate -mutable-alias controls. Full request/cap associations, actual SDK handoffs and core measured-round -reuse are not implemented or accepted by that record. ADR-0102/0103 implementation, the remaining -provider record, Steps 7–8/12, final CI/coverage/Sonar and all six W7 register items remain open. +7. **The MVP reorder (2026-11-04)** puts Gate 0, MVP-1…MVP-10 and v0.2.1 ahead of the Wave 3 remainder and + Waves 4a–7, moves v0.2.0 out of Wave 4b (M2.5.5-4's acceptance never contained it) and pulls the items listed + under [Pending after the MVP](#pending-after-the-mvp-not-declined) forward. The waves' internal order is + unchanged. + +## Standing rules + +> **Work on `development` itself** ([AGENTS.md](../../AGENTS.md)); create another branch only when the +> maintainer asks, and then cut it from `origin/development`, never from a local `main`. A long-lived +> `development` merged into `main` by merge commit leaves a stale local `main` behind after every PR, silently +> and without conflict. Run `git fetch` first. + +- The gate is `pnpm run ci` plus `pnpm coverage:enforced`; `development` is pushed after each accepted increment + and its push CI is observed. +- `main` carries branch protection (PR required; `lint · typecheck · test`, the coverage floor and, since + 2026-11-04, the Windows job required; no approval count, since GitHub refuses a self-approval — see + [code-review.md](../standards/code-review.md); force-push and deletion blocked) and a `v*` tag ruleset + limited to the maintain/admin roles; `release.yml` asserts its ref is reachable from `main` and publishes + from the `release` deployment environment. + +## History + +Finished work is recorded in its register; this page no longer restates it. + +- **Phase 2.6.5 (W0–W8)** — the [phase register](phases/phase-2.6.5-core-reliability-remediation.md), including + the W7 and W8 closing registers and merge checkpoints and the W4 start-gate ruling. +- **Phase 2.5.5** — the [phase register](phases/phase-2.5.5-hardening-and-remediation.md): the 2026-07-19 review + triage under Positioning and the traceability appendix, Waves 0–1 and the PR #81 closing list in the + [moved execution record](phases/phase-2.5.5-hardening-and-remediation.md#execution-record-moved-from-currentmd--2026-11-04), + the Wave 2 and Wave 3 Steps 1–8 merge checkpoints, and every Wave 3 increment's checkpoint and acceptance. +- **Phase 2.6** — the 2.6.F and 2.6.C implementation records and open obligations, in their + [workstream sections](phases/phase-2.6-conversational-authoring.md). +- **Phases 1, 2 and 2.5** — [phase 1](phases/phase-1-engine-and-llm.md), [phase 2](phases/phase-2-cli.md) and + [phase 2.5](phases/phase-2.5-cli-consolidation.md). +- **Repository checkpoint (recorded 2026-07-26; publication status corrected 2026-10-13):** the + [npm registry](https://registry.npmjs.org/relavium) records `relavium@0.1.1` published on 2026-06-28. The release + procedure is [ADR-0051](../decisions/0051-cli-distribution-thin-bundle-private-engine.md) / + [release-a-surface.md](../runbooks/release-a-surface.md). diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 436ff9b0..e74c90cc 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-18 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are accepted on `development`, with PR #95 under review. Existing approved deferrals remain. +> Last updated: 2026-11-04 — the CLI MVP comes first (maintainer, 2026-10-26; decisions D58–D72 ruled 2026-11-04): Wave 3 is paused after Step 9b.2 and its Steps 9c–19 are pending, not declined; the order is in [current.md](current.md#next-work--the-cli-mvp). Wave 2 is merged in PR #94 and Wave 3 Steps 1–8 in PR #95; PR95, 9a.1, 9a.2a, 9a.2b, NI (its Windows clause closed on 2026-10-26), 9b.1 and 9b.2 are accepted on `development` with their recorded findings below. The independent PR #95 review's findings keep the maintainer's 2026-10-19 owner rulings, and the masker's edge cases the 2026-10-21 bar and the 2026-10-22 rulings. Existing approved deferrals remain; entries the MVP pulls forward say so. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project @@ -368,6 +368,745 @@ Severity is the review's verified rating. Check an item off in the PR that resol there is concrete surface demand or telemetry showing operators need an earlier signal. *(1.AC; ADR-0028; config-spec.md; workflow-yaml-spec.md)* +## PR #95 independent review — unactioned findings (2026-10-18) + +An independent systematic review of [PR #95](https://github.com/HodeTech/Relavium/pull/95) at `19c99968` +reported no blocker or high finding. Its medium and three of its four low findings were corrected in `a3e164b5` +([disposition table](phases/phase-2.5.5-hardening-and-remediation.md#independent-review-of-the-published-head-and-its-corrections)); +what follows was **not** actioned in that pass. **Owner rulings, 2026-10-19:** the maintainer approved an owner for +every open item ([rulings](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19)). +Each `- [ ]` below is now **Scheduled** to a Wave 3 increment (the PR95 increment runs before Step 9a) and stays unchecked until +that increment lands, except LOW-1 and LOW-2, which are approved deferrals with their recorded triggers. Items +marked *(reviewer)* are quoted from that review and were not re-verified by the author; items marked *(author)* were +measured or counted here. + +- **DONE 2026-10-19 — SonarCloud's quality gate was red on `main` after every merge.** *(author)* The `main` + analysis failed `new_security_rating` (C, required A) against the `previous_version` baseline (2026-06-04) because nine + vulnerabilities were open there: seven `githubactions:S6505` ("omitting `--ignore-scripts`") on the + `pnpm install --frozen-lockfile` lines in `.github/workflows/ci.yml` (120, 432, 463, 487), `release.yml` (94) and + `models-catalog.yml` (51, 95); one `jssecurity:S8707` at `tools/budget-replay-compat/lifecycle-smoke.mjs:47`; one + `javascript:S4036` at `tools/cli-package/aggregate.mjs:58`. The gate had failed after the PR #90, #92, #94 and #95 + merges while the PR-level analyses passed, because SonarCloud statuses are per branch and the existing dispositions lived + on PR branches and on other `main` records. SonarCloud is not a required check. At the maintainer's direction the + dispositions were applied on `main` through the SonarCloud API after re-verifying each rationale at `6d2a7372`: the seven + S6505 and the S8707 as false positives (same reasons as their `main` siblings; the install-script posture comment at the top of + `ci.yml` and the Sonar backlog section below), the S4036 accepted as on the PR #95 branch. SonarCloud then reported zero + open vulnerabilities and an OK gate for `main`. No rule, threshold or gate was changed; the per-issue record is local + and Git-ignored. *(was medium · process)* +- **DONE 2026-10-22 — Wave 3 PR95 increment, closed under the maintainer's round cap. An `api-key` header label erases the `Bearer`/`Basic` label that follows it, so the token leaks.** *Low (security).* *(author)* `scrubSecrets('api-key:Bearer abcdefghijkl123456')` returns `api-key:[REDACTED] abcdefghijkl123456` — the `api-key` rule consumes the word `Bearer` as the header's value and the Bearer rule then has no label to anchor on. The pre-PR matcher returned `api-key:Bearer [REDACTED]`. Surfaced by the post-merge security sitting, reproduced independently; it comes from the PR's rule order, not from `a3e164b5`, and is **not fixed**. A plain `Authorization: Bearer …` is unaffected. Fix idea: let the header value skip a following `Bearer`/`Basic` label, or run those rules first. *(low · packages/llm/src/scrub-secrets.ts)* +- **DONE 2026-10-22 — Wave 3 PR95 increment, closed under the maintainer's round cap. URL userinfo containing a raw `?` or `#` before the first `:` (or with no `:`) is not masked.** *Ruled 2026-10-19:* mask unless the text before the `?`/`#` is host-shaped (a dotted hostname, `localhost` or a bracketed IP, with an optional `:port`) **and** an `=` assignment lies between the `?`/`#` and the authority `@`; a plain host-shaped exemption would still leave `https://my.user:1234?TOKEN@h/` open. The same predicate applies to core's twin in `packages/core/src/tools/bounding.ts` (its colonless short-handle rule and `]` delimiter stay, recorded as differences). *Low (security).* *(author, from two independent differential runs)* `https://us?er:TOKEN@h/`, `http://#TOKEN@h` and `https://my.user:1234?TOKEN@h/` pass through unchanged; the pre-PR matcher masked all of them. It follows from the deliberate host-query policy in `maskUrlUserinfo` (a raw `?`/`#` may begin public query text, e.g. `https://host?contact=a@b.c`) and the colon-first cases are pinned by tests; it is identical at `19c99968` and is not introduced by the follow-up. **A maintainer call** whether a username shaped that way is worth the false positives. Fix idea: exempt only when the text before the `?`/`#` is non-empty and host-shaped (contains `.`, a `:port`, `localhost` or a bracketed IPv6), otherwise mask. *(low · packages/llm/src/scrub-secrets.ts)* +- **DONE 2026-10-22 — Wave 3 PR95 increment, closed under the maintainer's round cap. Credentials in URLs of other schemes are outside the finite rule.** *Ruled 2026-10-19:* extend the finite list with postgres, postgresql, mysql, mariadb, redis, rediss, mongodb, mongodb+srv, amqp, amqps, ssh, sftp, ldap, ldaps, smtp and smtps, masking only colon-bearing userinfo for the new schemes; malformed `https:/u:p@h`, a raw `/` inside a password and invisible characters inside the scheme stay documented limits. *Low, informational.* `postgres://`, `redis://`, `mongodb(+srv)://`, `ssh://` userinfo, malformed `https:/u:p@h`, a raw `/` inside a password, and invisible characters inside the scheme are not masked, identically before and after the follow-up. The file states its shapes are finite; a database connection error is the realistic carrier, so the scheme list may deserve a decision. *(info · packages/llm/src/scrub-secrets.ts)* +- **DONE 2026-10-22 — Wave 3 PR95 increment, closed under the maintainer's round cap. The URL rule's split-credential scan costs 2.1–2.5× on letter-heavy input, and nothing times that loop.** *Low / nit.* With `NO_BOUNDARY` the scanner tests every ASCII letter start for the URL rule; at 8 MiB the worst case stays under about 1.1 s and linear (1–16 MiB scaling checked). A prefilter on `h`/`w`/`f` would recover most of it. The existing 8 MiB timing case contains no `@`, so it returns before the loop; a guard would use a leading `a\nhttps://u@h ` plus 8 MiB of letters, but see the load-sensitive timing item below before adding another wall-clock assertion. SonarCloud was not run on the never-matching `NO_BOUNDARY = /(?!)/`; check its next analysis. *(low · packages/llm/src/scrub-secrets.ts)* +- **DONE 2026-10-22 — Wave 3 PR95 increment, closed under the maintainer's round cap. Two latent invariants in `hasSplitCredential` are unchecked.** *Nit.* A boundary-bearing rule with neither `\b` nor a `joinedPattern` makes `scrubSecrets` throw a `SyntaxError` on LF/TAB input no earlier rule has redacted (the suite fails at once, so it is caught, but it would break `makeLlmError`'s no-throw claim); and `wordBlocksStart` tests `boundary.test('a')` as a proxy for "rejects every ASCII word character", true for all current boundaries. An assertion at module load would pin both. *(nit · packages/llm/src/scrub-secrets.ts)* +- **DONE 2026-10-22 — Wave 3 PR95 increment, closed under the maintainer's round cap. The userinfo control's joined-view half is not separately witnessed.** *Low (coverage).* *(author, from an independent run)* Mutating only `joinedPattern` of the URL rule leaves 111/111 tests green. The whitespace-stripped residual probe independently catches the class the existing cases use, so it may be largely an equivalent mutant; an added case that did not discriminate it was reverted rather than kept. No causal control pins the glued-scheme behaviour either — only ordinary tests do. *(low · tools/secrets-controls/mutations.mjs)* +- [ ] **Approved deferral (maintainer, 2026-10-19). `LOW-2` (the 1 s default `waitFor` on a real child) was examined and deliberately not changed.** *Low.* The only wait that observes a real child booting is `holdNativeToolProcess`'s marker wait, whose sole caller sits inside `drive-home.test.ts` waits (lines 633 and 637) that use the 1 s default and enclose the boot, so a longer inner bound cannot help; a retune must raise those outer waits together. No failure was reproduced (0 in dozens of loaded runs). `testing.md` also states that no readiness or case deadline is extended to pass the workspace gate. **Trigger: a recorded boot-time failure.** *(low · apps/cli/src/commands/native-forwarding.test-support.ts, apps/cli/src/home/drive-home.test.ts)* +- [ ] **Scheduled → Wave 3 17b. Join-table test nits.** *Nit.* The `process-close.test.ts` gate margin is nominally 100 ms but `waitFor`'s 50 ms poll makes it 100–150 ms, so a publication more than about 150 ms late is invisible; the three rejection cases assert the message but not the `ProcessCapabilityError` class; the `mcp-orphan.e2e.test.ts` comment's statement that its cases have only ever run on POSIX is historical and unverifiable; the abort case's `await spawned.promise` is unbounded (a spawn that never happens fails only as the 5 s case timeout), its closed-child `waitFor` carries no tag, and the 100/300/3,000 ms margins are bare literals. *(nit · apps/cli/src/engine/tool-host/process-close.test.ts, apps/cli/src/harness/mcp-orphan.e2e.test.ts)* +- [ ] **Approved deferral (maintainer, 2026-10-19). Native-lifetime watchdogs are fixed 5 s bounds inside 10 s cases (`LOW-1`).** *Low.* The watchdogs in + `apps/cli/src/process/native-process-owner.test.ts`, `engine/tool-host/os-native-lifetime.test.ts`, + `render/editor-native-lifetime.test.ts`, `render/fixtures/native-editor.mjs` and + `tools/cli-package/smoke/native-process.test.mjs` assert that a slow-but-correct close did not trip them, so a + loaded runner produces red without a defect. *(author)* The constants are **coupled** — the Home editor case + has a 5 s fixture watchdog, a 3 s ready wait, a 6 s release wait and a 10 s case bound — so raising one alone + does nothing, and three runs of the three vitest files (52 tests) passed at a load average of 80–96 on macOS. + The reviewer could not observe the Windows root cause either. **Trigger: a Windows trace that names the bound + that fired**; retune the coupled set together then. *(low · the files above)* +- **DONE 2026-10-22 — Wave 3 PR95 increment, closed under the maintainer's round cap. `bounds admitted-size failed header and URL preparation` is load-sensitive.** *Low.* *(author)* It allows + 5 s for two 8 MiB scrubs that take about 0.75 s in total, and it failed twice with the load average near 65 (6.4 s + in the run that logged the duration). Measured alternately, the pre-change code and the corrected code take the same + time, so the failures were load, not the correction. Two independent reviewers saw it too (5,590 ms and 3,453 ms at load averages near 88–97) and recorded that `tools/secrets-controls/README.md` says the fixture runs without a machine-speed threshold, which this assertion contradicts; six controls run this file, so a loaded runner can turn a fixed phase red. The test + guards against a quadratic regression, so any loosening must stay far below minutes. **Trigger: the next CI + failure of this case.** *(low · packages/llm/src/scrub-secrets.test.ts, `SECRETS_B2_HEADER_PREPARATION`)* +- [ ] **Scheduled → Wave 3 17a (all four sites together). Raw exit-code literals where `EXIT_CODES` names exist.** *Nit.* *(reviewer)* reported + `result === 5 || result === 7 || result === 8` in `apps/cli/src/chat/session-surface-lifetime.ts:54`; *(author)* + the same literals occur in four places: that site, `commands/workflow-lifetime.ts:178`, + `process/command-lifetime.ts:110` and `commands/drive.ts:225`. Behaviour-neutral, but it sits on the exit-code + precedence that the review's red-team matrix held, so rerun the precedence tests with the change. + *(nit · the four files above)* +- [ ] **Scheduled → Wave 3: `session-ink-input.ts` → 15c, `HomeDeps.exit` → 14a, the `lazy-dispatch.ts` cause-wrap → 10a. CLI lifetime and API-drift nits.** *Nit, (reviewer).* `chat/session-ink-input.ts:40-42,124-131` guards only + the `readable` listener (Ink's kitty `data` path is latent because `kittyKeyboard` is unused); + `home/drive-home.tsx:165-166,1067,1115-1116` keeps `HomeDeps.exit` although the owned path never calls it; + `commands/lazy-dispatch.ts:49` imports the media route without the cause-wrap the sibling routes carry. + *(nit · the files above)* +- [ ] **Scheduled → Wave 3: the purity-guard root set → 10a; the remaining test-evidence nits → 17b. Test-evidence nits.** *Nit, (reviewer).* `process/pure-substrate-graph.test.ts:117-133` — the purity-guard + root set omits the ADR-0111 eager-boot files (the reviewer verified that opaque marking fixes it); three + no-argument `toHaveBeenCalled()` positives outside `native-process-owner.test.ts`; + `packages/llm/src/adapters/list-models-lifetime.test.ts:122` labels a fake-`fetch` case "real installed SDK"; + `apps/cli/src/harness/concurrency.e2e.test.ts:434-441` uses a fixed 100 ms contention margin; + `tools/cli-package/consumer-lifetime.test.mjs:54-59` pre-marks every synthetic case `passed: true` (by design; + real detection is the packed-consumer CI). *(nit · the files above)* +- [ ] **Scheduled → Wave 3: the two tooling nits → 9a.1; the duplicate headings → 19b. Tooling and documentation nits.** *The tooling half is accepted with Wave 3 Step 9a.1 (implemented 2026-10-22, accepted at `cf97b7f1` on 2026-10-24):* the unreachable `TAR_FORMAT` branches are removed, and `installedModulesOwner` (`tools/cli-package/smoke/installation.mjs`) replaces the ancestor loop; the duplicate headings stay with 19b. *Nit, (reviewer).* *Premise corrected 2026-10-19:* the repeated headings are the 2.5.5.A–I sub-stream headings and the Wave 2/Wave 3 "Decisions presented together before implementation" pair, not Wave 2 checkpoint sub-headings. Unreachable `TAR_FORMAT` branches at + `tools/cli-package/archive.mjs:126,135`; no `break` in the ancestor loop at `tools/cli-package/smoke/worker.mjs:64-65` + (the outermost `node_modules` wins); and repeated heading texts in the phase register, whose later occurrences + take a `-1` slug suffix. *(author)* A scan finds repeated H2–H4 headings (the 2.5.5.B–I sub-stream headings and the + Wave 2 checkpoint's sub-headings among them); no link is broken today. *(nit · the files above)* +- **No action, recorded 2026-10-19 — informational items kept in view.** No change is proposed for any of them; the `git_commit` approval wording is already covered by Wave 3 Step 5's #308 disposition. *(reviewer)* `packages/mcp/src/manager.ts:228-236` — a comment omits + the bare `Error` in the throwing-observer branch (behaviour is canonical); `packages/core/src/tools/bounding.ts` — + the removed 4k pre-clip in `makeSummary` is required for correctness (8 MiB takes about 49 ms, about 200 ms with + line feeds) and the alias DAG is `2^d` (depth 18 is 786k nodes in 404 ms) for host/embedder graphs only, never model + JSON; `packages/core/src/tools/builtins.ts:451-452` — the `git_commit` description promises human-gate approval + while no producer sets `gateApproved=true` (not verified by the reviewer); `commands/workflow-lifetime.ts:204` + can return a raw signal code above 9 that the physical exit never uses, a note for future embedders. No action + is proposed for any of these. + +## PR #95 masker edge cases — approved deferrals (2026-10-21 and 2026-10-22) + +Under the [masker acceptance bar](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20) +that the maintainer set on 2026-10-21, a low finding in the open-ended masker work +(`packages/llm/src/scrub-secrets.ts` and core's twin projection in `packages/core/src/tools/bounding.ts`) is +recorded here instead of being corrected in the review loop, and under the round cap of the same day so is +every medium and low finding of round 10, the last round, after which only high findings are corrected. The +entries up to round 9 are not regressions against base `9902e0a0`; the round-10 entries say when they are, in the +shapes they name. The maintainer counted round 10's GEN-R10-1 (a `.env` password in a structured result's +summary) as high on 2026-10-22, so its summary half is corrected and only its journal half is recorded here, and +approved the two private-key block gaps the author found after round 9. The shapes that neither masker reads by design stay described in +their canonical homes, [built-in-tools.md](../reference/shared-core/built-in-tools.md) for core and +[error-handling.md](../standards/error-handling.md) for the scrubber; this list holds the review findings and the +re-checks they need. **Trigger for every entry:** a real tool result, journal entry or diagnostic that carries the +shape, or the next change to the rule the entry names. The generative harnesses that found them are not part of +the repository, so each entry names the input and the property to re-check by hand or in a test. + +- [ ] **Approved deferral (maintainer, 2026-10-21). A header label split by a JSON escape is not joined in a + summary.** *Low, unchanged from base.* *(author)* `boundForModel({ stdout: 'api-k\tey: ' })` shows the + value in its summary, while the same text as a string result is `[redacted]`. The summary's second reading + (`visitEscapedSeparators`) runs the standalone, scheme-label and split-credential rules, not the split + header-label join, which rewrites the text instead of reporting ranges. **Re-check:** whether a real tool output + splits a header label across a line break; a fix reports the joined labels as ranges of the escaped view. + *(low · packages/core/src/tools/bounding.ts)* +- [ ] **Approved deferral (maintainer, 2026-10-21). A private-key block split across array elements shows its + body in the journal and display projections.** *Low, unchanged from base.* *(author)* + `redactSecretShapedValue(['--private-key', '', 'MIIE…', ''])` keeps `'MIIE…'` and the + END marker, and `{ argv: ['ssh', '-i', '\nMIIE…', '…\n'] }` stays whole. Both + projections read each string alone; only a summary reads a structured value's JSON as one text. **Re-check:** + `redactSecretShapedWalk` under `projectSecretShapedValue` and `redactSecretShapedValue`; a fix changes `v5:` + journal digests, so it needs the compatibility review of + [effect-journal.md](../reference/shared-core/effect-journal.md#11-secrets-what-a-row-may-hold) §11. + *(low · packages/core/src/tools/bounding.ts)* +- [ ] **Approved deferral (maintainer, 2026-10-21). A summary does not read a key's unquoted value behind an + escaped line break and indentation.** *Low, unchanged from base.* *(round 9 critic, CRIT-R9-1)* + `boundForModel({ stdout: 'database:\n password:\n Qx7vT2mN9pL4kR8s\n user: app' })` shows the value, while + the same text as a string result is masked. The key rule reads a value behind escaped separators only when a + quote opens it (`escapedValueStart`), and the summary's escaped-separator view runs no key rule. **Re-check:** a + fix lets the summary's key rule read an unquoted value behind escaped separators and spaces, or runs it over + `escapedSeparatorView`; the journal is unchanged. *(low · packages/core/src/tools/bounding.ts)* +- [ ] **Approved deferral (maintainer, 2026-10-21). Core does not read a credential glued to a terminal + sequence's final byte.** *Low, unchanged from base.* *(round 9, R9-GEN-4)* `ESC[1msk-` and + `ESC[0mBearer ` keep the credential: the SGR final byte `m` is a word character, so the standalone and + scheme rules see no boundary; built-in-tools.md names the limit. **Re-check:** read those two rules in a view + where CSI sequences are filled, as the backstop's normalized probe does. *(low · packages/core/src/tools/bounding.ts)* +- [ ] **Approved deferral (maintainer, 2026-10-21). The quoted value's later-closing rule has tests but no causal + control.** *Low, partial.* *(round 9, TST-R9-2)* The mutations that keep a single reading, drop the kept + backslashes or skip the summary's second closer are each killed by + `closes a quoted value at the later of its readings with and without the masks`, but no declaration in + `tools/secrets-controls/mutations.mjs` targets it. **Re-check:** add a control (for example `keepingEscapes` + returning `ranges`) when the secrets family is next extended, with its counts. + *(low · tools/secrets-controls/mutations.mjs)* +- [ ] **Approved deferral (maintainer, 2026-10-22). A private-key block cut off before its END marker is not + masked.** *Unchanged from base; found by the author after round 9, outside any review finding.* + `redactSecretShapedText('\nMIIE…')` and the summary of the same text keep the whole body, as does + a block whose END marker starts more than 20,000 characters after its BEGIN marker (`PEM_BODY_MAX`): + `visitPemBlocks` masks only a BEGIN…END pair, and the LLM backstop reads no private-key block. A real tool output + can cut a key short (`head -n 5 key.pem`, a tool that clips its own output). built-in-tools.md names the limit. + **Re-check:** mask an unclosed BEGIN marker through its base64 body lines in every projection, linear at admitted + size; a journal change needs the compatibility review of + [effect-journal.md](../reference/shared-core/effect-journal.md#11-secrets-what-a-row-may-hold) §11. + *(packages/core/src/tools/bounding.ts)* +- [ ] **Approved deferral (maintainer, 2026-10-22). A structured result's summary does not read a private-key + marker split by a line break or tab.** *Unchanged from base; found by the author after round 9.* + `boundForModel({ a: '\nMIIE…\n-----END RSA PRIVATE\nKEY-----' })` shows the block in its summary, + while the same text as a string result is `[redacted]`: JSON escapes the line break or tab, so the wrapped-marker + patterns, which read raw whitespace, see none; more than one space still collapses in the summary's one-line + pass. built-in-tools.md names the limit. **Re-check:** read the wrapped markers in `escapedSeparatorView` as well + (`visitEscapedSeparators`). *(packages/core/src/tools/bounding.ts)* +- [ ] **Approved deferral (round cap, 2026-10-21). A summary reads a wrapped private-key marker only within its + whitespace bounds.** *Medium, regression against base.* *(round 10, R10-SCRUB-1)* + `boundForModel('PRIVATE_KEY=-----BEGIN PRIVATE KEY-----\nMIIE…\n-----END PRIVATE\n KEY-----')` shows the + body in its summary, which base masked: the wrapped patterns take one to eight whitespace characters between + `PRIVATE` and `KEY` and up to 40 characters of type words, and the key's unquoted value takes the BEGIN marker + before the one-line pass can pair the block. A line break with seven or more spaces of indentation in CRLF text, + nine tabs and a 31-column hanging indent leak the same way. built-in-tools.md names the bound. **Re-check:** let + each whitespace run grow as one simple quantifier (`PRIVATE\s+KEY`, type words separated by `\s+` with a bounded + word count) under the admitted-size case deadline, and add summary tests with nine or more whitespace characters + behind a key prefix. *(medium · packages/core/src/tools/bounding.ts `visitPemBlocks`)* +- [ ] **Approved deferral (round cap, 2026-10-21). A U+E000 in a structured result brings back walk-first masking + in its summary.** *Medium, regression against base.* *(round 10, R10-SCRUB-2, INT-R10-1 and PR95-R10-SEC-1)* + `boundForModel({ icon: ' ok', private_key: '\nMIIEAAAAfirst', rest: 'MIIEBBBBsecond\n' })` + shows the second half of the body and the END marker in its summary; `{ argv: ['mysql', "--password='correct", + 'horse', "battery'"], icon: '' }` shows `horse` and `battery'`. The summary marks each whole-masked value with + a U+E000 placeholder and, when the result's JSON already holds that character, falls back to the walk's masks + applied before the text rules. A planted character or a glyph (a Nerd Font icon) is enough. built-in-tools.md + names the limit. **Re-check:** pick a placeholder that the result's JSON does not hold (an unused private-use code + point or a nonce) so that the unwalked reading always runs, and add tests with U+E000 beside a cross-member PEM + block and an argv password. *(medium · packages/core/src/tools/bounding.ts `structuredSummaryText`)* +- [ ] **Approved deferral (round cap, 2026-10-21). A private-key block ends at its first END marker even where + another rule's match is glued to it.** *Medium, regression against base since `b89ac62e`.* *(round 10, + R10-SCRUB-3)* `redactSecretShapedText("api_key='x Wsecret12 -----BEGIN RSA PRIVATE KEY-----' ?sig=<24 a>-----END RSA PRIVATE KEY-----")` + is `api_key='x Wsecret12 [redacted]` (base: `[redacted] ?sig=[redacted] RSA PRIVATE KEY-----`), and + `'\n?s=<26 characters>W'` leaves `W`, which base masked; base's + rules ran in turn, so the glued match erased the marker. built-in-tools.md names the relaxation. **Re-check:** end + a block only at an END marker that no other rule's match is glued to, or read a value's closing quote inside a + PEM range as closing. *(medium · packages/core/src/tools/bounding.ts `visitPemBlocks`, `visitSecretKeyValues`)* +- [ ] **Approved deferral (round cap, 2026-10-21). Whitespace inside a wrapped marker's type words has no killing + test.** *Medium, test gap.* *(round 10, TST-R10-1)* Changing either wrapped pattern's type class + `[A-Z0-9\s]{0,40}` to `[A-Z0-9 ]{0,40}` leaves every bounding case passing, while the summary of + `PRIVATE_KEY=-----BEGIN RSA\nPRIVATE KEY-----\nMIIE…\n-----END RSA PRIVATE KEY-----`, or of the same split in the + END marker, then shows the body. **Re-check:** add both inputs to `reads a private-key marker that a line break, a + tab or a doubled space splits in a summary` and run both class mutants. + *(medium · packages/core/src/tools/bounding.test.ts)* +- [ ] **Approved deferral (round cap, 2026-10-21). The summary's no-throw handling of a whole-masked value with no + JSON form has no killing test.** *Low, test gap.* *(round 10, TST-R10-2)* With `jsonValue` reduced to + `return JSON.stringify(value);`, `boundForModel({ password: 10n, ok: 1 })` and + `boundForModel(['--password', 10n, 'x'])` reject and every test still passes; the existing no-JSON-form test holds + no credential key. **Re-check:** add both inputs (summaries `{"[redacted],"ok":1}` and + `["--password","[redacted]","x"]`) and run the mutant. *(low · packages/core/src/tools/bounding.test.ts)* +- [ ] **Approved deferral (maintainer, 2026-10-22). The journal projection of JSON text keeps a value whose escaped + opening quote an earlier mask took.** *Unchanged from base; the journal half of GEN-R10-1, whose summary half is + corrected.* `redactSecretShapedText(JSON.stringify({ stdout: 'DATABASE_URL=postgres://db.example.com/app?sslmode=require\nPOSTGRES_PASSWORD=""\n' }))` + shows ``: the query value runs on through the JSON-escaped line break and the next line's key and takes the + backslash of the value's escaped quote. A summary opens such a value where that mask keeps its backslashes; the + journal does not, so `v5:` digests do not change, and a test pins that. **Re-check:** drop the `throughCloser` + gate of the kept opening in `visitSecretKeyValues`, with the compatibility review of + [effect-journal.md](../reference/shared-core/effect-journal.md#11-secrets-what-a-row-may-hold) §11. + *(packages/core/src/tools/bounding.ts)* + +## Wave 3 Step 9a.1 — recorded low findings (2026-10-24) + +The internal rigor pass over Step 9a.1 at `1f247772`, run before its round 13 under the +[2026-10-23 acceptance amendment](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-implementation-checkpoint--2026-10-22), +corrected every verified code defect and test gap it could close inside the increment. These low findings are +recorded instead: each needs a change of scope (a new CI leg, a refactor of a Step 7b script) or concerns a +local-only path no claim covers. **Trigger for every entry:** the next change to the file it names, or a +real run that shows the shape. + +- [ ] **The local consumer installs under the shared temporary directory.** *Low, unchanged since Step 7b.* + *(rigor pass, SEC-1/PLAT-6)* `consume.mjs` installs the packed CLI under `mkdtemp(tmpdir())`, so the + installed CLI's optional bare requires (for example `ws`'s `require('bufferutil')`, loaded by the + `provider list` smoke case) fall through to `/node_modules`, which another local user can create on + a POSIX host with a shared `/tmp`. CI runners are single-tenant and macOS's per-user temporary directory is + not shared; packaging.md names the limit. **Re-check:** install under a per-user parent no other user can + write (for example a 0700 directory in the user's cache directory), with a test that plants + `/node_modules/bufferutil` and shows it never runs, as round 10's corepack test does. + *(low · tools/cli-package/consume.mjs)* +- [ ] **An interrupt during the consumer's publication leaves its receipt.** *Low.* *(rigor pass, PLAT-4)* + `publish()` runs inside `completeConsumer`'s interrupt scope, and the interrupt path removes the fixtures, + smoke roots and installation but not `installed-consumer.json`, so a consumer interrupted while publishing + exits 130 beside a success receipt (the producer empties its artifact directory in the same case). A later + local run into the same report directory then fails on the `wx` write; CI uploads only on success. + **Re-check:** write the receipt under a temporary name and rename it after the interrupt scope returns, or + retract it on the interrupt path, with a test that interrupts during `publish`. + *(low · tools/cli-package/consumer-lifetime.mjs)* +- [ ] **Two producers can share one artifact directory.** *Nit.* *(rigor pass, PLAT-9)* Both pass the + emptiness check (`mkdirSync` recursive plus `readdirSync`), so one seal can read the other's tarball. It + needs user error: the local check always uses a fresh root. **Re-check:** require that the directory does + not exist yet and create it without `recursive`, or take an exclusive lock file first. + *(nit · tools/cli-package/produce.mjs)* *Re-checked in Step 9a.2a (2026-10-24), which changed + `produce.mjs` to add the symbolication work directory and the producer-Node refusal, which runs before the + emptiness check: unchanged, still needs user error.* +- [ ] **The consumer's and the smoke worker's composition has no node test.** *Low, unchanged since Step 7b.* + *(rigor pass, F6 and F7)* `consume.mjs` hands `completeConsumer` the `supportRoot` that + `runInstalledSmoke` uses and gives the local source lookup `isolatedGitEnvironment()`, and the worker picks + its `node_modules` owner with `installedModulesOwner` and its root with `createInstalledSmokeRoot`; each + line can be changed wrongly without a node test failing, and a CI consumer, never interrupted, would not + notice a wrong `supportRoot`. The helpers themselves are tested. **Re-check:** export the consumer's + composition with ports, as `produce.mjs` is, and assert that one `supportRoot` binding reaches both calls and + that the Git call's environment equals `isolatedGitEnvironment()`. + *(low · tools/cli-package/consume.mjs, tools/cli-package/smoke/worker.mjs)* +- [ ] **The Windows interrupt, deadline and mode branches are not exercised.** *Low.* *(rigor pass, F14 and + MCG-9)* `stopLiveCommands`' taskkill path, `runCommand`'s ungrouped deadline, `forwardInterrupts`' skip of + ungrouped children, `stopBoundedChildren`'s fallback and the inventory's `not-observable-windows` mode + evidence run only on Windows, and `pnpm test:acceptance-tools` runs only on Linux; packaging.md says so. + **Re-check:** a Windows leg of the interrupt-free lifecycle checks (a stage past its deadline is killed at + once with `graceMs > 0`) and of `tools/bundle-closure/closure.test.mjs`. + *(low · tools/cli-package/process.mjs, tools/cli-bootstrap/process.mjs, tools/bundle-closure/inventory.mjs)* + +Round 13, the acceptance round at `cf97b7f1` (no blocker, high or medium finding; security sitting clean), +reported ten verified findings, six of them distinct. Under the 2026-10-23 bar they are recorded here; the +documentation-only ones were also corrected in the acceptance commit, which changes no code. + +- [ ] **`forwardInterrupts`' group-wide forwarding has no test.** *Low.* *(round 13, TCA-9A1-R13-1)* + `forwardInterrupts` sends the signal to each child's process group (`process.kill(-child.pid, signal)`), as + its docblock says, but changing it to `process.kill(child.pid, signal)` passes `process.test.mjs` and + `check.test.mjs`: the forwarding test checks only the leader's own marker. Every stage child today is + detached into its own group, so the real composition behaves the same. **Re-check:** in the forwarding + harness, have the detached child start a non-detached grandchild in its group that writes a marker on + SIGINT, and assert both markers. *(low · tools/cli-package/process.mjs)* +- [x] **The map-source length bound is pinned on its admitted side only.** *Nit; closed in Step 9a.2a (2026-10-24), whose finalizer tests add the 1025-character source.* *(round 13, TCA-9A1-R13-2)* + `finalize.test.mjs` admits a 1024-character source and refuses a 1031-character one, so loosening + `source.length > 1024` to any bound up to 1030 passes. The source must still resolve to an existing original + file, so nothing is exposed. **Re-check:** add a 1025-character source (`../src/${'a/'.repeat(507)}x.ts`) + expecting `MAP_SOURCE_PATH`. *(nit · tools/cli-package/finalize.test.mjs)* +- [ ] **A rule that lowers the exit code would let the guard exit 0.** *Nit, code half.* *(round 13, + CRIT-9A1-R13-N1)* `release-guard.mjs` sets `process.exitCode = 1` once, so a rule that set + `process.exitCode = 0` or called `process.exit(0)` would make the publish guard exit 0; today's rule always + throws, and the guarantee that no way of invoking the guard skips the refusal holds. The wording that said + the entry point "exits 1" is corrected. **Re-check:** when Step 9d replaces the rule, re-assert the code before + the final line and keep it with an `exit` listener, with a test for each variant. + *(nit · tools/cli-package/release-guard.mjs)* +- [x] **Documentation corrected in the acceptance commit.** *(round 13)* The contract's store-discovery list + now names `NPM_CONFIG_STORE_DIR`, which the code forwards (low; PLAN-9A1-R13-1 = INT-9A1-R13-1, also + PFC-R13-N1 and TCA-9A1-R13-3); the deadline's group kill is described as happening when the grace ends or as + soon as the stage exits (PLAN-9A1-R13-N1); the guard's wording says it sets its exit code to 1 before its rule + runs (CRIT-9A1-R13-N1, wording half); and this file's freshness line is current (PLAN-9A1-R13-N2 = + INT-9A1-R13-N1). **Re-check:** the contract's list equals `STORE_DISCOVERY_VARIABLES` in + `tools/cli-package/candidate.mjs`. *(docs/reference/cli/packaging.md, tools/cli-package/README.md, + .github/workflows/release.yml)* + +Observed in 9a.1's push CI and its preflight for 9a.2 (2026-10-24): + +- [ ] **A migration-running database test lacks the explicit timeout its siblings carry.** *Low.* *(author, + push CI run 40050981969)* In `packages/db/src/session-effect-turns.test.ts`, the parametrized case + `refuses issuance inside an outer %s transaction before any key can escape a rollback` runs every migration + per case under vitest's default 5 s, while its siblings that migrate pass `30_000`; on the advisory Windows + runner the `exclusive` case took 7.7 s and failed. 9a.1 changed no database source. The advisory Windows job + failed 6 times in the last 40 CI runs, each a default 5 s timeout on a different SQLite- or + process-heavy test (`fs-workspace-admission`, `concurrency.e2e`, the runner-abort ownership case and this + one); the job is advisory and every required job passed. **Re-check:** give the case its siblings' `30_000` + bound, and review the other three tests' bounds against the Windows runner, when the file is next edited or + the advisory job next fails. *(low · packages/db/src/session-effect-turns.test.ts)* +- [ ] **`--verbose` never prints a real stack.** *Low, for Step 10a.* *(9a.2 preflight)* + `apps/cli/src/process/render-user-facing.ts` (`ownErrorStack`) prints a stack only when `stack` is an own + data property, but V8 defines a real error's `stack` as an accessor (get/set), so a thrown `Error` prints + nothing under `--verbose`; `render-error.test.ts` constructs a data `stack`, so the tests pass. Step 10a + replaces this raw-stack branch (row 10a). **Re-check:** the replacement prints a real error's frames, tested + with a thrown `Error` rather than a constructed property. *(low · apps/cli/src/process/render-user-facing.ts)* + +## Wave 3 Step 9a.2a — recorded low findings (2026-10-24) + +The internal rigor pass over Step 9a.2a at `1ab4c134`, run before its first review round under the +[2026-10-23 acceptance amendment](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-implementation-checkpoint--2026-10-22), +corrected both high findings, every medium one and every low or nit it could close inside the increment +([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-implementation-checkpoint--2026-10-24)). +These are recorded instead. **Trigger for every entry:** the next change to the file it names, or a real run +that shows the shape. + +- [ ] **The symbolication child is not sandboxed.** *Low; a named limit.* *(rigor pass, SEC-9A2A-2)* + `captureFrames` runs the candidate CLI's runtime import closure — packages that before 9a.2a ran only in the + consumer jobs — as the producer's user, with network access, in the job whose sibling `staging` directory + holds the private maps and manifest. Its isolated environment and owner-only work directory confine it, + and, within the capture process only, the refusal of any module outside the candidate and the refused + keyring and native addons (added after review round 1); a thread or subprocess the candidate started + without the preload (an eval Worker, a `node -e` child) would have none of them, and `relavium import` + starts none, though nothing enforces that or that it stays offline. packaging.md names the limit. + **Re-check:** when Step 9b designs the custody job, run the capture child and every descendant under an + OS sandbox or a separate user that cannot read `staging`. *(low · tools/cli-package/symbolicate.mjs)* + *Re-checked after rounds 1 and 2 (2026-10-24): the module confinement narrows the limit; the record and + packaging.md now say the refusals are in-process only.* *Re-recorded 2026-10-25 (Step 9b preflight + decisions): Step 9b adds a library and synthetic controls and designs no custody job, so the trigger moves + to Step 9d, which designs the release job.* +- [ ] **Two packs of one source can differ in the packed manifest's key order.** *Low, for Step 9c.* *(rigor + pass, integration lens)* Two productions of the same tree packed the five `@relavium/*` `devDependencies`, + which pnpm rewrites from `workspace:*` to versions at pack time, in different orders (`core, llm, shared, + mcp, db` and `core, db, shared, llm, mcp`; the source order is alphabetical). Every other tarball entry was + byte-equal and the sorted contents agree, but the two tarballs' SHA-256 differ (`84065f84…` and + `d0f5b8da…`). The package-test receipt binds the one tarball it inspected, so nothing is wrong today; a + release candidate simply cannot be rebuilt to the same digest. **Re-check:** in Step 9c's no-rebuild rules, + never compare a rebuilt digest, or make the published manifest's key order deterministic, with a test that + packs twice and compares the digests. *(low · tools/cli-package/prepack.mjs and pnpm's pack step)* + +Round 3, the acceptance round at `a92a7602` (no blocker, high or medium finding and no regression; the security +sitting, plan, correctness and integration lenses and the critic clean), reported three verified findings. The +wording nit is corrected in the acceptance commit, which changes no code (the PLAT-9 re-check note above now +names both `produce.mjs` changes, INT-9A2A-R3-1); the two test gaps are recorded here. + +- [ ] **Symbolication's three read bounds have no test.** *Low.* *(round 3, TCA-9A2A-R3-1)* `symbolicate.mjs` + reads the dirty source copy under 8 MiB (`readSourceRevision`), the staged map under 16 MiB (`mapsOf`) and + each re-hashed frame file under 16 MiB; raising all three tenfold leaves `symbolicate.test.mjs` passing. It is + defense in depth: the manifest step reads the same maps and sources under the same bounds, and every file is + bound by digest. **Re-check:** one case per site, as `manifest.test.mjs`'s bounded reads do — pad the dirty + copy, the staged map (re-recording its manifest digest) and a dist chunk (re-sealing) one byte past the bound + and expect a refusal before any digest comparison — then rerun the three mutants. + *(low · tools/cli-package/symbolicate.mjs, tools/cli-package/symbolicate.test.mjs)* +- [ ] **No test shows that an injected anchor table is enforced.** *Nit.* *(round 3, TCA-9A2A-R3-2)* A mutant + that loops over the module's `ANCHORS` instead of the `anchors` argument survives: the only injected table + that reaches mapping, `[...ANCHORS, ANCHORS[0]]`, repeats a reached anchor. Production uses the default. + **Re-check:** add an anchor whose source the manifest maps and whose call site resolves in the source revision + but that no captured frame reaches, and expect `ANCHOR_UNMAPPED`. *(nit · tools/cli-package/symbolicate.test.mjs)* + +## Wave 3 Step 9a.2b — recorded low findings (2026-10-24) + +The internal rigor pass over Step 9a.2b at `77ee5d0a`, run before its first review round under the +2026-10-23 acceptance amendment, corrected its high finding, every medium one and the lows it could close +inside the increment +([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-implementation-checkpoint--2026-10-24)). +These are recorded instead. **Trigger for every entry:** the next change to the file it names, or Step 9d +for the two release entries. + +- [ ] **The runner's cleanup and its copies' placement have no test.** *Low.* *(rigor pass, F5)* Removing + `runMapControls`' removal of its own temporary root, or moving a control's copies out of the staging, passes + every test; the real runner shows the placement, but nothing asserts it. **Re-check:** a test that runs the + runner's composition with recording ports (as `produce.test.mjs` does) and asserts every `scratch` path lies + under the staging and the temporary root is gone afterwards. *(low · tools/cli-package/map-controls-core.mjs)* +- [ ] **An interrupted runner can leave its own temporary directory.** *Low.* *(rigor pass, PLAT-9A2B-6 = + SEC-9a2b-5 = INT-2; round 2, CRIT-9A2B-R2-1)* The producer's interrupt handler empties the artifact directory and + exits, so `runMapControls`' `finally` never removes `relavium-map-controls-*/artifact`; and an interrupt after + `producePackage` returned, during the final identity check, leaves the baseline public tarball and its + `package_test` receipt in it. Nothing private. + **Re-check:** remove the runner's root from the interrupt path too (for example with `exitOnInterrupt` around + the run), with an interrupt test. *(low · tools/cli-package/map-controls-core.mjs)* +- [ ] **The runner's own self-check refusals have no test.** *Low.* *(rigor pass, CRIT-9A2B-2)* + `PACKED_CLI_MAP_CONTROL_CHECKOUT_CHANGED`, `_RESTORE`, `_NO_ANCHOR_CHUNK`, `_TARGET`, `_KIND` and `_ORIGIN` are + checked only by reading and, for the target and kind checks, by a runner mutation run. **Re-check:** unit + tests that drive each refusal through the runner's composition with stubbed ports. *(low · + tools/cli-package/map-controls-core.mjs)* +- [ ] **Release artifacts are not bound to a successful `ci` run at their commit.** *Medium, for Step 9d.* + *(rigor pass, PLAT-9A2B-2 = CLM-9A2B-4 = INT-1)* A release dispatch with `allow_off_main`, or of an intermediate + commit, writes a `package_test` receipt at a commit no `ci` run (so no map-controls run) ever covered; + packaging.md now says the mode is a claim and when it is backed, and the interim guard refuses every such + artifact. **Re-check:** Step 9d's release graph or custody validator requires a successful `ci` run (with + `map-controls`) at exactly the candidate's commit. *(medium · .github/workflows/release.yml)* +- [ ] **The release pack job keeps its `pnpm` dependency cache.** *Low, for Step 9d.* *(rigor pass, + CLM-9A2B-8)* `release.yml`'s pack job sets `cache: pnpm` while `ci.yml`'s `pack-cli` and `map-controls` refuse + it; the store holds only content-addressed public registry packages, which packaging.md permits, so no map + can enter it. **Re-check:** when Step 9d rebuilds the release graph, drop the cache or record why the release + pack keeps it. *(low · .github/workflows/release.yml)* + +Round 2, the acceptance round at `0f73c253` (no blocker, high or medium finding and no regression; the plan and +correctness lenses clean, the security sitting one low), reported seven verified findings. The plan's increment +count, the freshness line and the interrupted-runner wording are corrected in the acceptance commit, which changes +no code (INT-9A2B-R2-1, INT-9A2B-R2-2, CRIT-9A2B-R2-1, the last also amending the entry above); these are recorded. + +- [ ] **The workflow-level environment of the map-controls job is not pinned.** *Low.* *(round 2, SEC-9A2B-R2-1 = + TCA-9A2B-R2-3)* `checkMapControls` pins the job's own `env` but not ci.yml's top-level `env`, which every job + inherits, nor a job `container` or `services` key; a workflow-wide `NODE_OPTIONS` preload or a secret would reach + the controls step with every pin passing. **Re-check:** pin `Object.keys(workflow.env)` to the two Turbo keys the + job overrides, assert `job.container` and `job.services` are undefined, and add the negative cases. + *(low · tools/ci/require-packed-cli.test.mjs)* +- [ ] **A narrowed set of anchor-bearing chunks goes unnoticed.** *Low.* *(round 2, TCA-9A2B-R2-1)* The verdict + derives the per-chunk ids from the runner's own chunk count, required only to be at least one, so a regression + that drops a chunk (for example `.slice(0, 1)` on the derivation) leaves the runner green with 21 controls. + **Re-check:** derive the expected count independently — every ANCHORS source covered by at least one controlled + chunk — or test the composition with recording ports. *(low · tools/cli-package/map-controls-core.mjs)* +- [ ] **Removing a refused control build's `dist` has no test.** *Low.* *(round 2, TCA-9A2B-R2-2)* Deleting the + `rmSync` in `pack()` passes everything; the bundler's maps would stay in the candidate's `dist` (inside the + producer's owner-only root) until the next build. **Re-check:** with the composition test of the F5 entry, assert + `dist` holds no `.map` file after each refused control build. *(low · tools/cli-package/map-controls-core.mjs)* + +## Wave 3 Step 9b.1 — named limits (2026-10-25) + +Recorded with the Step 9b.1 implementation +([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-implementation-checkpoint--2026-10-25)). +**Trigger for every entry:** the next change to the file it names, or a release producer on that platform. + +- [ ] **The custody tests do not run on Windows, and `encryptOnce` refuses there.** *Low; a named limit.* + No Windows job runs `test:acceptance-tools`; the `gpg` on the Windows image's PATH is Git for Windows' + MSYS2 build in `usr\bin`, and MSYS2's own GnuPG under `C:\msys64` is installed off PATH, neither exercised + here, and the tests use POSIX process and permission tools (`ps`, + owner-only modes), so `docs/runbooks/local-dev-setup.md` sends Windows developers to WSL. Windows modes + say nothing about access and no ACL check exists, so since round 2 (SEC-9B1-R2-2) no private root + qualifies there and every encryption refuses `PRIVATE_ROOT` before any GnuPG call. **Re-check:** before + a release producer runs on Windows, add an ACL-based owner-only check for the private root and its work + directory, then run the custody tests on that runner and record which of the two `gpg.exe` builds `resolveGpg` selects + and whether its Windows path handling and gpgconf's line ending pass. *(low · + tools/cli-package/custody.mjs, tools/cli-package/gnupg.mjs)* +- [ ] **An interrupted custody test run can leave its ephemeral key home.** *Low; test-only.* *(rigor pass, + SEC-9B1-PS-2)* `gpg-fixture.mjs` removes the round-trip key's home, its synthetic secret export and its + agent in `after()`, which a SIGINT to `node --test` skips: the `rg-*` and `rge-*` directories under the + temporary directory and an orphaned `gpg-agent` remain. The key is synthetic and never decrypts a + product archive. **Re-check:** stop the agent and remove both directories from a process exit hook as + well. *(low · tools/cli-package/gpg-fixture.mjs)* +- [ ] **The pre-check counts only the keys GnuPG lists.** *Low; fail-closed.* *(rigor pass, SEC-9B1-PS-5)* + `--show-keys` omits a key an import would reject, for example one whose user ID packet was removed, so a + recipient file holding such a key before the pinned one passes the one-key rule. GnuPG then encrypts to + the first key in the file and refuses it, and `KEY_CONSIDERED` and the recipient check refuse any other + outcome, so nothing is published. **Re-check:** count primary-key packets independently of import + filtering if a recipient file is ever produced by anything but `gpg --export`. *(low · + tools/cli-package/custody.mjs)* +- [ ] **Secret material GnuPG neither lists nor reads is admitted.** *Low; round 1, R1-COR-2.* A binary + secret key appended after the armored public block is ignored by GnuPG's listing and encryption, so the + pre-check admits the file and the ciphertext is published. The secret stays in the recipient file and + never reaches the ciphertext, the record or a log. **Re-check:** refuse any byte after the armored public + block's end line (or any secret-key packet tag in a packet walk) if recipient files are ever produced by + anything but `gpg --armor --export`. *(low · tools/cli-package/custody.mjs)* +- [ ] **The atomic publish has no fault-injection test.** *Low; round 1, T-9B1-R1-3.* `publish` writes a + temporary sibling and hard-links it to the final name; writing the final name directly also passes every + test, because no test makes the write fail midway. **Re-check:** inject a write failure (a full + filesystem or a writer port) and assert no file exists under the final name afterwards. *(low · + tools/cli-package/custody.mjs)* +- [ ] **The GnuPG 2.2.8 floor is enforced, never exercised.** *Low; a named limit.* The version check + refuses an older GnuPG, but the tests ran only on 2.5.21 (local) and 2.4.4 (Ubuntu 24.04), so neither + encryption nor the `gpg -d | tar -x` recovery is proven on 2.2.8. **Re-check:** run the custody tests + against GnuPG 2.2.8 (for example in a container that ships it) before a recovery runbook names that + version. *(low · tools/cli-package/gnupg.mjs)* + +## Wave 3 Step 9b.1 — recorded low findings (2026-10-25) + +Recorded with the Step 9b.1 acceptance after round 4 +([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)). +**Trigger for every entry:** the next change to the file it names, or Step 9d's release integration. + +- [ ] **A temporary sibling that cannot be removed after the link reports `UNEXPECTED`.** *Low; round 4, + COR-9B1-R4-1.* `publish` removes its `.partial` sibling in a `finally`. If that removal fails (`EACCES`, + `EROFS`, `EIO`) after `linkSync` succeeded, `encryptOnce` refuses `UNEXPECTED` with no record although the + ciphertext is published; the sibling, ciphertext only and never plaintext, stays beside it, and a retry + refuses `ALREADY_ENCRYPTED`. Example: make the destination 0500 right after the link. **Re-check:** make the + removal best-effort once the link succeeded, or refuse with a code that says the ciphertext was published, and + inject that failure in a test alongside T-9B1-R1-3's. *(low · tools/cli-package/custody.mjs)* +- [ ] **No test pins the metafile's 4 MiB bound.** *Low; round 4, T-9B1-R4-1.* The container's record-bound + tests pad from `RECORD_BYTES` itself, so changing `'metafile-esm.json'` to 8 MiB passes every test while the + contract says 4 MiB; `manifest.test.mjs` pins the other two values. The container and `seal.mjs` share the + constant, so every record still stays within its reader's bound. **Re-check:** assert `RECORD_BYTES`'s literal + values, or pad the metafile to a literal 4 MiB plus one byte. *(low · tools/cli-package/manifest.mjs)* +- [ ] **No test builds the container from a real finalizer's staging.** *Low; round 4, INT-9B1-R4-1.* The + container requires the finalizer record's exact key sets, but every container and custody test uses + `candidate-fixture.mjs`'s hand-written record and a stub metafile. A field added to `finalize.mjs`'s record + would pass every gate and refuse `CONTAINER_RECORD` on every real staging. A reviewer's probe built, + encrypted, decrypted and extracted a real production's staging at `147c2788` without a refusal. **Re-check:** + build and read back the container once from a real production's sealed staging (map-controls' seal port holds + one), or pin the record's key set in `finalize.test.mjs`; at the latest in Step 9d. *(low · + tools/cli-package/container.mjs, tools/cli-package/finalize.mjs)* +- [ ] **The overlap check misses containment through a bind mount.** *Low; round 4, CRIT-9B1-R4-1.* `within` + compares device and inode along each path's own ancestors. It catches one directory reached by two spellings + (a case-insensitive variant, or a bind mount of exactly that directory), but not a root that is a bind mount + of a directory inside the destination, or the reverse: the work directory and its plaintext then sit inside + the destination's tree while GnuPG runs. A bind mount needs privileges to set up. The acceptance commit + narrows the contract's sentence; `custody.mjs`'s comment still says a bind mount can give one directory two + spellings, without the limit. **Re-check:** in Step 9d, create the root and destination as fresh sibling + directories of the job, or compare against the mount table, before relying on the refusal; narrow the comment + on the next change to `custody.mjs`. *(low · tools/cli-package/custody.mjs)* + +## Wave 3 Step 9b.2 — named limits and handoffs (2026-10-25) + +Recorded with the Step 9b.2 implementation +([decisions](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-preflight-decisions--2026-10-25)). +The release modules are library code with synthetic controls only: nothing outside their tests calls them. +**Trigger for every entry:** Step 9d's release integration, or the next change to the file it names. + +- [ ] **An orphaned ciphertext can never be sealed.** *Low; a named limit.* Only a record the gate returned in + the same process seals, so a ciphertext whose seal never succeeds before its process exits — after a crash, or + a refusal on an input the gate does not check, such as a malformed origin or an entry added to the + destination after encryption — is named by no record. The gate refuses a linked or unresolvable destination, + any entry but the tarball and a different tarball before encrypting, which removes the likeliest causes. The candidate then needs manual recovery + (packaging.md's missing-asset state); it is never re-encrypted. **Re-check:** in Step 9c's reservation + states, classify such a directory as incomplete and route it to manual recovery. *(low · + tools/cli-package/release-custody.mjs)* +- [ ] **Reuse cannot re-check the ciphertext's recipients.** *Low; a named limit.* `verifyRetainedCandidate` + binds each asset's size and SHA-256 to the receipt, and the receipt's recipient to the expected history, but + re-reading the ciphertext's public-key packets (`ENC_TO`) needs GnuPG, which reuse never calls. The digest + ties the bytes to the ones the sealer re-read after encryption checked their single recipient. **Re-check:** + if a retained ciphertext can ever come from anything but the sealer, verify its recipients with the isolated + runner before reuse. *(low · tools/cli-package/release-binding.mjs)* +- [ ] **A sibling left after the receipt's link keeps the candidate from reuse.** *Low; a named limit (rigor + pass, SIT-9B2-6).* When the sealer cannot remove its `.partial` sibling after the link, the receipt is + published and the seal succeeds, but `verifyRetainedCandidate` admits exactly three entries and refuses + `CANDIDATE_ENTRIES` until the sibling is removed by hand; a test pins that state. **Re-check:** in Step 9c's + state classification, report such a directory as sealed with a stray sibling. *(low · + tools/cli-package/release-custody.mjs)* +- [ ] **The fixture denylist covers only the fingerprints the fixtures' pins file lists.** *Low; a named limit + (rigor pass, CLAIM-9B2-2).* `two-encryption-subkeys.asc` carries a second encryption subkey, + `7BC68AC7D2944C2AB3240A521D784594073951D5`, that `custody-fixtures/pins.json` does not list, so a production + history naming it would load. The loader reads no key material (it makes no GnuPG call), and encrypting to it + would still need its armored key in the committed recipient file. **Re-check:** list every committed fixture + fingerprint in the denylist when the fixtures are next generated. *(low · tools/cli-package/release-recipient.mjs)* +- [ ] **`encryptOnce` called directly follows a link spelled with a trailing separator.** *Low; a named limit + (round 2, COR-R2-2).* 9b.1's `plainDirectory` and `privateDirectory` lstat the path as given, and `lstat('link/')` + follows the link, so a direct caller of `encryptOnce` can encrypt into a linked destination or use a linked + private root. The release gate requires both paths in normalized absolute form and refuses a link itself, so + no release path reaches this; nothing outside the gate and the tests calls `encryptOnce`. **Re-check:** on the + next change to `custody.mjs`, require normalized absolute paths there too. *(low · tools/cli-package/custody.mjs)* +- [ ] **The release entry points accept the test ports.** *Low; a named limit (round 1, SIT-R1-5).* + `encryptCandidate` forwards `encryptOnce`'s `run` and `gpg` ports and `sealReleaseBinding` takes a `write` + port, in production scope too, so an in-process caller could replace GnuPG and defeat its checks. Like the + brand, the ports guard against an honest mistake, not hostile code in the same process. **Re-check:** in Step + 9d, call both without ports from the release job and add a permanent control that no product call passes + them. *(low · tools/cli-package/release-custody.mjs)* +- [ ] **The release modules do not run on Windows.** *Low; a named limit.* Like the 9b.1 tests, the release + tests run only in `test:acceptance-tools`, which no Windows job runs; `encryptCandidate` refuses there + through `encryptOnce`'s `PRIVATE_ROOT`, while reuse is platform-neutral but unexercised. **Re-check:** with + the 9b.1 Windows limit. *(low · tools/cli-package/release-binding.mjs)* +- [ ] **Step 9d handoffs.** *Scheduled → Step 9d.* Wire `encryptCandidate` and `sealReleaseBinding` into the + originating job, with the origin from `originFromEnvironment`; bind the receipt's `origin` to the trusted run + and attempt (on reuse, the original run) and `mapCheck` to a successful `ci` run with the product map + controls at `source.commit`; derive release smoke's public inventory and package metadata from the + digest-verified tarball with the package-test inspector; check the production recipient before any + reservation, so a recipient refusal never leaves an incomplete reservation; if the package-test receipt + travels beside the tarball, widen the gate's, the sealer's and reuse's directory admission together, since all + three admit exact entries (round 2, CRIT-9B2-R2-1); replace the interim guard; then update + `release-scope.test.mjs`'s interim control that no product path or workflow names a release module, keeping + its permanent controls (the loader takes no argument and its test seam stays out of product code; only the + gate imports `encryptOnce`); and include `tools/cli-package/custody-fixtures/pins.json` in every checkout + that loads the production recipient, because the loader reads it as its denylist. The guard test's + non-canonical `relavium-0.1.1.tgz.gpg` name is corrected with the guard. *(scheduled · + tools/cli-package/release-custody.mjs, tools/cli-package/release-guard-core.mjs)* + +## Wave 3 Step 9b.2 — recorded low findings (2026-10-26) + +Recorded with the Step 9b.2 acceptance after round 4 +([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-acceptance--2026-10-26)). +**Trigger for every entry:** Step 9d's update of `release-scope.test.mjs` and the release modules, or the next change +to the file it names. + +- [ ] **The `custody.mjs` control does not decode identity or legacy octal escapes.** *Low; round 4, PLAN-9B2-R4-1, + COR-R4-2.* `namesFile` decodes `\x`, `\u`, `\u{…}` and percent escapes, but a specifier such as + `'./custo\dy.mjs'` (an identity escape) or a sloppy-mode octal escape still loads `custody.mjs` unseen. The + spelling is deliberate, never an honest mistake. **Re-check:** also match a variant with each stray backslash + removed and octal escapes decoded, or refuse any backslash in a specifier. *(low · + tools/cli-package/release-scope.test.mjs)* +- [ ] **The static controls skip `tools/lint-fixtures/assert-fence.mjs`.** *Low; round 4, PLAN-9B2-R4-2.* The scan + treats all of `tools/lint-fixtures/` as test-only, but `assert-fence.mjs` is CI tooling (`pnpm lint:fence-check`), so + a `custody.mjs`, fixture or test-seam reference there passes the controls. **Re-check:** exempt only the quarantined + `.ts` fixtures and add the script to the scan's own assertion. *(low · tools/cli-package/release-scope.test.mjs)* +- [ ] **The closure's specifier scan has blind spellings.** *Low; round 4, SIT9B2-R4-L1, COR-R4-3, T-R4-3.* A comment + between `from` or `import` and the specifier, a specifier containing the other quote character, and computed member + access such as `process['getBuiltin' + 'Module']` each pass the closure control, which proves reuse starts no + process. Each spelling is deliberate, and today's closure contains none. **Re-check:** strip comments and tokenize + string literals properly before scanning, and refuse computed access on `process` in closure files. *(low · + tools/cli-package/release-scope.test.mjs)* +- [ ] **The loader reads the root by path after its link check.** *Low; round 4, COR-R4-4.* `lstat` then `readdir` + and the two reads leave a window in which a swapped root is read through a link and branded. Only a committed, + maintainer-owned directory is read in production. **Re-check:** open the root with `O_DIRECTORY | O_NOFOLLOW` and + compare its device and inode before and after the reads, as `readOrdinaryFile` does for files. *(low · + tools/cli-package/release-recipient.mjs)* +- [ ] **Some exact rules have no isolating test.** *Low; round 4, T-R4-1, T-R4-2, T-R4-4.* No history puts its current + entry before a retired one, so choosing the last entry would pass; the toolchain's prerelease suffix (its character + set and 64-character bound) and the target's digit count (three admitted, four refused) are unpinned; and a nit: + the sealed-link case asserts the first link's target, and the non-POSIX fallback of the linked-tarball case removes + the tarball instead. **Re-check:** add those cases on the next change to the release tests. *(low · + tools/cli-package/release-recipient.test.mjs, tools/cli-package/release-binding.test.mjs, + tools/cli-package/release-custody.test.mjs)* + +## PR #96 review — recorded findings and Sonar dispositions (2026-11-04) + +Recorded by the PR #96 review correction. The review's verified high and medium findings and SonarCloud's 27 bugs and +10 vulnerabilities were fixed in that correction; these are the verified lows and the code smells the maintainer +ruled on 2026-11-04 to record rather than refactor now. **Trigger for every entry:** the next change to the file it +names, unless the entry names another. + +**Sonar code smells (gate-neutral; maintainability stays A):** + +- [ ] **Cognitive complexity (`S3776`) in fifteen functions.** *Low.* `packages/core/src/tools/bounding.ts` (five, + 25–38), `packages/llm/src/scrub-secrets.ts:411`, `tools/cli-package/source-map.mjs:47` (35), `map-controls-core.mjs`, + `release-custody.mjs`, `candidate.mjs`, `release-binding.mjs`, `symbolicate.mjs`, `ustar.mjs`, + `tools/bundle-closure/closure.mjs` and `inventory.mjs`. Each is security-critical redaction or release tooling, so a + refactor needs its own mutation sweep and review. *(low · the files named)* +- [ ] **Regex complexity (`S5843`) in eight patterns.** *Low.* `bounding.ts`'s standalone and key rules and five + `scrub-secrets.ts` rules: their "complexity" is the length of a finite name or shape alternation (as the existing + `NOSONAR` notes say), and each is a causal-control anchor. The three `S5869` reports on `scrub-secrets.ts:138` are + the `i` flag making `A-Z` and `a-z` overlap, not a duplicate. **Re-check:** split a rule only with its controls. + *(low · packages/core/src/tools/bounding.ts, packages/llm/src/scrub-secrets.ts)* +- [ ] **`S1994` on the closing-view loop.** *Low.* `bounding.ts`'s backward scan tests `next` but steps `index`; a + `while` rewrite must keep the step that its `continue` skips. *(low · packages/core/src/tools/bounding.ts)* +- [ ] **`S6582` in `media-store-paths.ts` (three).** *Low.* The optional-chain rewrites are equivalent, but the lines + are filesystem causal-control anchors; change them with those controls. *(low · packages/db/src/media-store-paths.ts)* +- **DECLINED — `S9382` (seven sequential awaits) and `S7758` (five `charCodeAt`).** The awaits are ordered process + steps by design (`map-controls-core.mjs`, `pipeline.mjs`, `symbolicate.mjs`), and `source-map.mjs` is a code-unit + VLQ parser that advances one code unit at a time, which `codePointAt` would misstate. + +**Redaction:** + +- [ ] **The LLM backstop leaves a `/` in a URL password.** *Low.* `scrubSecrets('https://user:pa/ss@host')` keeps the + password, because its userinfo pattern stops at `/`; core masks it since this correction. Matching core needs its + port guard (`host:8080/a@b` stays public). *(low · packages/llm/src/scrub-secrets.ts)* +- [ ] **A glued `-pSECRET` (`mysql -uroot -pSecret`) is not masked.** *Low.* A glued `-p` is ambiguous with + `cp -pR` and `tar -pxf`, so only the separate and `=` spellings are read. *(low · packages/core/src/tools/bounding.ts)* +- [ ] **The object-key rule masks count keys such as `max_tokens`.** *Low.* Its plural folding reads `tokens` as + `token`, so two calls that differ only in `max_tokens` share a journal identity. The argv rule reads words as written + and does not. *(low · packages/core/src/tools/bounding.ts)* +- [ ] **Zero-width characters break the `sk-` and `Bearer` shapes; header names are matched with hyphens only.** + *Low; verified.* *(low · packages/llm/src/scrub-secrets.ts)* +- [ ] **A model-supplied tool name reaches an error message unmasked and unbounded.** *Low; verified.* *(low · + packages/core/src/tools/errors.ts:21)* +- [ ] **Two redaction engines and two fixed-point copies; the parity test compares booleans only.** *Low; verified.* + *(low · packages/core/src/tools/bounding.ts, packages/llm/src/scrub-secrets.ts)* +- [ ] **Permanent causal controls for this correction's three mechanisms.** *Low.* The credential-flag words, the + `,`/`/` password reading and the omitted-media bytes are pinned by permanent tests and a manual mutation sweep (every + mutant killed), not by `check:secrets-mutations` controls. The secrets evidence has about 15 MiB of headroom per + shard. *(low · tools/secrets-controls/mutations.mjs)* + +**Release, custody and tooling (verified lows):** + +- [ ] **`isProductionRecipient` accepts seam values; the distinction rests on a grep control.** *(low · + tools/cli-package/release-recipient.mjs:52)* Trigger: Step 9d or Wave 4b. +- [ ] **A revocation is visible only in the committed recipient snapshot.** *(low · docs/reference/cli/packaging.md)* +- [ ] **`resolveGpg` checks `isFile`, not `X_OK`.** *(low · tools/cli-package/gnupg.mjs:33)* +- [ ] **A developer's own `core.excludesFile` directory can let private untracked files into the candidate.** The + public tarball is not affected; an explicit `RELAVIUM_GIT_EXCLUDES_FILE` naming a directory already fails in Git. + *(low · tools/cli-package/candidate.mjs)* +- [ ] **`prepack.mjs` selects the finalizer by environment variable; `symbolicate.mjs` may load TypeScript from a parent + directory.** The closure check is equivalent for the first. *(low · tools/cli-package/prepack.mjs:36, + tools/cli-package/symbolicate.mjs:306)* +- [ ] **`runCommand` does not follow detached grandchildren after a successful exit; `causal-controls` starts a + detached group without a Ctrl-C handler; the smoke's cleanup can skip its PID kill after a parse error.** *(low · + tools/cli-package/process.mjs, tools/causal-controls/, tools/cli-package/smoke/)* +- [ ] **`mcp-orphan.e2e` scans the process table by fixture name, which sibling tests share; `vi.waitFor` keeps its + 1-second default.** *(low · packages/mcp)* +- [ ] **The case-insensitive custody tests run in no CI job.** *Medium as reviewed, low in practice.* Ubuntu skips + them and no macOS or Windows job runs `custody.test.mjs`; the local `pnpm run ci` on macOS runs them. A CI lane + needs a macOS job inside `ci`'s success aggregation. *(low · tools/cli-package/custody.test.mjs:830, .github/workflows/ci.yml)* +- **DECLINED — declaring `RELAVIUM_CLI_CANDIDATE_STAGING` in `turbo.json`.** The candidate build never runs through + Turbo (`pnpm exec tsc` and `pnpm pack`), and Turbo 2's strict environment mode withholds an undeclared variable, so + a Turbo build is always map-free; declaring it would let one become a candidate build. + +**Not yet recorded here:** the review's unverified lows (its section F) wait for the consolidated file from that +review session, and two roadmap lows ("DONE entries whose body still describes the open state", "MVP items without an +annotation") need the entries they name. + +## Wave 3 NI — recorded low findings (2026-10-25) + +Round 1 of the native-identity increment at `fc5af701` accepted it with these lows +([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). **Trigger for +every entry:** the next change to the file it names. + +- [ ] **The lifecycle smoke's precondition reads can revert to numbers unnoticed.** *Low; test-side.* *(round 1, + PLAN-NI-R1-1)* Restoring `fs.lstatSync(candidate.owned).ino` at `tools/budget-replay-compat/lifecycle-smoke.mjs` + lines 325 and 369 still passes `pnpm smoke:budget-replay`: in the collision mode a bigint and a Number never + compare strictly equal, and in the other modes the real inodes differ. Production's retention identity is + witnessed separately. **Re-check:** assert both values are bigints before the comparison; reverting either read + must then fail the smoke. *(low · tools/budget-replay-compat/lifecycle-smoke.mjs)* +- [ ] **Nothing tests the projection's `mtimeMs` formula.** *Low.* *(round 1, NI-T-1)* Replacing it in + `packages/db/src/media-store-paths.ts` `exactStats` with whole milliseconds (`Number(info.mtimeNs / 1_000_000n)`) + or whole seconds passes every media-store test. The bit-identical claim rests on an author probe of 307 files, and + a silent change could shift `listHandles()`'s `mtimeMs` and the sweep's age inputs by up to a second. + **Re-check:** a case that sets a fractional and a negative fractional mtime and asserts the projection is + `Object.is`-equal to `lstatSync(path).mtimeMs`; both mutants must fail. *(low · + packages/db/src/media-store-paths.ts)* +- [ ] **The catalog's dense-multibyte case catches only a byte bound below 1.5 times the character cap.** *Low.* + *(round 1, NI-T-2)* `apps/cli/src/workflows/catalog.test.ts` writes three-byte characters to about + 1.5 × `MAX_SOURCE_CHARS` bytes, so `BigInt(MAX_SOURCE_CHARS * 3)`, `* 2` or a `>=` boundary in `catalog.ts`'s + precheck still pass, although a file of four-byte characters just under the character cap would then be refused. + **Re-check:** use four-byte characters to within one character of 4 × `MAX_SOURCE_CHARS` bytes; the 3×, 2× and + `>=` mutants must fail. *(low · apps/cli/src/workflows/catalog.test.ts)* + +## Native file identity on Windows — approved increment (2026-10-24) + +Found by Step 9a.2a's push CI (the advisory Windows job of run 40105286543). The maintainer approved a separate +increment, after 9a.2b and before 9b (register row NI), rather than a deferral; this entry tracks it until then. +NI is independently accepted on `development` at `fc5af701` (2026-10-25, +[acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)); the item stays +open until the Windows clause below closes it after three consecutive green advisory Windows runs on pushed heads. +**Closed 2026-10-26:** the third consecutive green advisory Windows run (push CI run 40276471445 at `5a328bca`, probe +`0 of 200`) closed the Windows clause ([NI acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)); the scoped limits below stay open. + +- [x] **Identity checks compare `ino` as a JavaScript number, which Windows file IDs overflow.** *High; security, + Windows.* `statSync`, `lstatSync`, `fstatSync` and `FileHandle.stat` without `{ bigint: true }` return `ino` as a + number, but on Windows `ino` is the volume's 64-bit file ID; above 2^53 neighbouring IDs round to the same value, + so a replaced file or directory can compare as the original. `apps/cli/src/engine/tool-host/fs-policy.test.ts`, + "revokes a selected subdirectory replacement even when its enclosing grant is unchanged", read the replacement on + the Windows runner (its first failure in sixty runs). Sites: `apps/cli/src/engine/tool-host/fs-policy.ts` + (`pinDirectory`/`validatePin`), `apps/cli/src/workflows/catalog.ts` (the open check), and + `packages/db/src/media-store-paths.ts` (directory pins and `assertOwnedFile`), `media-store-read.ts` (the + read's before/after check) and `media-store-temporaries.ts` (`same` and the `Number(...)` comparison in + `nativeFile`). **Re-check:** every identity is read with `{ bigint: true }` and compared as `bigint` (or its exact + string), with a replacement test per site that fails when the comparison goes back to a number; the Windows job + passes the fs-policy test repeatedly. *(high · the five files above)* + +Scoped limits recorded with NI (2026-10-25): + +- [ ] **ReFS, Dev Drive, FAT and exFAT do not give unique, stable 64-bit file IDs.** *Low; Windows ReFS/Dev + Drive and FAT/exFAT volumes.* Microsoft documents that the 64-bit identifier "is not guaranteed to be unique on + ReFS", and that a FAT file ID is derived from the position of the file's directory entry, which renaming a file + to a longer name moves. NI makes every comparison exact, so the claim is exact identity on filesystems whose + 64-bit file IDs are unique, such as NTFS, ext4 and APFS. On ReFS two files could share an index; on FAT or exFAT + a replacement created in a freed directory entry could inherit the pinned ID, so a check would admit it (fail + open), whatever the comparison's precision. Minimal example: a pinned directory `selected` on a FAT volume is + renamed to `selected-old` and a new `selected` is created in the freed entry. **Re-check:** if ReFS, Dev Drive, + FAT or exFAT workspaces become a supported target, read the 128-bit `FILE_ID_INFO` identity, pair the identity + with a second discriminator such as creation time, or refuse such volumes; native code or a dependency needs an + ADR. *(low · `apps/cli/src/engine/tool-host/fs-policy.ts`, `packages/db/src/media-store-paths.ts`)* +- [ ] **The causal-control source graph's hard-link check still compares `ino` as a number.** *Low; tooling, + fail-closed.* `tools/usage-coverage-controls/source-graph.mjs` asserts that each copied source file's `ino` + differs from its original's, so a collision can only refuse a copy falsely; four causal families share the + file, and NI left it unchanged rather than perturb them. **Re-check:** switch both reads to `{ bigint: true }` + the next time that file changes. *(low · `tools/usage-coverage-controls/source-graph.mjs`)* + ## Phase 2.6.5 deferrals — `CR-93` and `CR-95` (long half), recorded 2026-09-06; `CR-70`, `CR-94` (remainder) and the in-turn overflow continuation, recorded 2026-09-14; the errored-or-aborted turn residual, the pre-upgrade free-page bytes and the estimator's calibration, recorded 2026-09-18 **Phase closure — 2026-10-11:** W8 merged to `main` in PR #92 and Phase 2.6.5 is complete @@ -401,6 +1140,8 @@ surface ships. CR-95's long continuation and the exact-capacity residual remain ADR-0081's role boundary; ids unique per session; validity proven by the live conformance suite; a user-scoped `[preferences]` toggle. *(high · packages/core/src/engine/agent-session.ts, turn-messages.ts; `CR-70`, ADR-0095)* + **Scheduled 2026-11-04 (D67):** v0.2.1, right after the CLI MVP's v0.2.0, designed together with ADR-0096 §5's + in-turn overflow continuation ([current.md](current.md#next-work--the-cli-mvp)); the trigger above is unchanged. - [ ] **Binding a budget allowance to its model, and giving it an expiry (`CR-94`, remainder).** *Medium.* [ADR-0097](../decisions/0097-a-budget-approval-is-an-allowance-not-an-exemption.md) builds the minimum @@ -429,6 +1170,8 @@ surface ships. CR-95's long continuation and the exact-capacity residual remain - **Trigger.** The first surface that needs to show, or resume from, what a failed turn did — a coding assistant replaying a half-finished edit, or an audit view over a session's tool use. *(medium · apps/cli/src/chat/persister.ts, packages/core/src/engine/agent-session.ts; `CR-71`, ADR-0095)* + **Pulled into the CLI MVP 2026-11-04 (D67, MVP-4) as "the CR-71 residual"** — CR-71 itself closed in `W7`: a failed, denied or aborted turn stays in context as a + content-free interruption marker (ADR-0095 §1), behind an ADR-0095 amendment drafted in Gate 0. - [ ] **Two at-rest residuals `secure_delete` does not cover: a blocked WAL checkpoint, and pages freed before the upgrade.** *Low.* Recorded 2026-09-18 with the `secure_delete` decision, and sharpened the same day after a @@ -1265,6 +2008,9 @@ so directly: these "should remain visible rather than disappear behind the green > **Found while fixing 2.6.C's F1** (the `/models` reseat blanking the alt-screen viewport). Deliberately scoped > OUT of that fix, and recorded here so it is tracked work rather than a discovery that gets lost. +> +> **Scheduled 2026-11-04:** the CLI MVP's MVP-2 repaints a resumed session's transcript +> ([current.md](current.md#next-work--the-cli-mvp)); close this entry when that lands. `relavium chat-resume ` restores the model's context from `history.db`, but opens on an **empty viewport** — the prior conversation is never repainted. This is **not** the 2.6.F regression F1 is: it has been @@ -2171,6 +2917,28 @@ future test cannot silently re-acquire it. ## Wave 3 approved later-wave ownership — 2026-10-16 +- [ ] **Scheduled → Wave 4b: production recipient key constraints.** Approved with the Wave 3 + Steps 9–19 rulings (2026-10-19, [Step 9 mechanics ruling](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19)): + the confirmed production recipient for encrypted release custody must be an OpenPGP v4 key + with a cv25519 or RSA≥3072 encryption-capable subkey. Wave 3 Step 9 implements custody with + synthetic keys only; an actual release fails closed until this recipient, its offline + recovery check and backup ownership exist. *Added 2026-10-25 (Step 9b preflight decisions):* the key + must not advertise the AEAD feature, because custody requires SEIPDv1 with MDC and AES-256 so that + `gpg -d | tar -x` with GnuPG ≥ 2.2.8 stays true; and the committed recipient file is the export of the + primary with only the pinned encryption subkey (`gpg --armor --export '!'`), + because a second usable encryption subkey refuses. *Added after Step 9b.1's review round 1:* its + cipher preferences must include AES256, since custody passes an AES-256 preference and refuses any + other cipher. *Added with Step 9b.2:* the recipient is committed at + `tools/cli-package/release-recipient/` as `recipient.asc` and `pins.json`, whose history is append-only + (exactly one `current`, a rotated recipient `retired`, never removed); a fixture fingerprint refuses. Wave 4b + then updates `release-recipient.test.mjs`'s absence test and `release-scope.test.mjs`'s interim control that no + recipient directory is committed. Any expiry margin is passed as the `now` the custody check uses. The + loader sees one revision of `pins.json` and cannot detect a removed or rewritten entry, so Wave 4b adds a + control that every committed revision keeps the previous one's entries, with only `current` becoming + `retired` (rigor pass, CLAIM-9B2-3). With the files committed, Wave 4b also replaces the absence test with a + load through `loadProductionRecipient()` itself, so the production path's own denylist and root wiring are + exercised: in Wave 3 the absent directory refuses before either is read (round 1, TEST-9B2-R1-4). + - [ ] **Scheduled → Wave 5a: alt-screen word wrapping (#61).** The maintainer approved retaining character-wrap replacement with render-v2/D18 and picker positioning #60, so that renderer restructuring and the palette sweep occur @@ -2179,3 +2947,5 @@ future test cannot silently re-acquire it. [approved execution plan](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and [Wave 5a owner](current.md#wave-5a--paint-before-you-build) retain the split; close only after actual word-wrap implementation and regression evidence land. + **Moved 2026-11-04 (D65):** into the CLI MVP's MVP-2 ([current.md](current.md#next-work--the-cli-mvp)); the Wave 5a + ownership ends, and the close condition above stands. diff --git a/docs/roadmap/phases/phase-2-cli.md b/docs/roadmap/phases/phase-2-cli.md index 91a92851..1121f33c 100644 --- a/docs/roadmap/phases/phase-2-cli.md +++ b/docs/roadmap/phases/phase-2-cli.md @@ -911,3 +911,212 @@ All must be true to start Phase 3 (desktop): | No OS keychain in CI/headless hosts. | Cannot run in pipelines. | Documented env-var → `secrets.enc` fallback with a hard error on absence, no silent plaintext (2.C). | | High token volume overwhelms the TUI. | Dropped events / janky render. | Throttle/batch with `sequenceNumber` gap-detection + resync; verify event count vs JSON stream (2.E). | | Cross-OS install differences. | "Works on my machine" binary. | Multi-OS smoke tests of the published artifact before sign-off (2.L). | +## Status record moved from current.md — 2026-11-04 + +> Moved verbatim from `docs/roadmap/current.md` as it stood at `5a328bca` by the +> [PR #96 review correction](phase-2.5.5-hardening-and-remediation.md#moved-by-the-pr-96-review-correction--2026-11-04). +> Only relative links were rewritten for this directory. + +**Phase 2 — CLI (milestone M3) is feature-complete** (every in-phase workstream 2.A–2.S merged; published as +**v0.1.1**). The CLI is the first real +`@relavium/core` consumer and doubles as the engine's regression harness — validating the +engine API ergonomics before the desktop and VS Code surfaces. **Landed:** the CLI skeleton + +process contract (**2.A**) and the two-level config-resolution loader (**2.B**), both ✅ Done +(PR #40, 2026-06-22) behind [ADR-0047](../../decisions/0047-cli-framework-commander-ink-clack.md) +(commander/ink/@clack) and [ADR-0048](../../decisions/0048-toml-config-parser.md) (smol-toml); and +**2.D** (`relavium run` wired to `@relavium/core` — the M3 keystone and first real engine +consumer), ✅ Done (PR #41, 2026-06-22), which also adds the `defaultProviders()` seam registry; +and **2.F** (the `--json` CI machine-output contract — pure-NDJSON stdout, diagnostics → stderr), +✅ Done (PR #42, 2026-06-22) behind [ADR-0049](../../decisions/0049-cli-machine-output-contract.md); +and **2.K** (the engine regression harness, now the engine's CI regression gate), ✅ Done +(PR #43, 2026-06-23) — **reaching global milestone M3**; and **2.H** (durable local run history via +`@relavium/db` — the `RunStore` writer + read API the gate-resume/list/logs/status surfaces consume), +✅ Done (PR #44, 2026-06-23) behind [ADR-0050](../../decisions/0050-cli-history-db-at-rest-posture.md); +and **2.C** (the `relavium provider` commands — a provider registry + API keys in the OS keychain via +`@napi-rs/keyring`, resolved keychain → `RELAVIUM__API_KEY` env var → error), ✅ Done +(PR #45, 2026-06-23) behind [ADR-0019](../../decisions/0019-cli-node-keychain-library.md) + +[ADR-0006](../../decisions/0006-os-keychain-for-api-keys.md) (no new ADR — `secrets.enc` deferred past v1.0); +and **2.E** (the `ink` streaming TUI — the third `RunRenderer` over the one event bus: live per-node status ++ spinners, the active node's streaming tokens, a running cost footer, a persistent final summary; cooperative +Ctrl-C cancel), ✅ Done (PR #46, 2026-06-24) behind [ADR-0047](../../decisions/0047-cli-framework-commander-ink-clack.md) +(ink + React 19, confined to `apps/cli`; no new ADR); +and **2.G** (the interactive human-gate prompt — a `@clack/prompts` card during `run` — plus the out-of-band +`relavium gate ` cross-process resume: reload snapshot → reconstruct checkpoint → `resumeFromCheckpoint`, +idempotent, secret-input fail-closed), ✅ Done (PR #47, 2026-06-24) behind +[ADR-0047](../../decisions/0047-cli-framework-commander-ink-clack.md) (`@clack/prompts`; Node floor 20.11→20.12; +no new ADR) — **fully closing 2.K's deferred gate-resume half**; +and **2.I** (the read commands `list` / `logs` / `status` / `gate list` over durable history — go/no-go #2, the +read side; surfaces the pending `gateId`s the 2.G `gate` command points at), ✅ Done (PR #48, 2026-06-24) +(no new ADR — an additive workflow-agnostic `@relavium/db` read seam + a `@relavium/core` `parseAgent`); +and **2.L** (packaging, distribution & install verification — go/no-go #7, the last gate-closing spine PR: the +`tsup` engine-inlined ESM bundle, the bundle-closure drift guard, and the tag-triggered cross-OS install-smoke +`Release CLI` workflow), ✅ Done (PR #49, 2026-06-24) behind +[ADR-0051](../../decisions/0051-cli-distribution-thin-bundle-private-engine.md) — **closing go/no-go #7, so the +Phase-2 spine is complete and all seven Phase-3 exit criteria now hold (Phase 3 may start)**. +**Also landed — the first additive lane:** **2.S** (media host-wiring — the surface half of the multimodal +sub-spine: the `model_catalog` reader → `resolveMediaSurface` routing + the D15 catalog load-check shared by +`run`/`gate`, the content-addressed `MediaStore` de-inline to a `media://` handle, the SSRF-validated +`EgressCapability.fetch` egress, the containment-checked `save_to` write port, durable fail-cost on the terminal +events, the produced-media render surface, and the best-effort run-end host media GC), ✅ Done (PR #52, 2026-06-25) +behind [ADR-0042](../../decisions/0042-engine-media-storage-substrate-mediastore-deinline-retention.md)–[ADR-0046](../../decisions/0046-inline-media-out-via-generate-streaming-triad-deferred.md) +(no new ADR). +**Also landed — the first user-facing `AgentSession` surface:** **2.M** (`relavium chat` — the agent-first +interactive REPL over `AgentSession`: streaming tokens, tool-call annotations, the FS-scope tier + `allowedCommands` +allowlist honored, `git_commit` denied; `/exit` / `/cancel` / an input-stream EOF / raw-mode Ctrl-C all end the +session with **exit code 4** — over ONE framework-free command core driving both an `ink` TTY app and a plain +non-TTY line loop; a built-in default agent over `[chat].default_model` for a zero-config first run; durable +per-turn persistence to the shared `history.db` that round-trips via `reconstructSessionState`; the ADR-0028 +cost cap wired; model output + pasted input sanitized of terminal control sequences at the display boundary), +✅ Done (PR #54, 2026-06-26) — **no new ADR** (covered by [ADR-0024](../../decisions/0024-agent-first-entry-point-agentsession.md), +[ADR-0047](../../decisions/0047-cli-framework-commander-ink-clack.md), [ADR-0028](../../decisions/0028-workflow-resource-governance.md), +[ADR-0050](../../decisions/0050-cli-history-db-at-rest-posture.md), [ADR-0029](../../decisions/0029-tool-policy-hardening.md)). +`read_media` **input** access (D12) — which 2.S had pointed at 2.M — was **split into a dedicated, +security-reviewed follow-up** (maintainer-approved); the 2.M REPL shipped without it (tracked in +[deferred-tasks.md](../deferred-tasks.md)). +**Also landed — the rest of the agent-first chat family:** **2.N** (`relavium chat-resume` — reload + continue a +persisted session over a shared REPL), **2.O** (`chat-list` — over a new additive `SessionStore.listSessions` +read seam), **2.P** (`chat-export` + the in-REPL `/export` — session → `.relavium.yaml` scaffold, [ADR-0026](../../decisions/0026-session-export-to-workflow.md)), +and **2.Q** (`chat --json` — a headless `SessionEvent` NDJSON driver — + the one-shot `relavium agent run` with a +minimal in-house `--fixture` cassette for deterministic offline replay), all ✅ **Done (PR #55, 2026-06-26)** — +**no new ADR** — completing the agent-first CLI lane. (`agent run --input` is reserved/rejected until session +`{{ctx.*}}` prompt interpolation lands — a tracked engine follow-up in [deferred-tasks.md](../deferred-tasks.md).) +**Also landed — 2.R (the inbound MCP client):** agents now consume external MCP servers' tools across `chat`, +`run`, and one-shot `agent run`. The **`@relavium/mcp`** foundation (the SDK-fenced package, the dependency-free +JSON-Schema→Zod compiler, the fail-loud connect-all manager, the `mcp_{server}_{tool}` namespacing) is ✅ **Done +(PR #56, 2026-06-26)**, and the **host wiring** (chat/run/agent-run), the **network transports** (`http`/`sse`/ +`websocket`) behind the SSRF pre-connect floor + the per-server `allow_local_endpoint` opt-in, **named secrets** +via the isolated `mcp-secret:*` keychain namespace, the by-name `ref` registration form, and the **real-spawn +e2e** are ✅ **Done (PR #57, 2026-06-27)** — behind [ADR-0034](../../decisions/0034-mcp-client-sdk-dependency.md), +[ADR-0052](../../decisions/0052-inbound-mcp-client-package-lifecycle-registration.md), and +[ADR-0053](../../decisions/0053-mcp-network-transport-egress-security.md). It was off the M3 critical path and the +Phase-3 go/no-go (capability without gating). Residual MCP hardening — the connect-by-validated-IP dialer, +network header-auth, tool-list caching, mid-call abort propagation — is tracked in +[deferred-tasks.md](../deferred-tasks.md). The **stdio spawn is gated** as of 2.6.5 `CR-16` +([ADR-0084](../../decisions/0084-consent-before-a-local-mcp-spawn.md)): consent per machine and per resolved +declaration, on every path that can open an MCP-bearing artifact. The "import-trust" framing is retired — a +`git pull` changes a committed artifact with no import step — and only the `npx` version/integrity pinning half +remains deferred. +**Also landed — 2.J (the YAML-authoring lifecycle), the last in-phase lane:** `relavium create` (a +`@clack/prompts` wizard scaffolding an agent **or** a minimal single-agent workflow, validated against the +kind-appropriate `@relavium/shared` schema before write, dual-TTY-gated), `relavium import ` (schema- +validated copy-in with **project-global** id uniqueness), and `relavium export ` (a portable copy +**re-serialized from the validated AST** — canonical, comment-free, no provider key by construction), sharing one +`assertSlugAvailable` cross-catalog guard, ✅ **Done (PR #58, 2026-06-28)** — **no new ADR** (covered by +[ADR-0026](../../decisions/0026-session-export-to-workflow.md)/[ADR-0047](../../decisions/0047-cli-framework-commander-ink-clack.md)). + +**Phase 2 — CLI is feature-complete.** Every in-phase workstream (2.A–2.S) is merged and the published CLI is cut +as **v0.1.1**; M3 was reached at 2.K and the Phase-3 go/no-go held from 2.L. See the +[Phase 2 workstreams]() and the +[sequencing plan](#sequencing--parallelization). The full status-aware history is the +[Remaining build order](#remaining-build-order) section (its queue is now empty). + +**Phase 2.5 — CLI Consolidation & Conversational Home has started.** **2.5.A** (the spine's secure base — a +shared `assembleToolEnv({ profile, fsScopeTier, workspaceDir })` factory wired into **both** the chat and +workflow-run paths, the host-side `fs` (`realpath`+`commonpath` jail, read-only chat fail-close, single-fd +`O_NOFOLLOW`/`O_NONBLOCK` reads) and `process` arms, the advertise-filter, **EA1** `tool_unavailable`, and **EA2** +real failed-turn usage) is ✅ **Done (PR #60, 2026-06-28)**, behind +[ADR-0055](../../decisions/0055-cli-host-capability-seam-tool-environment-factory.md) — **reaching milestone +M2.5-1 (secure base)**. The `egress`/`os` arms, the `project`-tier `extraRoots` allowlist, and a write-capable +chat are deferred to **2.5.E**/[ADR-0057](../../decisions/0057-cli-chat-modes-and-per-tool-approval.md) (tracked in +[deferred-tasks.md](../deferred-tasks.md)). **2.5.B** (the bare-invocation Home) is ✅ **Done +(PR #61, 2026-06-29)**, behind [ADR-0054](../../decisions/0054-cli-bare-invocation-interactive-home.md) (Accepted): the +TTY-gated bare `relavium` opens a read-only management strip (recent sessions/runs/agents + an "Attention +required" section of pending human gates / failed runs) over a bounded, indexed `history.db` read seam, sitting +above a live prompt that graduates into an in-process chat; rendered as a single ink tree (one `useInput` owner) +with one SIGINT/SIGTERM lifecycle (clean Home exit `0`; an external signal → the conventional `128+signo`, 130/143; +the in-Home chat's exit-`4` consumed, never leaked) and bracketed paste (DECSET 2004), all while every +non-interactive path keeps the byte-for-byte help + exit-`0` meta-op ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)). +Canonically homed in [home.md](../../reference/cli/home.md). **2.5.C** (the in-app command system) is ✅ **Done +(PR #62, 2026-06-30)**, behind [ADR-0056](../../decisions/0056-cli-in-app-slash-command-system-and-manifest.md) +(Accepted): a curated **two-registry** model (the shell `COMMAND_MANIFEST` driving `commander` + `--help --json` ++ the `executeCommand` dispatch, vs the in-REPL `REPL_COMMANDS` driving a filterable `/` palette + slash commands +in **both** the chat and the bare Home — no command in both); `/help`, the `notice` output channel, `/workflows`, +`/cost`, and `/doctor` (fast tier: keychain/config/wired-tools; `--deep`: a **redacted** provider-key probe + a +**read-only** MCP-status report — a security-review decision: it reports the live session's already-connected +servers, never a fresh connect/spawn); plus the `name + args` slash dispatch and a context-aware footer hint-bar +surfacing `/ for commands`. Canonically homed in [commands.md](../../reference/cli/commands.md) + +[chat-session.md](../../reference/cli/chat-session.md). **2.5.E** (chat modes + per-tool approval + mid-turn abort) +is ✅ **Done (PR #63, 2026-07-03)**, behind +[ADR-0057](../../decisions/0057-cli-chat-modes-and-per-tool-approval.md) (**Accepted** after the mandatory +security review): the reseat-less mode system (ask / plan / accept-edits / auto on `Shift+Tab` + `/mode`), the +fail-closed per-tool `confirmAction` floor (`[y]/[a]/[n]` + a session once/always cache), the `Esc` mid-turn +abort (EA7), and the host arms closing the 2.5.A deferral — a write-capable `fs` tier + **protected paths** +(refused in every mode incl. `auto`), the SSRF-hardened `egress` arm (shared with media), and the `os` arm (now +a governed action class). Wired live into `relavium chat`, one-shot `agent run`, and the Home (each activates +the regime before its first turn). Every step passed the mandated opus + Sonnet 5 loop plus the dedicated +holistic security review (~50 findings fixed, 4 HIGH). A same-PR chat-UX follow-up also landed: a host tool +EXECUTION failure on the interactive surface (a file-not-found READ) is fed back to the model to recover +(`recoverToolFailures`, scoped to IDEMPOTENT tools via a stamped `ToolExecutionError.recoverable`; a governed / +side-effecting failure stays fail-fast) plus a static secret-free `tool_failed` hint. **With 2.5.E the CLI +Consolidation spine (2.5.A → C, E) is complete.** **2.5.D** (chat input ergonomics) is ✅ **Done (PR #64, merged +2026-07-03)** — the first experience-arm workstream: the pure-ergonomics half (`Ctrl+J` multiline, +`↑/↓` history + `Ctrl+R` reverse-search, readline motions, a shared cursor-bearing `EditorState`) plus the two +data-moving affordances behind [ADR-0061](../../decisions/0061-cli-input-layer-file-injection-and-shell-escape.md) +(**Accepted** after a two-round maintainer security review): **`@`-mention** (dir-navigable file completion that +reads through the SAME `FsCapability` `read_file` uses — jail + the expanded sensitive-read confidentiality floor ++ listing-gate + binary/size guards — and injects as UNTRUSTED, nonce-fenced, byte+line-bounded context) and +**`!`-shell** (the additive `AgentSession.runUserCommand` — EA8 — routing `!command` through the one `run_command` +boundary: `enforcePolicy([chat].allowed_commands)` BEFORE the mode-aware `confirmAction` → `spawn`/`shell:false`; +**empty-default allowlist ⇒ `!` inert**, secure-by-default, with an actionable deny hint). Each of the 5 steps +passed the mandated opus + sonnet adversarial-review loop, and the ADR-0061 mandatory security pass ran inside the +step-4/5 loops (14 findings fixed across the two `@`-mention rounds; on the `!`-shell the opus + security pass +confirmed 0 defects after adversarial verification, and the sonnet second pass caught a HIGH — a `!`-command in +flight left no host-visible busy signal, so a message typed mid-command could crash the session — now fixed with a +`shellBusy` input gate, plus a LOW type-hygiene fix). A post-implementation comprehensive review then refined the +`@`/`!` presentation to a **pending-attachment (chip) model** (an inline `@path` marker + a read-only `!`-output +preview, expanded into the SAME UNTRUSTED nonce-fenced frame only at submit — byte-identical model context, a +clean prompt), and two follow-up review passes hardened the `[chat]` allowlist resolution (exact + glob arrays are +now a **coupled unit**) and fixed a Backspace regression (ink reports the Unix physical Backspace as `key.delete`); +all recorded in the ADR-0061 "Refined at implementation" append. **2.5.F** (the ADR-0062 context commands) is ✅ +**Done (PR #65, merged 2026-07-05)** behind [ADR-0062](../../decisions/0062-context-compaction-and-cli-history-commands.md): +`/clear` (a host-level fresh-session swap across `chat`/`chat-resume`/Home, TTY-interactive only), the +`session:compacting` "Summarizing…" moment event (amending ADR-0036), and the footer context-fullness indicator — +completing compaction alongside the earlier `/compact` + `/trim` + automatic compaction. **2.5.G** (onboarding +wizard + Home `/models` + the live model catalog) is ✅ **Done (PR #66, 2026-07-07)** — its scope **expanded to +Option A**: a **live** model catalog (per-key provider discovery + a DB cache + refresh + a static/live merge) +and a complete model-pricing story (user-supplied pricing that governs cost), behind three ADRs +([ADR-0063](../../decisions/0063-cli-config-write-contract.md) config-write · +[ADR-0064](../../decisions/0064-live-model-catalog.md) live catalog · +[ADR-0065](../../decisions/0065-provider-economics-and-extensibility.md) provider economics), across 12 reviewed +steps (all landed); with it **milestone M2.5-2** is reached. The additive lane **2.5.H** (reasoning render + live-turn feedback + an actionable error taxonomy) is ✅ +**Done (PR #67, 2026-07-07)** — behind **EA6** (a new dual-envelope `agent:reasoning` stream event that *amends* +[ADR-0036](../../decisions/0036-run-loop-substrate-event-bus-and-execution-host.md); no new top-level ADR): a +host-emit of the reasoning the `@relavium/llm` seam already carries (ADR-0030), a collapsible "thinking" panel +(`/thinking` / `Ctrl+T`), the `Thinking…/Working… {elapsed}s · Esc to stop` timer, a visible `…` elision marker +(fixing the silent live-buffer head-drop), per-attempt `via {model}` failover attribution, and a static, +secret-free per-`ErrorCode` recovery hint (session-survives; a context-overflow message heuristic → `/compact`· +`/trim`). Four steps, each opus + Sonnet reviewed (3 HIGH fixed: a run-path silent-drop, a frozen Home timer, a +scrollback elision loss; + a one-shot `agent run` hint-leak). **With 2.5.E this reaches milestone M2.5-3.** The +consolidation lanes **2.5.I** (regression harness + DB concurrency hardening — `loadFull` read-txn snapshot, +`BEGIN IMMEDIATE` writes with a deterministic `SQLITE_BUSY` retry, the concurrent chat+run + cassette-chain + +perf-budget e2es, an advisory Windows CI lane) and **2.5.J** (docs-debt: the accurate unencrypted-history +posture per ADR-0050, and `NO_COLOR`/`FORCE_COLOR`/`--color` resolution) are ✅ **Done (2.5-close-out, PR #69, +2026-07-08)** — **reaching milestone M2.5-4, so Phase 2.5 is complete, merged to `main`** — landed alongside the doable-now +Batch A–E backlog (test-hardening; 2.5.H TUI polish; `AgentParseError` line/col; the ADR-0057 approval/security +batch — `[c]` reject-with-reason, non-TTY policy, SCOPE-denial recovery, Ctrl+T-in-approval, the Trojan-Source +bidi floor, behind an append-only ADR-0057 amendment; the profile-aware advertise-filter + the in-house +`.gitignore` matcher), each implement → Opus → Sonnet with a security-review pass on the approval batch. Two +`gate.ts`-resume items (`relavium budget resume` + secret re-provide) are deferred to a focused follow-up, and +the session `{{ctx.*}}` interpolation stays with the Proposed ADR-0060 (Phase-2.6); all tracked in +[deferred-tasks.md](../deferred-tasks.md). See the +[Phase 2.5 workstreams](phase-2.5-cli-consolidation.md). A **post-2.5.G model-UX follow-up** (from six +maintainer questions on model/provider/wizard behavior) then landed as a sequenced plan: `/models` key-awareness, +the onboarding-wizard live key-validation + retry UX, wizard-dynamic provider docs, **mid-session model switching** +(the `/models` reseat across `relavium chat` + the in-Home chat — [ADR-0059](../../decisions/0059-cli-mid-session-model-reseat.md), +now Accepted), and **normalized reasoning-effort control** ([ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md), +now Accepted, **merged 2026-07-07**): a provider-agnostic effort tier (`off`/`low`/`medium`/`high`/`max`) authored in +agent YAML or the `[chat].reasoning_effort` config default, each adapter mapping it to its provider's **native** tier — +**all four providers**, DeepSeek-v4's `thinking` param included (doc-verified against api-docs.deepseek.com) — gated +per-model by a host-injected capability resolver (plus a conservative id heuristic for a live-discovered model). It is +changed **interactively**: `/effort` opens a keyboard-owning tier-selector **overlay** in `relavium chat` + the in-Home +chat (§6), and the `/models` picker's **effort sub-step** either applies a **per-turn session override** in a live chat +(no reseat, §5) or — in the **bare Home** — **writes** the model + effort as the next session's config defaults (a new +global `[preferences].reasoning_effort` key), the active tier shown in the footer. Two multi-agent review passes +hardened it: the first caught the picker (wrongly) routing effort through a full model reseat — the P0 fix rebuilt it +as the ADR-mandated session-level setter; the second caught the bare-Home effort default not being re-read for a +same-process next chat — fixed to mirror the model's fresh per-chat re-read. + +Carry-over hardening is tracked in [deferred-tasks.md](../deferred-tasks.md) — Phase 2 picks +items up as it first touches each file. Notable inheritances: 1.AH's host-wiring half +(distributed across Phases 2–6), the media-egress host-side SSRF mechanism, and the +keychain no-raw-key IPC test. diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 91f78042..abc4dacc 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -1,6 +1,6 @@ # Phase 2.5.5 — Hardening and Remediation -> Status: 🚧 **Open — Wave 2 Steps 1–8 and review corrections merged to main as PR #94 (2026-10-15); later waves remain open.** Live progress and execution order are canonical in +> Status: 🚧 **Open — Wave 2 merged as PR #94 (2026-10-15); Wave 3 Steps 1–8 merged as PR #95 (2026-10-18); Wave 3 is paused after Step 9b.2 since 2026-10-26 behind the CLI MVP ([priority reset](#mvp-priority-reset--2026-11-04)); Steps 9c–19 and later waves remain open.** Live progress and execution order are canonical in > [../current.md](../current.md#execution-order--phase-255--phase-26-temporary); this banner does not > restate them. A full-project multi-agent code review completed > 2026-07-19 at HEAD `f88b0e8` — 377 findings, 373 surviving two-lens adversarial @@ -139,6 +139,11 @@ reached. the `coverage` job (`pnpm coverage:enforced`); no item in this phase widens the `LLMProvider` seam, adds a platform import to `packages/core`, or changes the `--json`/CI machine-output contract ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)). + *Reconciled 2026-10-19 with [exit criterion 4](#exit-criteria):* machine-output compatibility + changes are limited to the explicitly approved [ADR-0110](../../decisions/0110-cli-diagnostics-and-discovery-have-explicit-compatibility.md) + scope (diagnostics/exits, JSON discovery, aliases and consent); run-event stdout and actual + workflow-outcome contracts stay unchanged. Wave 3 Step 19 promotes `packages/core` into the + enforced coverage floor alongside `llm` and `mcp`. ## Scope @@ -194,26 +199,26 @@ reached. - ✅ **DONE (Wave 1, PR #81, 2026-07-30 — #91 · `rawPreviewFor` keeps the classification copy, `previewFor` scrubs the display copy).** **Redact tool-approval previews like `sanitizeInput()` already does.** `previewFor()` (`packages/core/src/tools/registry.ts:436`) builds the persisted, `--json`-visible `ToolActionPreview` by echoing `target.command`/`target.path` verbatim, never calling the `redactSecretShapedValue` scrub that its sibling `sanitizeInput()` (same file, line 563) applies to `toolInput` moments later in the same dispatch. A live credential in a `run_command` arg or `write_file` path is written unredacted into the durable, persisted `agent:approval_requested` event. Call the same redaction helper in `previewFor()` before the preview is built. This is the review's canonical propagation-gap example and should land first. *(S · `packages/core/src/tools/registry.ts` · #91)* - ✅ **DONE (Wave 1, PR #81, 2026-07-30 — G38 · `reservedAdmissions` ledger in `budget-governor.ts`).** **Add a reservation/admission step to `BudgetGovernor`'s pre-egress check.** `checkPreEgress` (`packages/core/src/engine/budget-governor.ts`) is a pure read against a snapshot that is only updated post-hoc after `cost:updated` lands, with no reservation/admission-control step — two or more parallel agent-node branches under `max_parallel` can each pass `checkPreEgress` against the same stale snapshot, letting the sum exceed `max_cost_microcents` even with `on_exceed:'fail'`. Add an in-flight reserved-cost ledger that is checked and updated atomically at dispatch time, and add a test combining budget + `max_parallel` (none exists today). *(L · `packages/core/src/engine/budget-governor.ts` · G38)* - ✅ **DONE (Phase 2.6.5 `CR-22`, 2026-08-27)** — closed early, out of wave order, because `W2`'s liveness item owns the same function. **Compute the remaining timeout budget on every resume instead of re-arming a fresh timer.** `#armRunTimeout` (`packages/core/src/engine/engine.ts`) re-arms a full-duration timer on every checkpoint/gate/media-job resume instead of computing remaining budget from `#elapsedMs()`, unlike the checkpoint's own `startEpochMs` and the media-job absolute-deadline pattern it should mirror — a workflow with `timeout_ms` can run far longer than authored if it pauses and resumes repeatedly. Fix `#armRunTimeout` to compute the remaining budget from elapsed wall-clock time on every re-arm. *(M · `packages/core/src/engine/engine.ts` · #75)* -- **Thread the missing `AbortSignal` into `resolvePrompt` and recheck cancellation in `human-gate.ts`.** `resolvePrompt` (`packages/core/src/engine/agent-runner.ts:868`) omits the `AbortSignal` that every other call site passes into `resolveTemplate`, so cancellation during prompt-template resolution is not observed. Separately, `human-gate.ts` (`packages/core/src/engine/node-handlers/human-gate.ts`) is the one node handler doing real async work yet has no post-resolution `ctx.signal.aborted` recheck before returning `'paused'` — a cancelled run can emit `human_gate:paused` right before `run:cancelled`. Add the signal to `resolvePrompt` and the recheck to `human-gate.ts` in one pass. *(M · `packages/core/src/engine/agent-runner.ts`, `packages/core/src/engine/node-handlers/human-gate.ts` · #76, G9)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)).** **Thread the missing `AbortSignal` into `resolvePrompt` and recheck cancellation in `human-gate.ts`.** `resolvePrompt` (`packages/core/src/engine/agent-runner.ts:868`) omits the `AbortSignal` that every other call site passes into `resolveTemplate`, so cancellation during prompt-template resolution is not observed. Separately, `human-gate.ts` (`packages/core/src/engine/node-handlers/human-gate.ts`) is the one node handler doing real async work yet has no post-resolution `ctx.signal.aborted` recheck before returning `'paused'` — a cancelled run can emit `human_gate:paused` right before `run:cancelled`. Add the signal to `resolvePrompt` and the recheck to `human-gate.ts` in one pass. *(M · `packages/core/src/engine/agent-runner.ts`, `packages/core/src/engine/node-handlers/human-gate.ts` · #76, G9)* - ✅ **DONE ([Wave 3 Step 2](#wave-3-step-2-acceptance--2026-10-16)).** **Extract a shared `messageFor` diagnostic helper for `parser.ts` and `agent-parser.ts`.** `parser.ts`'s `messageFor` has no case for Zod's `invalid_literal` code, so a `schema_version` mismatch reports only "invalid value" with no expected-value indication. `agent-parser.ts`'s `AgentParseError` carries only field paths, never reason text, so distinct error kinds report identically on the `.agent.yaml` authoring surface — a gap distinct from 2.6.B's visibility-wiring task. Extract `messageFor` into a schema-agnostic helper consumed by both `parser.ts` and `agent-parser.ts`. *(M · `packages/core/src/parser.ts`, `packages/core/src/agent-parser.ts` · #85, #86)* -- **Reconcile `system_prompt_append`'s documented interpolation with its verbatim-append implementation.** `workflow-yaml-spec.md` documents `system_prompt_append` as an interpolation target, but `agent-runner.ts` appends it verbatim by design (ADR-0038 anti-injection), while `packages/core/src/dag.ts` still wires it into real scheduling/taint-scan sites — an author using `{{ }}` there gets literal unresolved text sent to the model with zero error. Fix by either correcting the spec or adding a parse-time rejection of `{{ }}` syntax in that field. *(M · `packages/core/src/engine/agent-runner.ts`, `docs/reference/*/workflow-yaml-spec.md` · #87)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)) — documented as a literal append; Proposed ADR-0060 stays inactive.** **Reconcile `system_prompt_append`'s documented interpolation with its verbatim-append implementation.** `workflow-yaml-spec.md` documents `system_prompt_append` as an interpolation target, but `agent-runner.ts` appends it verbatim by design (ADR-0038 anti-injection), while `packages/core/src/dag.ts` still wires it into real scheduling/taint-scan sites — an author using `{{ }}` there gets literal unresolved text sent to the model with zero error. Fix by either correcting the spec or adding a parse-time rejection of `{{ }}` syntax in that field. *(M · `packages/core/src/engine/agent-runner.ts`, `docs/reference/*/workflow-yaml-spec.md` · #87)* - ✅ **DONE (Wave 1, PR #81, 2026-07-30 — G39/G47/G49 · `warningArmed` re-arm + projected `thresholdPct`).** **Re-arm `budget:warning` on continued overage and compute `thresholdPct` from the projected post-call total.** `#warningEmitted` (`packages/core/src/engine/budget-governor.ts:112`) is a one-shot lifetime boolean, so a long-lived `AgentSession`/chat process gets exactly one `budget:warning` ever and then silently proceeds no matter how far over cap spend climbs. Separately, `thresholdPct` is computed from pre-call cumulative spend rather than the projected post-call amount, so a single large call that jumps past the cap reports a near-0% warning. Fix both in the same pass: re-arm the warning on continued overage, and derive `thresholdPct` from the projected post-call total. *(M · `packages/core/src/engine/budget-governor.ts` · G39, G47, G49)* -- **Give `assertNoTaintedArgs()` an actual producer for `ctx.secretArgKeys`.** `ctx.secretArgKeys` — the runtime ADR-0029(c) taint-check input that `assertNoTaintedArgs()` (`packages/core/src/tools/registry.ts`, `packages/core/src/tools/types.ts`) correctly enforces when present — has no producer anywhere in the engine, so the check is inert everywhere, including on the `AgentSession`/chat path with no static-analysis backstop. Either wire a producer that actually populates `ctx.secretArgKeys`, or explicitly record this as a tracked, known gap in the ADR. *(M · `packages/core/src/tools/registry.ts`, `packages/core/src/engine/agent-session.ts` · #92)* -- **Cap `write_file.content`, `http_request.body`, `run_command.args`, and `git_commit.message` at a schema-level max length.** These four fields carry no length ceiling in `builtins.ts`'s Zod schemas despite heavy investment in bounding tool *output* — an oversized model-driven payload reaches dispatch and an unbounded regex redaction scan with no schema-level circuit breaker. Add a generous max-length ceiling on each field that won't break legitimate large writes. *(M · `packages/core/src/tools/builtins.ts` · #95)* -- **Rewrite `llmVisibleParams` descriptions across the built-in tool set.** No built-in tool's `llmVisibleParams` schema has per-property descriptions (byte-vs-line ranges, glob dialect, argv-vs-shell-string semantics are left to the model's guess), and several descriptions leak engineer-facing implementation vocabulary (`shell:false`) instead of actionable denial-recovery guidance. One clustered rewrite pass over `builtins.ts`'s tool descriptions, distinct from 2.6.M's new-tool scope. *(M · `packages/core/src/tools/builtins.ts` · #96, #98)* -- **Remove cycle-guard `WeakSet` entries on recursion unwind.** `redactInlineMediaForText`/`redactSecretShapedWalk` (`packages/core/src/tools/bounding.ts`) add to the shared cycle-guard `WeakSet` but never remove on the way back out of recursion, so any object referenced twice non-cyclically renders as a bogus `[cyclic]` in observability output. Remove the object from the `WeakSet` when recursion unwinds and add a regression test. *(S · `packages/core/src/tools/bounding.ts` · #94)* -- **Decide the real backpressure granularity for `RunHandle` and `SessionHandle` in one pass.** `RunHandle.whenConsumersReady()` (`packages/core/src/engine/run-handle.ts`) is only awaited once per node dispatch, never within a single node's own streaming duration, so a long token/reasoning stream against a stalled consumer buffers unboundedly. `SessionHandle.whenConsumersReady`/`whenDrained` (`packages/core/src/engine/session-handle.ts`) is implemented identically but has zero callers anywhere. Decide and implement the actual granularity needed for both handles together. *(M · `packages/core/src/engine/run-handle.ts`, `packages/core/src/engine/session-handle.ts` · #77, G48)* -- **Unescape the quoted filter-argument text `parseArg` returns.** `parseArg` (`packages/core/src/interpolation/references.ts`) returns the raw backslash-escaped text of a quoted filter argument instead of unescaping it, contradicted by its own test's comment. Fix `parseArg` to unescape the quoted argument text per its documented contract, and correct the misleading test comment. *(S · `packages/core/src/interpolation/references.ts` · #82)* +- ✅ **DISPOSED ([Wave 3 Step 3](#wave-3-step-3-acceptance--2026-10-16)) — ADR-0113 records the unsupported tool mapping honestly instead of fabricating a producer; the reproduced context leak is fixed.** **Give `assertNoTaintedArgs()` an actual producer for `ctx.secretArgKeys`.** `ctx.secretArgKeys` — the runtime ADR-0029(c) taint-check input that `assertNoTaintedArgs()` (`packages/core/src/tools/registry.ts`, `packages/core/src/tools/types.ts`) correctly enforces when present — has no producer anywhere in the engine, so the check is inert everywhere, including on the `AgentSession`/chat path with no static-analysis backstop. Either wire a producer that actually populates `ctx.secretArgKeys`, or explicitly record this as a tracked, known gap in the ADR. *(M · `packages/core/src/tools/registry.ts`, `packages/core/src/engine/agent-session.ts` · #92)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)).** **Cap `write_file.content`, `http_request.body`, `run_command.args`, and `git_commit.message` at a schema-level max length.** These four fields carry no length ceiling in `builtins.ts`'s Zod schemas despite heavy investment in bounding tool *output* — an oversized model-driven payload reaches dispatch and an unbounded regex redaction scan with no schema-level circuit breaker. Add a generous max-length ceiling on each field that won't break legitimate large writes. *(M · `packages/core/src/tools/builtins.ts` · #95)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)).** **Rewrite `llmVisibleParams` descriptions across the built-in tool set.** No built-in tool's `llmVisibleParams` schema has per-property descriptions (byte-vs-line ranges, glob dialect, argv-vs-shell-string semantics are left to the model's guess), and several descriptions leak engineer-facing implementation vocabulary (`shell:false`) instead of actionable denial-recovery guidance. One clustered rewrite pass over `builtins.ts`'s tool descriptions, distinct from 2.6.M's new-tool scope. *(M · `packages/core/src/tools/builtins.ts` · #96, #98)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)).** **Remove cycle-guard `WeakSet` entries on recursion unwind.** `redactInlineMediaForText`/`redactSecretShapedWalk` (`packages/core/src/tools/bounding.ts`) add to the shared cycle-guard `WeakSet` but never remove on the way back out of recursion, so any object referenced twice non-cyclically renders as a bogus `[cyclic]` in observability output. Remove the object from the `WeakSet` when recursion unwinds and add a regression test. *(S · `packages/core/src/tools/bounding.ts` · #94)* +- ✅ **CERTIFIED ([Wave 3 preflight](#preflight-corrections-and-already-satisfied-work)) — closed by Phase 2.6.5 CR-30 session backpressure and the W8/ADR-0087 contract; preserved, not reimplemented.** **Decide the real backpressure granularity for `RunHandle` and `SessionHandle` in one pass.** `RunHandle.whenConsumersReady()` (`packages/core/src/engine/run-handle.ts`) is only awaited once per node dispatch, never within a single node's own streaming duration, so a long token/reasoning stream against a stalled consumer buffers unboundedly. `SessionHandle.whenConsumersReady`/`whenDrained` (`packages/core/src/engine/session-handle.ts`) is implemented identically but has zero callers anywhere. Decide and implement the actual granularity needed for both handles together. *(M · `packages/core/src/engine/run-handle.ts`, `packages/core/src/engine/session-handle.ts` · #77, G48)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)).** **Unescape the quoted filter-argument text `parseArg` returns.** `parseArg` (`packages/core/src/interpolation/references.ts`) returns the raw backslash-escaped text of a quoted filter argument instead of unescaping it, contradicted by its own test's comment. Fix `parseArg` to unescape the quoted argument text per its documented contract, and correct the misleading test comment. *(S · `packages/core/src/interpolation/references.ts` · #82)* - ✅ **DONE ([Wave 3 Step 2](#wave-3-step-2-acceptance--2026-10-16)).** **Apply consistent duplicate/cross-field admission; certify existing input and session guards.** Shared leaf/full admission now enforces the approved duplicate, human-gate timeout and logical Run status matrices. Existing input defaults/enums and session compaction-role rules are certified. Historical session/workflow readers retain their former contracts; no rows are migrated or silently repaired. *(M · `packages/shared/src/{node,agent,workflow,session,run,common}.ts` · #89, #90, G18, #212, G16, G17, #210, #211)* -- **Extend `analyzeSecretTaint`'s leak-reporting loop to cover context-value sites.** `analyzeSecretTaint` (`packages/core/src/interpolation/analyze.ts`) deliberately skips context-value sites in its leak-reporting loop, so a direct `{{secrets.*}}` reference never read elsewhere is not reported at parse time — the workflow only fails later, differently-shaped, at run start. Not an actual secret leak, but a real completeness gap in the "reject before a run starts, uniformly" contract. *(S · `packages/core/src/interpolation/analyze.ts` · G22)* +- ✅ **CERTIFIED ([Wave 3 preflight](#preflight-corrections-and-already-satisfied-work)) — Wave 2's context-namespace refusal closes it.** **Extend `analyzeSecretTaint`'s leak-reporting loop to cover context-value sites.** `analyzeSecretTaint` (`packages/core/src/interpolation/analyze.ts`) deliberately skips context-value sites in its leak-reporting loop, so a direct `{{secrets.*}}` reference never read elsewhere is not reported at parse time — the workflow only fails later, differently-shaped, at run start. Not an actual secret leak, but a real completeness gap in the "reject before a run starts, uniformly" contract. *(S · `packages/core/src/interpolation/analyze.ts` · G22)* - ✅ **DONE ([Wave 3 Step 4](#wave-3-step-4-acceptance--2026-10-16)).** **Retain and validate the public event envelope without stripping full payloads.** `BaseEventSchema` shares the full run family's parsed exactly-one-correlation check; `BaseEvent` and `HumanGateEvent` remain exported. Complete payload, strict accounting, lenient opposite-key and stored forward-reader contracts are preserved. *(S · `packages/shared/src/run-event.ts` · #213)* - ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)).** **Clarify descriptive dispatch scope and retain immutable host filesystem authority.** Accepted [ADR-0105](../../decisions/0105-filesystem-roots-are-invocation-grants-not-project-authority.md) decides the host boundary; `ctx.fsScope` describes dispatch context and cannot widen the host's grant. No per-turn `FsCapability` escalation is introduced. *(S · `packages/core/src/tools/{types,registry,builtins}.ts` · #93)* -- **Correct three stale engine doc comments to match current code.** `checkpoint.ts`'s module doc (`packages/core/src/engine/checkpoint.ts`) claims a "runId+nodeId+retryCount idempotency key" bounds resumed side effects; no such key exists. `builtins.ts`'s top comment still says egress is "not wired yet"; it shipped. `workflow-yaml-spec.md` documents `workflow.version` as Semver-format-enforced; `WorkflowSpecSchema` enforces no format. One sweep correcting each. *(S · `packages/core/src/engine/checkpoint.ts`, `packages/core/src/tools/builtins.ts`, `docs/reference/*/workflow-yaml-spec.md` · #78, #97, #214)* -- **Remove the dead `RunEventBusOptions.validate` escape hatch and replace `kahnOrder`'s full re-sort with a priority structure.** `RunEventBusOptions.validate` (`packages/core/src/engine/event-bus.ts`) is documented as a hot-path escape hatch but no caller ever sets it — dead config, negligible cost today. `kahnOrder` (`packages/core/src/dag.ts`) re-sorts the entire ready-list on every pop instead of using a priority queue, also negligible at today's node counts. Both essentially-free cleanups; batch into one pass and revisit priority-queueing only if 2.6.P's subworkflows push node counts up. *(S · `packages/core/src/engine/event-bus.ts`, `packages/core/src/dag.ts` · #274, #275)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)) — #97/#214 corrected, #78 certified.** **Correct three stale engine doc comments to match current code.** `checkpoint.ts`'s module doc (`packages/core/src/engine/checkpoint.ts`) claims a "runId+nodeId+retryCount idempotency key" bounds resumed side effects; no such key exists. `builtins.ts`'s top comment still says egress is "not wired yet"; it shipped. `workflow-yaml-spec.md` documents `workflow.version` as Semver-format-enforced; `WorkflowSpecSchema` enforces no format. One sweep correcting each. *(S · `packages/core/src/engine/checkpoint.ts`, `packages/core/src/tools/builtins.ts`, `docs/reference/*/workflow-yaml-spec.md` · #78, #97, #214)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)) — #275's priority structure; #274's `validate` option is retained as an approved, documented compatibility surface.** **Remove the dead `RunEventBusOptions.validate` escape hatch and replace `kahnOrder`'s full re-sort with a priority structure.** `RunEventBusOptions.validate` (`packages/core/src/engine/event-bus.ts`) is documented as a hot-path escape hatch but no caller ever sets it — dead config, negligible cost today. `kahnOrder` (`packages/core/src/dag.ts`) re-sorts the entire ready-list on every pop instead of using a priority queue, also negligible at today's node counts. Both essentially-free cleanups; batch into one pass and revisit priority-queueing only if 2.6.P's subworkflows push node counts up. *(S · `packages/core/src/engine/event-bus.ts`, `packages/core/src/dag.ts` · #274, #275)* - ✅ **DONE ([Wave 3 Step 4](#wave-3-step-4-acceptance--2026-10-16)).** **Introduce minimal shared native error ancestry while retaining each family's contract.** The current export-scoped inventory replaces the historical 15/17 estimate: 46 roots across 37 files reach 83 module-exported production classes, with the additional unchanged private timeout descendant recorded separately. Constructors, names, discriminants, context/cause and structural/private-authority guards are preserved. *(M · `packages/shared/src/relavium-error.ts` and existing package families · #219)* -- **Chat-to-workflow export hardening: continuity caveat and non-ASCII slug fallback; certify existing tool-content exclusion.** W7/PR #90 and Accepted [ADR-0095](../../decisions/0095-what-an-agent-session-remembers-across-turns.md) structurally exclude model-issued tool arguments/results from durable session rows and exported scaffolds. The earlier raw-tool-result export claim and its proposed opt-in/redaction decision are superseded by that contract; do not add a raw-content flag. Wave 2 still adds a clear continuity/review notice and deterministic ASCII title normalization with a session-identity fallback. Retained user/assistant text, `@` attachments and `!` output can remain sensitive: this is not universal transcript masking. Certify producer→SQLite→export rejection fixtures with #115; the [Wave 2 decision package](#decisions-presented-together-before-implementation) records D14/D15. *(M · `packages/core/src/export/serializer.ts` · #80, #81, #83)* +- ✅ **CERTIFIED ([Wave 3 preflight](#preflight-corrections-and-already-satisfied-work)) — completed by W7/Wave 2.** **Chat-to-workflow export hardening: continuity caveat and non-ASCII slug fallback; certify existing tool-content exclusion.** W7/PR #90 and Accepted [ADR-0095](../../decisions/0095-what-an-agent-session-remembers-across-turns.md) structurally exclude model-issued tool arguments/results from durable session rows and exported scaffolds. The earlier raw-tool-result export claim and its proposed opt-in/redaction decision are superseded by that contract; do not add a raw-content flag. Wave 2 still adds a clear continuity/review notice and deterministic ASCII title normalization with a session-identity fallback. Retained user/assistant text, `@` attachments and `!` output can remain sensitive: this is not universal transcript masking. Certify producer→SQLite→export rejection fixtures with #115; the [Wave 2 decision package](#decisions-presented-together-before-implementation) records D14/D15. *(M · `packages/core/src/export/serializer.ts` · #80, #81, #83)* - ✅ **DONE ([Wave 3 Step 4](#wave-3-step-4-acceptance--2026-10-16)).** **Preserve typed budget-error discriminants with shared native ancestry.** `BudgetExceededError` and `BudgetPauseError` retain their `.code` fields and `instanceof` identity; the accepted D10 ruling and error-substrate increment resolve the former maintainer question. *(S · `packages/core/src/engine/budget-governor.ts` · G50)* -- **Collapse two always-co-occurring branches in `engine.ts`'s `#dispatch` retry-sleep handling.** Purely cosmetic dead-code/maintenance-trap: two branches that always co-occur, one unreachable dead weight. No behavioral bug, but real, trivially-scheduled cleanup. *(S · `packages/core/src/engine/engine.ts` · #79)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)).** **Collapse two always-co-occurring branches in `engine.ts`'s `#dispatch` retry-sleep handling.** Purely cosmetic dead-code/maintenance-trap: two branches that always co-occur, one unreachable dead weight. No behavioral bug, but real, trivially-scheduled cleanup. *(S · `packages/core/src/engine/engine.ts` · #79)* **Acceptance:** the CRITICAL `previewFor()`/approval-preview secret leak is closed and verified against a `run_command`/`write_file` fixture containing a fake credential; `BudgetGovernor` rejects a concurrent-branch overspend under `max_parallel` in a new test exercising the reservation ledger; `#armRunTimeout` computes remaining budget on every resume rather than re-arming full duration; cancel-signal propagation reaches `human-gate.ts` and `resolvePrompt`, and backpressure propagation reaches `SessionHandle`; `parser.ts` and `agent-parser.ts` share one `messageFor`-derived diagnostic helper; the approved duplicate/cross-field schema inventory is implemented or certified with its historical-reader limits in [Wave 3 Step 2](#wave-3-step-2-acceptance--2026-10-16); the cycle-guard `WeakSet` leak in `bounding.ts` is fixed with a regression test; no engine-core doc comment contradicts the code it describes; the budget-error `.code` ruling is resolved by D10 and [Wave 3 Step 4](#wave-3-step-4-acceptance--2026-10-16), while the export privacy/warning scope is resolved by the accepted D14/D15 Wave 2 contract. These are resolved decision gates, not current blockers; whole sub-stream A acceptance remains Wave 3 Step 19. @@ -309,20 +314,20 @@ This workstream is `apps/cli`'s outermost layer — the process/runtime host eve **Tasks:** -- **Close the five remaining gaps in the CLI's outermost crash/signal safety net.** `processIo()` (`apps/cli/src/process/io.ts`) has no EPIPE handling, so a `--json` consumer that closes its reader early dumps a raw stack trace instead of a structured envelope. `buildProgram()` in `apps/cli/src/run.ts` is the one synchronous step inside `run()` not wrapped in the standard error path, despite `run()`'s own JSDoc promising it never rejects. There is zero `process.on('unhandledRejection'|'uncaughtException', ...)` registration anywhere in production `apps/cli/src` — an async error escaping `run()`'s own promise chain falls straight to Node's default handler, and `process.on('exit')` never reaps detached `run_command` child process groups or closes DB/MCP connections on a crash. The last-resort crash message in `apps/cli/src/index.ts` unconditionally writes plain text to stderr with no `--json` check, the sole exception to the CLI's otherwise-universal structured-envelope convention. Fix all five in `index.ts`/`run.ts`/`process/io.ts` together: add EPIPE handling, wrap `buildProgram()`, register the missing global handlers with child-process reaping and DB/MCP close-on-exit, and make the last-resort message honor `--json`. *(L · apps/cli/src/index.ts, apps/cli/src/run.ts, apps/cli/src/process/io.ts · #40, #41, #42, #224, #269)* **Source correction — 2026-10-15:** the recoverable rejection net already exists and keeps alive; preserve it. Proposed ADR-0110 adds fatal/early-import/output handling through registered owners. Do not close DB/MCP from exit without actual owner ACK or claim universal descendant containment; see the revised preflight and canonical proposed teardown scope. +- ✅ **DONE ([Wave 3 Step 8](#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17)).** **Close the five remaining gaps in the CLI's outermost crash/signal safety net.** `processIo()` (`apps/cli/src/process/io.ts`) has no EPIPE handling, so a `--json` consumer that closes its reader early dumps a raw stack trace instead of a structured envelope. `buildProgram()` in `apps/cli/src/run.ts` is the one synchronous step inside `run()` not wrapped in the standard error path, despite `run()`'s own JSDoc promising it never rejects. There is zero `process.on('unhandledRejection'|'uncaughtException', ...)` registration anywhere in production `apps/cli/src` — an async error escaping `run()`'s own promise chain falls straight to Node's default handler, and `process.on('exit')` never reaps detached `run_command` child process groups or closes DB/MCP connections on a crash. The last-resort crash message in `apps/cli/src/index.ts` unconditionally writes plain text to stderr with no `--json` check, the sole exception to the CLI's otherwise-universal structured-envelope convention. Fix all five in `index.ts`/`run.ts`/`process/io.ts` together: add EPIPE handling, wrap `buildProgram()`, register the missing global handlers with child-process reaping and DB/MCP close-on-exit, and make the last-resort message honor `--json`. *(L · apps/cli/src/index.ts, apps/cli/src/run.ts, apps/cli/src/process/io.ts · #40, #41, #42, #224, #269)* **Source correction — 2026-10-15:** the recoverable rejection net already exists and keeps alive; preserve it. Proposed ADR-0110 adds fatal/early-import/output handling through registered owners. Do not close DB/MCP from exit without actual owner ACK or claim universal descendant containment; see the revised preflight and canonical proposed teardown scope. - ✅ **DONE — Wave 3 Step 1, 2026-10-16** ([acceptance](#wave-3-step-1-acceptance--2026-10-16)). **Carry `const exhaustive: never` from `engine.ts` to the three sibling switches it was never propagated to, two of which drop live data.** `apps/cli/src/chat/persister.ts`'s event switch — the sole writer of `agent_sessions`/`session_costs` — silently drops any unhandled event kind via a bare `default`, a real money/data risk. Four picker key-routing switches in `apps/cli/src/render/tui/chat-ink.tsx` and `apps/cli/src/render/tui/home-controller.ts` lack the same guard on the reseat surface that already caused a data-loss-class bug. `apps/cli/src/render/tui/run-view-model.ts`'s `reduceRunEvent` silently drops `agent:reasoning` with no state field to hold it, so `relavium run` never shows reasoning that `chat` already renders — the exact bug class the guard in `apps/cli/src/chat/session-host.ts`'s `engine.ts`-derived pattern was invented to prevent, recurring on a different driver. Apply `const exhaustive: never` uniformly across all six sites. *(M · apps/cli/src/chat/persister.ts · #300, #301, G3)* - **Centralize `openLocalDb` wrap-and-redact instead of leaving it well-wrapped at only 3 of 8+ call sites.** On the `run`/`chat`/`gate` paths the well-wrapped open happens AFTER an earlier unwrapped one, making the clean `CliError` message largely unreachable in practice. Separately, `packages/db/src/client.ts` interpolates a raw absolute path into its error with no redaction backstop, unlike `packages/llm`'s `scrubSecrets`. Move the wrap-and-redact logic inside `openLocalDb` itself so every caller gets it for free instead of re-implementing it. *(M · packages/db/src/client.ts · #278, #280)* - **Wire `agent-run.ts`'s missing SIGINT handler and make chat's existing one actually abort the session.** `apps/cli/src/commands/agent-run.ts` registers zero SIGINT handler, so Ctrl-C orphans spawned MCP child processes. `drivePlain`/`driveJson`'s SIGINT handlers (chat driver) only close readline and never call `session.abort()`, letting the billed generation run to completion. Register a handler in `agent-run.ts` mirroring `driveRun`'s teardown, and make the chat handlers call `abort()`. *(M · apps/cli/src/commands/agent-run.ts · #21, #23)* - **Wrap `gate.ts`'s two idempotent-resume success paths in the same `--json` branch every other command uses.** Both write plain human text to stdout unconditionally, breaking the NDJSON contract for scripted callers, and neither path is tested. No dependency on the 2.6.K resume-core extraction — fix in place. *(S · apps/cli/src/commands/gate.ts · #20)* - **Fix `/doctor`'s hardcoded probe profile, which misreports the exact thing it exists to diagnose.** `apps/cli/src/chat/doctor.ts` probes tool-host wiring with a hardcoded profile instead of the session's actual one, so it always misreports `egress`/`os` capability as unwired regardless of the real config. Probe with the session's actual profile. *(S · apps/cli/src/chat/doctor.ts · G43)* - **Implement `--help --json`'s promised `COMMAND_MANIFEST` serialization, or correct the docs that claim it exists.** Doc comments in `apps/cli/src/commands/manifest.ts` and user-facing docs both claim `--help --json` emits `COMMAND_MANIFEST` as JSON; no code path serializes it, confirmed by a passing test that asserts help stays human text. Either implement the missing serialization or amend the docs to match reality. *(M · apps/cli/src/commands/manifest.ts · #13)* -- **Defer the CLI's eager `driveHome` import and unconditional `seedCatalog()` call behind command-need, closing the ~400ms `--help`/`--version` tax.** `apps/cli/src/run.ts`'s static import of `driveHome` pulls in its full transitive graph — ink, react, the home controller, the DB layer, the native keychain binding — on every process start, even for `--help`/`--version`. `seedCatalog()` (`apps/cli/src/engine/catalog-refresh.ts`) does an unconditional sync read/parse/validate of the cached model catalog before checking whether pricing data is even needed. Measured ~390-410ms wall time for `--version` against ~30ms bare Node startup; a separate measurement centered on `dispatch.ts` confirmed the same root cause independently. Gate both behind a dynamic import keyed on actual command need; likely requires enabling `splitting: true` in `apps/cli/tsup.config.ts`, which currently emits a single bundle file. *(L · apps/cli/src/run.ts · #43, #44, #271)* +- ✅ **DONE ([Wave 3 Step 7](#wave-3-step-7b-and-whole-step-7-acceptance--2026-10-16)) — lazy packaged runtime under ADR-0111.** **Defer the CLI's eager `driveHome` import and unconditional `seedCatalog()` call behind command-need, closing the ~400ms `--help`/`--version` tax.** `apps/cli/src/run.ts`'s static import of `driveHome` pulls in its full transitive graph — ink, react, the home controller, the DB layer, the native keychain binding — on every process start, even for `--help`/`--version`. `seedCatalog()` (`apps/cli/src/engine/catalog-refresh.ts`) does an unconditional sync read/parse/validate of the cached model catalog before checking whether pricing data is even needed. Measured ~390-410ms wall time for `--version` against ~30ms bare Node startup; a separate measurement centered on `dispatch.ts` confirmed the same root cause independently. Gate both behind a dynamic import keyed on actual command need; likely requires enabling `splitting: true` in `apps/cli/tsup.config.ts`, which currently emits a single bundle file. *(L · apps/cli/src/run.ts · #43, #44, #271)* - ✅ **DONE — Wave 3 Step 1, 2026-10-16** ([acceptance](#wave-3-step-1-acceptance--2026-10-16)). **Extend `authoring.ts`'s cwd-relative `displayPath` error-path discipline to `resolve.ts`, `catalog.ts`, and `chat/export.ts`.** `apps/cli/src/authoring/authoring.ts`'s `readAuthoredFile` already renders filesystem errors against a cwd-relative `displayPath` instead of the raw absolute path; `apps/cli/src/workflows/resolve.ts` and `apps/cli/src/workflows/catalog.ts` never adopted it, and neither did the filesystem `CliError`s in `apps/cli/src/commands/chat-export.ts`. Apply `displayPath` at all three remaining sites so no command leaks the user's absolute home-directory path into an error message. *(M · apps/cli/src/workflows/resolve.ts · #4, G11)* - ✅ **DONE — Wave 3 Step 1, 2026-10-16** ([acceptance](#wave-3-step-1-acceptance--2026-10-16)). **Fix `resolve.ts`'s 1x UTF-8 byte multiplier, which undersizes the load cap for non-ASCII workflow files.** `apps/cli/src/workflows/resolve.ts` compares `stats.size` directly against `MAX_SOURCE_CHARS` (a 1x multiplier); `authoring.ts` and `catalog.ts` both correctly use `MAX_SOURCE_CHARS * 4` to account for UTF-8's up-to-4-bytes-per-char worst case. On the `relavium run` load path this means a dense multibyte YAML file well under the real char cap can be wrongly rejected as oversized. Apply the same `* 4` multiplier used by its two siblings. *(S · apps/cli/src/workflows/resolve.ts · #3)* - ✅ **DONE — Wave 3 Step 1, 2026-10-16** ([acceptance](#wave-3-step-1-acceptance--2026-10-16)). **Extend `chat.ts`'s `closeQuietly()` helper to the four sibling commands still using an unguarded `.close()`.** `apps/cli/src/commands/chat.ts` wraps every store-close in `closeQuietly()` so a close-time throw can never overwrite a successful exit code; `apps/cli/src/commands/run.ts`, `apps/cli/src/commands/gate.ts`, `apps/cli/src/commands/chat-export.ts`, and `apps/cli/src/commands/chat-list.ts` all still call `.close()`/`opened.close()` unguarded. Extract the helper to a shared location and import it into all four. *(S · apps/cli/src/commands/chat.ts · #22)* - **Close `agent run`'s two UX-parity gaps against its sibling drivers in one pass.** `apps/cli/src/commands/agent-run.ts` blocks silently on a TTY with no piped stdin and prints no banner, unlike `chat`/`create` which both print one. `makePlainPrinter` (shared with `apps/cli/src/commands/chat.ts`'s plain-printer path) has no `cost:updated` case or final cost line for `agent run`'s non-JSON output, unlike `run`'s TUI/plain renderer. Both are the same shape of gap — `agent run` is the newest, least-mature driver — so add the stdin-TTY banner `create` already has and the cost line `run` already prints in the same change. *(S · apps/cli/src/commands/agent-run.ts · #24, #26)* -- **Add a dedicated exit code for unclassified internal faults, or amend the docs that call exit code 1 "workflow failed."** Exit code 1 is documented as meaning a failed workflow but is silently reused for any unclassified internal fault from any command, so a script branching on exit code cannot distinguish "your workflow failed" from "the CLI itself broke." Add a dedicated code or correct the documented contract. *(S · apps/cli/src/index.ts · G35)* -- **Resolve `--quiet`/`--verbose`/`RELAVIUM_DEBUG` into one coherent diagnostic-flag system and point the CLI's generic error fallbacks at it.** `--quiet` has exactly one consumer, and that consumer hardcodes the value away — a complete no-op. `--verbose` only gates `err.stack` on a top-level crash and has zero effect during a live session. `RELAVIUM_DEBUG` is undocumented and is a second, inconsistently-named escape hatch alongside `--verbose`. Both of the CLI's generic "internal error" fallback messages never mention `--verbose` or `RELAVIUM_DEBUG` as a way to get more detail — the same incoherence, seen from the user's side of the same broken flag system. Wire `--quiet` to actually suppress output, extend `--verbose` to gate live-session diagnostics, fold `RELAVIUM_DEBUG` into `--verbose`'s precedence chain, and append the resulting hint to both generic fallback messages in the same change; the fallback-message fix cannot be written correctly until the flag relationship is settled. **Approved 2026-10-16 in ADR-0110:** quiet+verbose conflict, explicit flags over exact DEBUG=1, independent JSON representation and preserved requested data/safety categories. Implementation remains staged. *(M · apps/cli/src/index.ts · #45, #264, #265, #268)* +- ✅ **DONE ([Wave 3 Step 8](#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17)) — `cli_internal`/`output_unavailable` exit 9 under ADR-0110, with Step 6's contracts.** **Add a dedicated exit code for unclassified internal faults, or amend the docs that call exit code 1 "workflow failed."** Exit code 1 is documented as meaning a failed workflow but is silently reused for any unclassified internal fault from any command, so a script branching on exit code cannot distinguish "your workflow failed" from "the CLI itself broke." Add a dedicated code or correct the documented contract. *(S · apps/cli/src/index.ts · G35)* +- **Resolve `--quiet`/`--verbose`/`RELAVIUM_DEBUG` into one coherent diagnostic-flag system and point the CLI's generic error fallbacks at it.** `--quiet` has exactly one consumer, and that consumer hardcodes the value away — a complete no-op. `--verbose` only gates `err.stack` on a top-level crash and has zero effect during a live session. `RELAVIUM_DEBUG` is undocumented and is a second, inconsistently-named escape hatch alongside `--verbose`. Both of the CLI's generic "internal error" fallback messages never mention `--verbose` or `RELAVIUM_DEBUG` as a way to get more detail — the same incoherence, seen from the user's side of the same broken flag system. Wire `--quiet` to actually suppress output, extend `--verbose` to gate live-session diagnostics, fold `RELAVIUM_DEBUG` into `--verbose`'s precedence chain, and append the resulting hint to both generic fallback messages in the same change; the fallback-message fix cannot be written correctly until the flag relationship is settled. **Approved 2026-10-16 in ADR-0110:** quiet+verbose conflict, explicit flags over exact DEBUG=1, independent JSON representation and preserved requested data/safety categories. Implementation remains staged. *Note 2026-10-19:* the [ADR-0110 2026-10-18 clarification](../../decisions/0110-cli-diagnostics-and-discovery-have-explicit-compatibility.md#fixed-infrastructure-diagnostic-clarification--2026-10-18) supersedes this bullet's "append the resulting hint" instruction: infrastructure and unclassifiable faults keep one fixed diagnostic at every verbosity and their message does **not** suggest `--verbose` or `RELAVIUM_DEBUG` ([commands.md](../../reference/cli/commands.md#diagnostic-policy)). #45 closes on that disposition in Wave 3 Step 10a. *(M · apps/cli/src/index.ts · #45, #264, #265, #268)* - **Certify durable-only logs (G15; source correction — 2026-10-15).** The earlier token-firehose premise is refuted: shipping token/reasoning emission is live-only and logs reads durable rows. No coalescing decision or new persistence is required. Preserve actual replay with a real engine/store/log regression and JSON sequence-gap controls; the [reopened preflight](#maintainer-review-reopened-the-proposal--2026-10-15) records the evidence. *(M · apps/cli/src/commands/logs.ts · G15)* - **Attach `GLOBAL_OPTIONS_HELP` to every subcommand's `--help`, not just the root program's.** `apps/cli/src/program.ts` defines `GLOBAL_OPTIONS_HELP` (documenting the position-independent flags `commander` itself doesn't parse, like `--json`/`--cwd`/`--verbose`) and attaches it only to the root program; `commander` does not propagate it to subcommand help output, so `relavium run --help` never shows it. Attach the same help text to each subcommand, or add an explicit pointer to `relavium --help`. *(S · apps/cli/src/program.ts · #17)* - **Add a `--force` gate or interactive confirmation to `provider remove-key`, the manifest's sole `effect: 'destructive'` command.** `apps/cli/src/commands/provider.ts`'s `remove-key` action has no confirmation gate at all, unlike the create/import/export family, which all require explicit confirmation for destructive effects. Add the same pattern here. *(S · apps/cli/src/commands/provider.ts · #19)* @@ -380,10 +385,10 @@ Cross-cutting code-quality debt with no owning subsystem, concentrated in `packa - **Expand tabs before measuring transcript width.** `stripTerminalControls` (`apps/cli/src/render/sanitize.ts`) deliberately preserves `\t` in transcript body text, but `displayWidth`/`graphemeWidth` (`apps/cli/src/render/tui/viewport.ts`) delegate to `string-width`, which measures any control character — including tab — as 0-width, while a real terminal expands a literal tab to the next 8-column stop. This breaks the viewport module's own documented invariant ("1 DisplayLine == 1 terminal row"): scroll offsets and mouse row→line mapping drift whenever a tab reaches the transcript (shell/tool output, TSV data, a tab-indented code block a model emits). Expand tabs to spaces at the same display boundary `sanitizeInline` already uses for short fields, or give `wrapLogicalLine`/`displayWidth` explicit tab-stop-aware accounting, and add a `viewport.test.ts` case pinning the chosen behavior. *(M · apps/cli/src/render/tui/viewport.ts · #65)* - **Replace four hand-rolled money/token reimplementations with the canonical helpers they already sit next to.** `capUsd` (`apps/cli/src/chat/effort-notice.ts`) hand-rolls the same ÷1e8 microcent→USD conversion `format.ts`'s `formatCostUsd`/`formatCostShort` already canonicalize — against `format.ts`'s own explicit "never a second hand-rolled divisor" comment (#236). `TOKENS_PER_MTOK` is declared independently in both `packages/llm/src/budget-estimator.ts` and `packages/llm/src/cost-tracker.ts`, despite `budget-estimator.ts` already importing other symbols from `cost-tracker.js` (#237). `mention.ts` (`apps/cli/src/render/tui/mention.ts`) hardcodes its own `Math.ceil(bytes / 4)` for the `@`-mention token warning instead of importing `CHARS_PER_TOKEN` from `packages/llm/src/adapters/shared.ts` (#242). `budgetWarningText` (`apps/cli/src/commands/chat.ts`) interpolates raw microcents directly ("~80% of the 500000000µ¢ cap reached") instead of calling `formatCostUsd`/`formatCostShort` — the one cost-facing string in the CLI that doesn't (#307). One mechanical replace-with-import pass, no new logic, closes all four. *(S · apps/cli/src/chat/effort-notice.ts, packages/llm/src/budget-estimator.ts, apps/cli/src/render/tui/mention.ts, apps/cli/src/commands/chat.ts · #236, #237, #242, #307)* - **Split the `Capability` vocabulary collision and unify it with `Port`.** `packages/llm/src/capabilities.ts`'s bare `Capability = keyof CapabilityFlags` (a model feature flag) collides with `packages/core/src/tools/types.ts`'s `FsCapability`/`ProcessCapability`/`EgressCapability`/`OsCapability`/`McpCapability` (host-injected ToolHost-seam implementations) — the same root word for two unrelated architectural roles, hurting greppability on exactly the seam a security review audits (#217). The identical host-injected pattern is separately spelled `Port` elsewhere (`HomeModelsPort` in `apps/cli/src/render/tui/home-controller.ts`, `SuspendPort`, `MediaWritePort`, `MediaReferencePort`), so `apps/cli` imports both spellings for one concept in sibling files (#222). Rename `packages/llm`'s bare `Capability` alias to `ModelFeature`/`FeatureFlag`, leave `core`'s `*Capability` family (already matching ADR-0037/ADR-0041's ToolHost vocabulary) as sole owner of the word, and settle on `Port` as the one name for "host-injected dependency interface," recording the convention in `docs/standards/architectural-principles.md`. *(S · packages/llm/src/capabilities.ts, packages/core/src/tools/types.ts · #217, #222)* -- **Pick one spelling each for `ok`/`success` and `model`/`modelId`.** `packages/core` uses `{ ok: true | false }` at 49 call sites across the package (`agent-runner.ts`, `agent-session.ts`), but `tools/registry.ts` and `engine/agent-turn.ts` use `success: true | false` for the identical binary-outcome shape (#216). Within `packages/llm`'s catalog/pricing layer, lookups split roughly 50/50 between `modelId` (`pricing.ts`'s `catalogPricing`, `cost-tracker.ts`) and `model` (`pricing.ts`'s `contextWindowForModel`, `output-cap.ts`, `adapters/shared.ts`) for the same lookup-key role. Rename the two `success` sites to `ok` (or document the wire-schema exception for `ToolResultEventData.success`), and standardize on `modelId` for every catalog/pricing-lookup parameter, reserving `model` for signatures that literally forward `LlmRequest.model`. *(S · packages/core/src/tools/registry.ts, packages/core/src/engine/agent-turn.ts, packages/llm/src/pricing.ts · #216, #221)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)) — #221; #216's canonical event `success` field is retained per the preflight disposition.** **Pick one spelling each for `ok`/`success` and `model`/`modelId`.** `packages/core` uses `{ ok: true | false }` at 49 call sites across the package (`agent-runner.ts`, `agent-session.ts`), but `tools/registry.ts` and `engine/agent-turn.ts` use `success: true | false` for the identical binary-outcome shape (#216). Within `packages/llm`'s catalog/pricing layer, lookups split roughly 50/50 between `modelId` (`pricing.ts`'s `catalogPricing`, `cost-tracker.ts`) and `model` (`pricing.ts`'s `contextWindowForModel`, `output-cap.ts`, `adapters/shared.ts`) for the same lookup-key role. Rename the two `success` sites to `ok` (or document the wire-schema exception for `ToolResultEventData.success`), and standardize on `modelId` for every catalog/pricing-lookup parameter, reserving `model` for signatures that literally forward `LlmRequest.model`. *(S · packages/core/src/tools/registry.ts, packages/core/src/engine/agent-turn.ts, packages/llm/src/pricing.ts · #216, #221)* - **Rename `swapAgentModel` to match the dominant "reseat" vocabulary.** ADR-0059 ("CLI mid-session model reseat") and the surrounding codebase use `reseat*` identifiers pervasively (100+ occurrences in `apps/cli/src/commands/chat.ts` alone: `reseatChat`, `ReseatTarget`, `createReseatRebuild`, `finalizeReseatOutcome`). Yet the actual mutation function performing the model rebind — `swapAgentModel` (`apps/cli/src/chat/session-host.ts`), whose own docstring calls it a "model SWITCH" — and the dispatcher `resolveSwapRebuild` (`chat.ts`), which branches on `outcome.kind === 'reseat'`, both use a third word for the same operation, so a grep for "reseat" against ADR-0059's implementation misses it entirely. 2.6.C is closed, so this is a maintenance rename against shipped code, not new scope: rename `swapAgentModel` → `reseatAgentModel`, keeping "swap" only as the deliberately broader umbrella term where `resolveSwapRebuild` genuinely branches on two outcome kinds. *(S · apps/cli/src/chat/session-host.ts · #218)* - **Delete three knip-confirmed dead exports and their false justifying comments.** The `@relavium/llm/adapters` subpath export (`packages/llm/src/adapters/index.ts`) has zero real consumers — `packages/llm/src/providers.ts` imports the three adapter factories directly from their own files, bypassing the barrel entirely (#239). `isDispatchableId` (`apps/cli/src/commands/dispatch.ts`) is exported with a doc comment claiming "a surface calls this before dispatch," but no call site exists anywhere in the repo (#240). `FRAME_MS`'s re-export from `apps/cli/src/render/tui/ink-renderer.ts` carries a comment claiming "existing importers" use it, but every real consumer (`drive-home.tsx`, `chat-ink.tsx`, `synchronized-output.test.tsx`) imports it directly from `tui-constants.js` instead (#241). Delete each dead export/re-export and its stale comment, or, for the `/adapters` subpath, replace the comment with a note on which future consumer it is reserved for if kept deliberately. *(S · packages/llm/src/adapters/index.ts, apps/cli/src/commands/dispatch.ts, apps/cli/src/render/tui/ink-renderer.ts · #239, #240, #241)* -- **Pin every identifier-sorting `localeCompare` call to break the machine-locale dependency.** `models refresh --json`'s provider sort (`apps/cli/src/engine/model-refresh.ts`) calls `a.provider.localeCompare(b.provider)` with no locale argument, so the comparison resolves against the invoking machine's ambient `Intl` locale (e.g. `LANG`) rather than anything Relavium controls — breaking the ADR-0049 byte-identical `--json` promise, since e.g. Turkish collation reorders relative to `en_US` (#258). The same unpinned pattern recurs at 7+ more sites sorting closed-set ASCII identifiers: `apps/cli/src/engine/mcp-servers.ts` (env var names, tool allowlist), `apps/cli/src/workflows/catalog.ts` (workflow/agent slug order feeding both the Home browser and `relavium list`), `apps/cli/src/commands/list.ts`, `apps/cli/src/render/tui/mention.ts`, and `packages/core/src/tools/registry.ts`/`errors.ts` (`UnknownToolError`'s message, which can propagate into a `node:failed` RunEvent's `outputSummary` and thus into the NDJSON stream too) (#259). One shared `compareIdentifiers(a, b)` helper — a plain ASCII ordinal comparison, matching the one site that already gets this right (`packages/llm/src/model-catalog.ts`'s `localeCompare(y.displayName, 'en')`) — closes the whole sweep in one pass. *(M · apps/cli/src/engine/model-refresh.ts, apps/cli/src/engine/mcp-servers.ts, apps/cli/src/workflows/catalog.ts, packages/core/src/tools/registry.ts · #258, #259)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)).** **Pin every identifier-sorting `localeCompare` call to break the machine-locale dependency.** `models refresh --json`'s provider sort (`apps/cli/src/engine/model-refresh.ts`) calls `a.provider.localeCompare(b.provider)` with no locale argument, so the comparison resolves against the invoking machine's ambient `Intl` locale (e.g. `LANG`) rather than anything Relavium controls — breaking the ADR-0049 byte-identical `--json` promise, since e.g. Turkish collation reorders relative to `en_US` (#258). The same unpinned pattern recurs at 7+ more sites sorting closed-set ASCII identifiers: `apps/cli/src/engine/mcp-servers.ts` (env var names, tool allowlist), `apps/cli/src/workflows/catalog.ts` (workflow/agent slug order feeding both the Home browser and `relavium list`), `apps/cli/src/commands/list.ts`, `apps/cli/src/render/tui/mention.ts`, and `packages/core/src/tools/registry.ts`/`errors.ts` (`UnknownToolError`'s message, which can propagate into a `node:failed` RunEvent's `outputSummary` and thus into the NDJSON stream too) (#259). One shared `compareIdentifiers(a, b)` helper — a plain ASCII ordinal comparison, matching the one site that already gets this right (`packages/llm/src/model-catalog.ts`'s `localeCompare(y.displayName, 'en')`) — closes the whole sweep in one pass. *(M · apps/cli/src/engine/model-refresh.ts, apps/cli/src/engine/mcp-servers.ts, apps/cli/src/workflows/catalog.ts, packages/core/src/tools/registry.ts · #258, #259)* - **Estimate attachment token warnings from injected content, not raw file size.** `fileAttachmentWarning` (`apps/cli/src/render/tui/attachments.ts`) documents its estimate as computed "from the ACTUALLY-INJECTED (bounded) length — NOT the raw file size," but the implementation always derives `injectedTokens` from the `sizeBytes` parameter, never from `content.length`. For ASCII content the two values are numerically equal, masking the bug, but for any multi-byte UTF-8 content (non-Latin scripts, emoji, i18n fixtures) `sizeBytes` exceeds `content.length`, so the non-truncated-branch warning reports an inflated token estimate against the function's own documented contract. Switch to `Math.min(content.length, INJECT_MAX_CHARS)`, and extend the existing test with a `content`/`sizeBytes` pair that diverge, since the current fixture passes identical values for both and cannot catch this. *(S · apps/cli/src/render/tui/attachments.ts · G54)* - **Harden two structurally-checked type escapes that rely on control flow, not the compiler.** `catalog-metadata.ts` (`apps/cli/src/engine/catalog-metadata.ts`) casts Zod-validated catalog data to `CatalogPriceTier[]`/`ReasoningControls`/`RequestCapabilities` via `as X` on a money/capability-adjacent boundary feeding the ADR-0028 cost cap — TS rejects a structurally-incompatible cast, but not a future schema change that stays structurally assignable while becoming semantically wrong (#303). `schema-compiler.ts` (`packages/mcp/src/schema-compiler.ts`) uses non-null assertions (`built[0]!`, `literals[0]!`) that are correct today only because an adjacent `if` guards the invariant, not because `noUncheckedIndexedAccess` enforces it — a future reorder of that guard would silently reintroduce a runtime crash with no compiler signal (#304). Add a round-trip fixture test pinning the catalog schema-to-interface mapping for the former; destructure-and-check instead of asserting for the cheap cases of the latter, commenting the rest. *(S · apps/cli/src/engine/catalog-metadata.ts, packages/mcp/src/schema-compiler.ts · #303, #304)* - **Bound `SGR_MOUSE`'s digit groups to match the file's own defensive posture.** `SGR_MOUSE = /^\x1b?\[<(\d+);(\d+);(\d+)([Mm])$/` (`apps/cli/src/render/tui/mouse.ts`) accepts an unbounded digit run in each field on the complete-report path, while the partial-fragment path in the same file caps at `MAX_PARTIAL_LENGTH` (24) specifically "so a malformed stream cannot grow a buffer." Not currently exploitable — `Number()` on an oversized digit string saturates rather than throwing, and a real terminal only ever emits 1-5 digit fields — but it is an inconsistency against the file's own stated bounding discipline. Cap each digit group to `\d{1,5}`, matching the fragment path's defensive intent. *(S · apps/cli/src/render/tui/mouse.ts · G55)* @@ -429,7 +434,7 @@ This substream is the development-process substrate itself — `.github/workflow - **Cap `InputHistory.entries`.** `input-history.ts:23`'s `entries` array has no upper bound, unlike the `MAX_TOKEN_CHARS`/`MAX_TOOL_LINES` siblings in the same directory. Add the same kind of bound. *(S · `apps/cli/src/render/tui/input-history.ts` · G4)* - **Cache per-cluster widths and debounce resize wrap-cache invalidation.** `viewport.ts:56`'s `graphemeWidth` calls the full `string-width` library per already-segmented cluster, 2.5–6x slower than necessary on CJK/wide-script lines (`#64`); `chat-projection.ts:273`'s `entryWrapCache` fully invalidates on every `SIGWINCH` tick during a drag-resize instead of only the final size, compounding the cost (`#66`). Cache per-cluster widths and debounce resize invalidation. *(M · `apps/cli/src/render/tui/viewport.ts` · #64, #66)* - **Mount the four overlay components under adversarial render tests.** `PromptEditor`/`PaletteView`/`ReverseSearchView` (`prompt-view.tsx:1`, `G41`) and `MentionView`/`EffortTierList` (`palette-view.tsx:1`, `G53`, overlapping heavily with G41's coverage) all carry explicit sanitization-safety claims in comments but are never mounted by any test in the repo. Add `ink-testing-library` render tests per component in one PR. *(M · `apps/cli/src/render/tui/prompt-view.tsx` · G41, G53)* -- **Make chat's own `git_commit` denial message satisfiable.** `packages/core/src/tools/registry.ts:292` emits a workflow-authored `human_gate` denial message into chat that asks the user to do something chat structurally cannot produce. Give chat its own denial copy. *(S · `packages/core/src/tools/registry.ts` · #308)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)).** **Make chat's own `git_commit` denial message satisfiable.** `packages/core/src/tools/registry.ts:292` emits a workflow-authored `human_gate` denial message into chat that asks the user to do something chat structurally cannot produce. Give chat its own denial copy. *(S · `packages/core/src/tools/registry.ts` · #308)* - **Align `chat-resume`/`chat-list`/`chat-export` with the CLI's noun-verb convention.** These three flat subcommands (`apps/cli/src/commands/specs.ts:131`) break the `gate list`/`provider add` pattern used everywhere else. Add `chat resume`/`chat list`/`chat export` as the canonical forms, keeping the flat spellings as deprecated aliases. *(M · `apps/cli/src/commands/specs.ts` · #309)* - **Add a persistent keyless-state indicator to the Home strip.** `apps/cli/src/home/home-store.ts:73`'s `HomeSnapshot` carries no field for "no provider key configured," so a skipped or cancelled onboarding wizard leaves no durable nudge once the wizard screen is gone. Add a keyless-state field and a persistent nudge row. *(S · `apps/cli/src/home/home-store.ts` · #51)* - **Make onboarding's model-binding failure recoverable and regression-tested.** A `writeDefaultModel` failure inside the wizard (`apps/cli/src/onboarding/wizard.ts:241`) silently re-introduces the exact provider/default-model mismatch the wizard exists to prevent (`#49`), and there is no end-to-end test proving the chosen provider actually binds the first chat's model (`apps/cli/src/home/drive-home.test.ts:562`, `#52`). Make the write retried/recoverable and add the regression test in the same change. *(M · `apps/cli/src/onboarding/wizard.ts` · #49, #52)* @@ -969,6 +974,13 @@ The review report itself is not tracked in git (it lives under the gitignored `d #### Routed into the active Phase 2.6 — these land in that phase's own workstreams, not in this file +> **Pull-forward note — 2026-10-19:** `G30` (`http_request` advertised in chat with no allowlist path) is no +> longer 2.6.M work. The approved Wave 3 plan pulled it forward to +> [execution row 13](#proposed-implementation-increments) under Accepted +> [ADR-0112](../../decisions/0112-chat-network-grants-and-onboarding-state-are-operator-owned.md): a global-only +> `[chat_http].allowed_domains` with live narrowing authority, not coupled to credential-resolver or `web_search` +> activation. The generated row below keeps its original routing as history. + | Finding | Severity | Location | Issue | Lands in | Work item | |---|---|---|---|---|---| | `#88` | `high` | `packages/core/src/validate-catalog.ts:65` | D15 catalog load-check only fires on a node-level model override — dead for the normal pattern | `2.6.A` | nodeCatalogIssue only reads a node-level model override, never the resolved agent's own model… | @@ -3629,6 +3641,11 @@ the token heuristic is now in LLM `request-estimator`, not an adapter module. #### Decisions presented together before implementation +> **Approved 2026-10-16** — see [Wave 3 maintainer approval and implementation checkpoint](#wave-3-maintainer-approval-and-implementation-checkpoint--2026-10-16). +> Steps 1–8 merged as PR #95; the 2026-10-19 rulings for Steps 9–19 are in +> [Wave 3 Steps 9–19 preflight review and maintainer rulings](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19). +> Live progress is canonical in [current.md](../current.md#wave-3--clear-the-ground). + All entries below are **proposed**, not maintainer rulings. Approval covers the concrete public/API/secrets plan as well as the new ADRs. No proposed choice silently changes a currently Accepted ADR or a canonical reference before approval. @@ -3655,6 +3672,15 @@ No credential values or request captures belong in tracked evidence. #### Proposed implementation increments +> **Approved 2026-10-16** ([checkpoint](#wave-3-maintainer-approval-and-implementation-checkpoint--2026-10-16)). +> Rows 7a–8d, 9a–9d and every `Recorded split` / `Recorded plan amendment` row below were added later with the +> maintainer's approval; the 2026-10-19 amendment is recorded in +> [Wave 3 Steps 9–19 preflight review and maintainer rulings](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19). +> Steps 1–8 merged as PR #95; live progress is canonical in [current.md](../current.md#wave-3--clear-the-ground). +> *Paused 2026-10-26 after Step 9b.2:* Steps 9c–19 are pending behind the CLI MVP; the recorded splits of rows 10b, 11b, +> 13a–13c, 15a, 15c, 16b and 17a that move into the MVP, with their order amendments, are in the +> [MVP priority reset](#mvp-priority-reset--2026-11-04). + | Step | Scope, finding IDs and principal verification | |---|---| | 1 | **Serialized multi-file queue first:** shared close handling #22 (four named siblings plus newer owned journal sites) → safe source labels #4/G11 plus #3's UTF-8 source admission parity → exhaustive session/picker/run switches #300/#301/G3 and actual reasoning rendering. Preserve primary outcomes and host-safe-close ownership. For #3, use the sibling 4× byte precheck plus authoritative parser character bound; test a dense multibyte source above the old byte cap but below the character cap, ASCII character overflow and true byte overflow. Add native error and mounted output regressions. | @@ -3671,20 +3697,54 @@ No credential values or request captures belong in tracked evidence. | 8b | **Recorded split — creation-owned native processes:** tool POSIX group versus MCP PID, actual close/retirement, verified absolute Windows teardown admission and helper lifetime; connect OS/editor helpers and replace independent reapers without claiming full descendant containment. | | 8c | **Recorded split — workflow and render integration:** run/gate/budget/driver owners, real return/throw health priority, cancellation, terminal/job-control leases and force-quit routing. No timer certifies DB close or receipt settlement. | | 8d | **Recorded split — sessions, terminal and activation:** fresh/resumed/one-shot/Home/pending/replaced session lifetimes, retained publication failure and shared actual cleanup promises; connect caught slash/rebuild faults, restore terminal custody before diagnosis, then activate early coordinator/background/sink integration. Actual native/platform/packed controls and complete cumulative Step 8 acceptance follow. | +| PR95 | **Recorded plan amendment, 2026-10-19 (maintainer-approved; new scope from the [PR #95 independent review](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18), not a split of rows 1–19) — scrubber residuals, before 9a:** the `api-key:Bearer`/`Basic` label erasure that leaks the token; raw `?`/`#` URL userinfo masked unless the text before it is host-shaped **and** an `=` assignment separates it from the authority `@`, applied to the LLM leaf and core's twin predicate; an extended finite scheme list (postgres, postgresql, mysql, mariadb, redis, rediss, mongodb, mongodb+srv, amqp, amqps, ssh, sftp, ldap, ldaps, smtp, smtps) masking colon-bearing userinfo for the new schemes; module-load checks of the split-credential scanner's two invariants, a start prefilter, joined-view and glued-scheme witnesses, the load-sensitive 8 MiB timing case, and the causal-control pins. One increment with ordered commits, two NEW clean complete cumulative rounds and a separate security sitting. *Acceptance amended on 2026-10-21 by the maintainer's masker acceptance bar and round cap; closed under the round cap on 2026-10-22 ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)).* | | 9 | **Release map/custody substrate:** #46/ADR-0111, independently reviewed after the finite graph. Upstream TS maps, sole prepack final build, pre-hash trailer removal/staging, exact tarball/CLI+engine symbolication, encrypt-once candidate with permissioned prebuild discovery/reservation → pack/reuse → smoke → archive → npm → finalization. Required artifact/platform aggregation, all rerun modes, lost-staging refusal and permission-enforcing offline API controls. Dedicated archive/crypto security sitting. Fixture proof never claims production recipient/custody configuration. | +| 9a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage #46/ADR-0111 unchanged, no new contract) — release maps, final build and symbolication:** fresh uncached upstream original-TypeScript maps, the sole prepack final CLI build, final trailer removal before hashes, map and private-graph staging outside public artifacts and caches, exact tarball-to-final-JS binding, and real packed CLI and engine frame symbolication with wrong JS/map/source and missing-map negative controls. Ordinary builds remain map-free. *Split again on 2026-10-19 into 9a.1 and 9a.2 below, which replace it as increments ([Plan shape](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19)).* | +| 9a.1 | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — candidate build and finalizer:** an owned isolated candidate copy, fresh uncached upstream maps, candidate-mode tsup selection with absent `sourcesContent`, the prepack finalizer (upstream and final map validation, repository-relative sources, exact trailer removal before hashes), owner-only private staging outside public artifacts and caches, closure/seal adaptation, the ordinary map-free control, the PR #95 archive/worker fold-ins, and a fail-closed `release.yml` publish guard that refuses any non-release candidate until 9d and Wave 4b custody exist. The receipt stays `compiled_package_test`. *Implemented on `development` on 2026-10-22 ([checkpoint](#wave-3-step-9a1-implementation-checkpoint--2026-10-22)); independently accepted at `cf97b7f1` on 2026-10-24 ([acceptance](#wave-3-step-9a1-acceptance--2026-10-24)).* | +| 9a.2 | **Recorded split — manifest, symbolication and package-test receipt:** the private map manifest keyed by `dist/.js`, actual packed CLI and engine frame symbolication against the matching source revision, wrong JS/map/source and missing/malformed-upstream negative controls (product copies via `check:map-controls`, also a required uncached CI job), and the atomic `package_test` receipt flip across seal, binding, consumers, aggregate and both workflows. *Split on 2026-10-24 (maintainer-approved; ID coverage #46/ADR-0111 unchanged) into 9a.2a and 9a.2b below, which replace it as increments ([preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)).* | +| 9a.2a | **Recorded split, 2026-10-24 (maintainer-approved) — decoder, private manifest and symbolication:** an in-house strict source-map decoder; strict mapping and range validation of upstream and final maps in the finalizer; the private map manifest (`staging/manifest.json`, keyed by `dist/.js`, with a per-source digest inventory and the commit's tree), sealed after the archive inspection; real packed CLI and `@relavium/core` frame capture through a tooling-only preload during an offline `relavium import`, run on the finalized `dist` that the inspection proves byte-equal to the tarball; symbolication of those frames against the matching source revision (clean: the checkout's Git objects; dirty, local only: the candidate copy), wired into the seal before the receipt is written. The receipt stays `compiled_package_test`. Security sitting. *Independently accepted on `development` at `a92a7602` on 2026-10-24 ([checkpoint](#wave-3-step-9a2a-implementation-checkpoint--2026-10-24), [acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)).* | +| 9a.2b | **Recorded split, 2026-10-24 (maintainer-approved) — product map controls and the package-test receipt:** `check:map-controls` over product copies (wrong JS by line and column, wrong map, permuted sources, wrong source, missing, malformed and truncated upstream maps, a shifted valid upstream map, and the package-test producer's post-receipt rebuild refusals that ADR-0111 lists), run in `pnpm run ci` and as a required uncached CI job inside `ci`'s needs (the required contexts stay `ci` and `coverage`); then the atomic `package_test` flip across seal, binding, consumers, aggregate and both workflows, version 1 kept. Security sitting. *Preflight decisions approved on 2026-10-24 ([decisions](#wave-3-step-9a2b-preflight-decisions--2026-10-24)); independently accepted on `development` at `0f73c253` on 2026-10-24 ([checkpoint](#wave-3-step-9a2b-implementation-checkpoint--2026-10-24), [acceptance](#wave-3-step-9a2b-acceptance--2026-10-24)).* | +| NI | **Native file identity on Windows (maintainer-approved 2026-10-24; after 9a.2b, before 9b):** every file and directory identity check compares `dev`/`ino` read with `{ bigint: true }` — the filesystem policy's pinned roots, the workflow catalog's open check and the media store's pins, read check and temporary claims — so a replaced file or directory can never compare equal on Windows, where `ino` is a 64-bit NTFS file ID that a JavaScript number cannot hold exactly; a replacement test per site. Found by 9a.2a's push CI ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)); recorded in [deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24). Security sitting. *Preflight decisions approved on 2026-10-25 ([decisions](#wave-3-ni-preflight-decisions--2026-10-25)); independently accepted on `development` at `fc5af701` on 2026-10-25 ([checkpoint](#wave-3-ni-implementation-checkpoint--2026-10-25), [acceptance](#wave-3-ni-acceptance--2026-10-25)); its Windows clause closes after three green advisory Windows runs on pushed heads.* | +| 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. *Split on 2026-10-25 (maintainer-approved, L8) into 9b.1 and 9b.2 below, which replace it as increments ([preflight decisions](#wave-3-step-9b-preflight-decisions--2026-10-25)).* | +| 9b.1 | **Recorded split, 2026-10-25 (maintainer-approved) — container, isolated GnuPG runner and encrypt-once:** the strict USTAR dialect extracted from the archive reader with its writer promoted; the private container built from the sealed staging in a staging-relative layout, each map's digest checked against the private manifest and its size against the finalizer's record, which must name exactly the manifest's outputs and maps (the metafile must be a JSON object); an isolated public-only GnuPG runner (GnuPG ≥ 2.2.8; `--recipient-file` with no keyring or agent, and no trustdb but the empty one GnuPG may create in the listing home); a colon-listing pre-check that refuses missing, expired, revoked, unusable (v4 with cv25519 or RSA ≥ 3072 only) or unexpected recipients and secret material; `encryptOnce` (SEIPDv1 with MDC and AES-256, exactly one `ENC_TO` for the pinned subkey, the exact produced asset, a no-clobber publish, plaintext cleanup on every path); committed synthetic public fixtures, an ephemeral round-trip key, and the GnuPG prerequisite documented. Library and synthetic controls only: product paths and workflows stay unchanged. Crypto/custody security sitting. *Independently accepted on `development` at `147c2788` (2026-10-25) ([decisions](#wave-3-step-9b-preflight-decisions--2026-10-25), [checkpoint](#wave-3-step-9b1-implementation-checkpoint--2026-10-25), [acceptance](#wave-3-step-9b1-acceptance--2026-10-25)).* | +| 9b.2 | **Recorded split, 2026-10-25 (maintainer-approved) — reuse, binding receipt and scope separation:** retained-ciphertext reuse without re-encryption (digest and size verified, zero GnuPG calls); the release binding receipt's schema v1, its validator and a sealer that writes only after the ciphertext exists (9d adds the origin, map-check and `ci`-run binding); the production recipient loader, absent in Wave 3; synthetic and production scope separation. Crypto/custody security sitting. *Independently accepted on `development` at `563402e6` (2026-10-26) ([decisions](#wave-3-step-9b2-preflight-decisions--2026-10-25), [checkpoint](#wave-3-step-9b2-implementation-checkpoint--2026-10-25), [acceptance](#wave-3-step-9b2-acceptance--2026-10-26)).* | +| 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | +| 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. *Added 2026-11-04 (PR #96 review, maintainer ruling):* ADR-0111's "encrypt once" holds per container, not per destination — `encryptOnce` refuses a second ciphertext only in the same destination, so one container can still yield two valid ciphertexts in two destinations; 9d adds a global encrypt-once record (or an equivalent refusal) with its regression. No encrypted map is produced before 1.0 under release path B (D59), so the gap is latent until then. | | 10 | **CLI behaviour/discovery integration:** #45/#264/#265/#268/#13/#17/#18/#310/#309/#19 under ADR-0110, consuming Steps 6–9. Production owner category matrix, real root/subtree JSON help, pricing/chat aliases, non-echoing Commander refusal and signal-safe deletion consent. G15 is refuted and receives real engine/store/log preservation evidence, with no invented token coalescing. Repeat affected packed gates. | +| 10a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — diagnostic policy and output owners:** #45/#264/#265/#268 policy wiring, the production output-owner category registry with a drift test, verbose detail whose frames come from a same-build allowlist (no second build or post-finalization write), removal of the raw stack branch with G36/G46 re-scoped to its replacement, and the PR #95 lazy-dispatch cause-wrap and purity-root fold-ins. | +| 10b | **Recorded split — descriptor grammar and JSON help:** canonical chat paths and pricing spellings with working aliases and conflict refusal (#309/#18/#310), non-echoing Commander refusal with the #311 static hint family (moved here from row 17; certified in 17a), excess-argument refusal, subcommand global help (#17), real root/subtree JSON help (#13, including `help [path] --json`) and the purity guard. | +| 10c | **Recorded split — consent, G15 evidence and landing:** signal-safe key-deletion consent (#19), G15 engine/store/log preservation evidence, promotion of the implemented Steps 6–10 commands.md sections (moved here from Step 19, per landing checklist item 3) and the repeated packed gates. | | 11 | **Chat normalization and abort:** #236/#237/#242/#307 → #218 → #21/#23. Canonical token/money source with approved additive LLM export, preserved two-decimal cap formatting and real immediate abort while a turn is blocked. Audit Home's imports. | +| 11a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged, G54 moved here from row 17) — canonical token/money source:** #236/#237/#242/#307, including USD presentation of the conservative-hold disclosures, G54's character-based attachment estimate on the same helper, then #218 reseat vocabulary. | +| 11b | **Recorded split — immediate abort:** #21/#23 certification and blocked-turn abort, including Ctrl-C mapped to `/cancel` while a tool approval is pending (an approved ADR-0057 amendment: the approval is rejected, never accepted), with an approval-path security sitting. | | 12 | **Chat decomposition:** #220/#244 after preceding chat fixes. Split lifecycle/drivers, line handler, printers and reseat/clear ownership. Preserve compatibility entry points, W7/W8 receipts, unknown-usage, full-screen bounds, disclosure activation and cancellation. | +| 12a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — chat leaves:** driver contracts, pure surface decisions, printers, headless drivers, the lifecycle seam and live notices move out of `commands/chat.ts` with no behaviour change (the first part of #220). | +| 12b | **Recorded split — line handler and wiring:** #244 line-handler decomposition and the rest of #220's session wiring, clear/reseat ownership, the REPL loop and command bodies. `commands/chat.ts` keeps only the command entry points and their argument/dependency types, importers are repointed, and `chat.ts`-internal duplication is consolidated (Home's chat wiring moves onto the shared helpers in 13c). | | 13 | **Chat domain policy:** G30/ADR-0112 after decomposition, all fresh/resume/Home/agent/rebuild paths. Invocation-live policy owner shared by old/pending hosts, post-approval/post-prepare target guard, pinned global selection, suspension/recovery and latest-only build publication. Actual failed/overlapping Home rebuild and approval-race controls, alongside no-authority/SSRF/ports; separate egress/config-trust security sitting. | +| 13a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — pure substrate:** the DNS-hostname grammar and comparator (also rejecting `.internal`, which the SSRF floor always blocks), the strict `[chat_http].allowed_domains` key, the frozen dispatch target and the additive pre-dispatch check. | +| 13b | **Recorded split — invocation authority:** the pinned global selection and the invocation-live, monotonically narrowing HTTP authority owner wired into standalone chat fresh/resume/clear/reseat and `agent run`. | +| 13c | **Recorded split — Home integration:** Home's in-chat wiring first moves onto the shared session helpers from 12b, then shares the authority across old/pending/replacement hosts with latest-only publication; `http_request` is omitted from each turn's advertised tools while the live authority is empty or suspended (dispatch stays authoritative). Separate egress/config-trust security sitting. | | 14 | **Home controller:** certify G37, fix #281, record/test #125/D26 before #215/#243 decomposition; then #217/#222 scoped naming. Overlay discriminated union, subcontrollers, stale async work/focus/approval invariants and both live cost surfaces. | +| 14a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — correctness first:** certify G37, make the palette rejection observable (#281), settle and test #125/D26 on both live chat surfaces, and remove the unused `HomeDeps.exit` (PR #95 fold-in). | +| 14b | **Recorded split — decomposition and naming:** the overlay discriminated union with subcontroller decomposition (#215/#243) in one change, preserving the Step 8 and Step 13 invariants; the shared `[c]` reason-capture reducer on both Home and chat-ink (a call-site swap only); then #217/#222 scoped naming. | | 15 | **Viewport/input:** #58/#62/G40/G51 → #65 → #64/#66, then #63/#69/G4/G55. Grapheme-correct editing/truncation, tabs, bounded caches, safe resizing, TERM=dumb and both small-terminal surfaces, input history and mouse bounds. CJK measurement and real emitted rows. | +| 15a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — grapheme and tabs:** the shared grapheme primitive (#58/#62/G40/G51) and 8-cell tab expansion before width measurement (#65). | +| 15b | **Recorded split — width cache and resize:** the bounded 1,024-entry width memo and latest-size frame coalescing with immediate narrowing safety (#64/#66). | +| 15c | **Recorded split — terminal bounds:** 80×24 on both chat surfaces with no hidden affirmative input there or on the Home strip (#63; an approved ADR-0057 amendment), the `TERM=dumb` plain fallback including the bare-Home gate (#69; an approved ADR-0054 amendment), the 500-entry history and SGR bounds (G4/G55) and the PR #95 input-listener fold-in. Approval-path security sitting. | | 16 | **Onboarding and overlays:** #49/#52/#51/#53/#55/G5/G52/G41/G53 and #34 under ADR-0112. Real next-model/ephemeral invocation binding across clear/reseat, recoverable persistence, explicit acknowledgement/current keyless nudge, shared pre-read writer symlink refusal, surfaced typed recovery and deterministic existing-file comment-loss notices. Hidden-approval safety/color-off; keychain/config security sitting. | +| 16a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — shared writer:** the shared preference writer with pre-read link refusal and typed recovery, the acknowledgement key (#34) and path-free configuration diagnostics (extending the safe display-path discipline to `ConfigError` and `/doctor`). | +| 16b | **Recorded split — onboarding truth:** typed key availability, truthful onboarding outcomes, the current-state keyless nudge and color-off across all four clack prompt sites (#49/#52/#51/#53/#55). | +| 16c | **Recorded split — selection and overlays:** the invocation selection binding (an explicit ephemeral choice wins for the invocation; a reseat stays session-scoped), surfaced Home refusal, the one-pending-approval invariant (G5) and the overlay/color-off controls (G52/G41/G53). Whole-Step-16 keychain/config security sitting. | | 17 | **Remaining CLI/hygiene:** #24/#26/G43/G2/#12/#54/G14/#11/#47/#48/#240/#241/#311/G54/#303/#304; certify #20/G36/G46. One-shot coverage-aware cost/banner, actual-host doctor, typed faults, no silent missing-dist skip, guard misuse, attachment estimate and compiler checks. #239 remains the approved documented compatibility surface. | +| 17a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged, G54 moved to 11a) — CLI tails:** one-shot stdin hint and cost line (#24/#26), actual-host doctor (G43), typed faults (G2/#12/#54), #311 certification, consumer-audited dead exports (#240/#241), comments (G14/#11) and the four PR #95 exit-code literal sites. | +| 17b | **Recorded split — test and control hygiene:** #47/#48, catalog/compiler checks (#303/#304), certification of #20/G36/G46 and the PR #95 test-evidence and join-table nits. | | 18 | **openLocalDb LAST:** #278/#280 across every moved caller. Centralize the already-integrated classification/wrapping with fixed safe errors, attempt-all secondary cleanup, preserved causes, privacy/checkpoint notices and exact writer lifetimes. Actual command-family failure probes; no duplicate wrapper left behind. | | 19 | **Whole-wave acceptance and publication:** every item disposition; A/E/G and M2.5.5-3 evidence; ordinary/enforced coverage and required CI/build/fences/smokes/causal/platform checks at attributed source; measure and promote core's ≥90% line/branch floor to required CI as testing.md requires after Wave 3 coverage work, fixing meaningful missing coverage before closure. Two new whole-wave cumulative reviews, canonical doc/ADR landings and PR. I/whole phases/later product waves stay open. | +| 19a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — coverage:** measure core at the accepted Step 18 head, close meaningful gaps, then promote core's ≥90% line and branch floor into the required coverage job, renamed `engine coverage floor` (the maintainer updates branch protection). | +| 19b | **Recorded split — whole-wave acceptance:** every disposition, canonical landings and the Proposed-section inventory, final exact-head gates, two NEW whole-wave cumulative reviews and the PR. | Steps are dependency-safe increments, not permission to parallel-edit shared hot -files. Within each row, arrows remain strict. The process/packaging sequence replaces the old fatal-before-lazy assumption: pure contracts → guarded lazy graph → native fatal integration → map/archive custody → remaining CLI behaviour. It is nineteen increments, with no parallel edits of shared hot files. Step 1 completes before all other +files. Within each row, arrows remain strict. The process/packaging sequence replaces the old fatal-before-lazy assumption: pure contracts → guarded lazy graph → native fatal integration → map/archive custody → remaining CLI behaviour. It was approved as nineteen steps; the recorded splits and the 2026-10-19 amendment make it 29 increments (see the rulings' Plan shape), and the 9a.2, NI and 9b splits of 2026-10-24/25 make it 32, with no parallel edits of shared hot files. Step 1 completes before all other code. Home cost is disposed or implemented before its decomposition; viewport geometry work never races editor/cache work. If review exposes a necessary smaller increment, split it with unchanged ID coverage and record the split before proceeding. @@ -3715,6 +3775,10 @@ preflight test pass or a clean proposal review. Approval is requested because th user explicitly requires it for new ADRs and implement-task step 3 requires it for public API/Accepted-decision/secrets changes. No product step is complete yet. +> *Superseded status, noted 2026-10-19:* this gate was satisfied on 2026-10-16 +> ([approval checkpoint](#wave-3-maintainer-approval-and-implementation-checkpoint--2026-10-16)), and +> Steps 1–8 merged as PR #95. The sentence above records the proposal stage. + #### Proposal review disposition — 2026-10-15 Two new independent complete cumulative round-1 reviews of all seven proposal paths @@ -9141,3 +9205,2529 @@ criteria; [PR #95](https://github.com/HodeTech/Relavium/pull/95) owns their live result. Source acceptance does not assert a green retry or completed publication. Steps **9–19 have NOT STARTED**. No merge, release, new user decision or deferral is claimed. + +### Wave 3 Steps 1–8 merge checkpoint — 2026-10-18 + +[PR #95](https://github.com/HodeTech/Relavium/pull/95) merged to `main` at +**22:44:24 UTC (01:44:24 Europe/Istanbul, 2026-10-19)**, merge commit +`6d2a737231e2445c6d9d54b2d7449084e036ade4` (parents `ab08209e` and `a3e164b5`). The merged +head is `a3e164b5608d386fc3db7fa3b8483d0adf2ed0fd`; the merge tree +(`415af9662f5690b675ebaf4ccfff5840ea6e3f29`) is byte-identical to it. GitHub records +649 changed files, +81,844 / −7,343. **Wave 3 Steps 1–8 are complete on `main`. Wave 3 is +not: Steps 9–19 have NOT STARTED and whole-wave acceptance remains Step 19.** ADR-0110–0113 +remain Accepted. The PR was the maintainer-authorized, scoped publication checkpoint recorded +in [Wave 3 Steps 1–8 scoped publication](#wave-3-steps-18-scoped-publication--2026-10-17); it +claimed no whole-wave acceptance and this checkpoint does not add one. + +#### Completed Steps 1–8 scope + +| Step | Complete on `main` | +| --- | --- | +| 1 | Shared close handling, safe source labels with UTF-8 admission parity, exhaustive session/picker/run switches and actual reasoning rendering. | +| 2 | Shared admission and diagnostics: status/duplicate matrices, shared parser reason helper and config numeric bounds. | +| 3 | Secret provenance at public sinks (ADR-0113): engine-owned context/input provenance across fresh, resume, condition, transform and merge paths. | +| 4 | Typed error/envelope substrate (`RelaviumError`, migrated classes, cross-bundle structural guards). | +| 5 | Tools and engine tails: bounds before effects, model-visible semantics, ordinal helper, path-local cycle guards, cancellation race and deterministic plan equivalence. | +| 6 | Pure process/discovery substrate (ADR-0110): categories, early flag grammar, exit/site inventory, descriptor registry and invocation owner registry. | +| 7 | Lazy runtime, finite compiled graph, exact-tarball inspection and the required Node 22.0.0 plus Linux/macOS/Windows consumers (ADR-0111). | +| 8 | Creation-owned native process integration, kind-specific owners, terminal custody and the total fatal budget across workflow, render and session lifetimes. | + +#### Review attribution + +Each step keeps its own acceptance register; none is widened by the merge. Whole Step 8 +is accepted at `c83c8575` (two NEW complete cumulative CLEAN rounds, 342 paths / 1,842 +ordered hunks, separate clean security sitting). The two PR #95 correction cycles are +accepted at `ec0f3e10` ([second corrective source-review completion](#pr-95-second-corrective-source-review-completion--2026-10-18)), +and the two later native test-fixture increments (`154d832d`, `1c7aec25`) have their own +[supplemental acceptance](#pr-95-supplemental-native-fixture-source-acceptance--2026-10-18). +Detailed review records remain local and Git-ignored. + +The difference between `ec0f3e10` and the merged head is exactly twelve non-documentation +paths plus three documentation files. Two test files are the independently accepted fixture +increments above. The other ten are the commit below. It merged with author verification only; its +post-merge independent review is recorded in [its own section](#independent-review-of-a3e164b5-and-its-follow-up-corrections--2026-10-19) after this checkpoint. + +#### Independent review of the published head and its corrections + +After the accepted sources, a further independent systematic review of the PR at `19c99968` +(reviewer-reported: two rounds, 14 agent sessions in total, all 648 changed files read at diff level, +about 5,000 targeted test runs, five red-team attacks on the headline invariants) reported **no blocker, no high, one +medium, four low and about seventeen nit/info/suggestion items**, with four items withdrawn after +its own verification. The attacks did not break a headline claim; the test-evidence attack held +only partially (LOW-4). Its detailed records stay local. The author reproduced each finding +before acting. Corrections landed as `a3e164b5`: + +| Finding | Disposition | +| --- | --- | +| **MEDIUM-1** — URL userinfo not masked when its scheme is glued to a word character (`xhttps://user:pass@host`) | Reproduced at `19c99968` through both `scrubSecrets` and the CLI sanitizer. Fixed in `packages/llm/src/scrub-secrets.ts`: the rule returns to its pre-PR word-adjacency (no `\b`) and supplies its own joined pattern, because the split-credential scanner otherwise strips two characters from a pattern that has none. Fourteen new `scrub-secrets` cases and one CLI case; reverting the production change fails all fourteen (the CLI case failed against the stale build before it was rebuilt). | +| **LOW-4** — failed-spawn test does not pin the native close join | Confirmed by mutation: removing `reservation.whenClosed` left all three `process-close` tests green. The reviewer's suggested `activeOwners === 0` assertion was also tried and **did not** kill that mutant, because the owner retires synchronously inside its own `close` handler on POSIX. Replaced by a case that gates the reservation's `whenClosed` and observes publication; it fails only when the join is removed. The post-merge review extended it to all four publication branches. | +| **LOW-2** — default 1 s `vi.waitFor` on a real spawned child | Added in `a3e164b5` (an explicit 4 s bound on four waits), then **reverted** after the post-merge review: the only wait that observes a real child booting is enclosed by 1 s waits in its sole caller, so a longer inner bound could not help. Examined and deliberately unchanged; see deferred-tasks.md. | +| **LOW-3** — POSIX `ps` suite not skipped on Windows | Fixed: `describe.skipIf(process.platform === 'win32')`. Prettier re-indented the suite body, so the raw diff of that file is 215 lines; `git show -w` shows the 7-insertion / 2-deletion semantic change. | +| **LOW-1** — fixed 5 s native watchdogs | **Not changed.** Three runs of the three affected native files (52 tests) all passed at a load average of 80–96, the reviewer could not observe the Windows root cause, and the constants are coupled to neighbouring bounds, so a retune would be speculative. Recorded in [deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18). | +| Nit — import above the module docblock in `parser.ts`, `agent-parser.ts` | Fixed (no behaviour change). | +| Nit — `packages/shared` curated export list | Closed without change: it is an illustrative existence sample, not an exhaustive contract. | +| Other nits and info items | Open, in the same deferred-tasks section. | + +The production change restores the pre-PR word-adjacency for http(s) (and gives the ws/wss/ftp/ftps +schemes this PR added the same behaviour), but it touches a secret-scrubbing rule and +the repository's causal controls: the `diagnostic-raw-password-userinfo` source anchor moved to +the new `URL_USERINFO_PATTERN` constant, and the pinned complete workloads changed (89→103, +179→180, 277→278) because the files gained cases. **At merge, verification was the author's only.** The +full local gate was run in two parts: `pnpm run ci` through the workspace tests, builds, format, +fences, bundle closure and smokes; then, after the anchor/pin update, `test:acceptance-tools` +(633 of 633) and the secrets, wire, usage and filesystem mutation gates (exit code 0 each, the +userinfo control 103 / 98 with five failing cases / 103), plus tools lint, typecheck and format and +the affected suites under `CI=true GITHUB_ACTIONS=true`. A second complete `pnpm run ci` after +the pin update and a coverage run were not performed. No independent review rounds or security +sitting had covered `a3e164b5` at merge; the review that followed is recorded after this checkpoint. + +#### Publication state observed at merge + +The merged head's [PR CI](https://github.com/HodeTech/Relavium/actions/runs/39169681878) and +[push CI](https://github.com/HodeTech/Relavium/actions/runs/39169678515) started at 21:51 UTC. +Observed at 22:48 UTC, four minutes after the merge, each had 14 of 15 jobs completed +successfully (including the four same-archive installed-CLI consumers and the causal-control +families) and no failed job. The aggregate `lint · typecheck · test` job, one of the two +required status checks on `main` (with `engine coverage floor (llm, mcp)`), was still running; in the PR run it +started at 22:46:21 UTC, after the merge. The merge commit's own +[push CI on `main`](https://github.com/HodeTech/Relavium/actions/runs/39173540595) started at +22:44:30 UTC. **That observation asserted none of the eventual conclusions.** Earlier full-CI, +coverage, SonarCloud and platform results stay attributed to the sources recorded in the preceding +registers and are not carried to `a3e164b5` or the merge commit. + +**Later observation, 2026-10-19 06:53 UTC.** All three runs completed with conclusion `success`, +**15 of 15 jobs each**, and both required checks succeeded on the merged head (`lint · typecheck · test` +completed 22:54 UTC in the push run and 23:00 UTC in the PR run). Job logs, coverage figures and +test counts were not re-read for this record. SonarCloud's PR analysis passed its quality gate +(102 new issues, 86 accepted). Its analysis of the **`main` merge commit failed the quality gate**: +`new_security_rating` C against the required A, from 9 open vulnerabilities on the `previous_version` +baseline (2026-06-04) — seven `githubactions:S6505` findings on workflow `pnpm install` lines +(created June–July, deliberately declined with a recorded rationale but still open on the `main` +branch, where SonarCloud already carries five other S6505 findings as false positives), one +`jssecurity:S8707` (created 2026-10-08; seventeen sibling findings are already false positives +on `main`) and one `javascript:S4036` (created 2026-10-17; won't-fix on the PR #95 branch, open on +`main`). SonarCloud statuses are per branch, so these are existing dispositions not yet applied +on `main`. The same gate also failed on the `main` merge +commits of PR #90, #92 and #94, so this was a standing condition of `main`, not a regression of +PR #95, and SonarCloud is not a required check. **Resolved 2026-10-19 10:50 UTC:** at the +maintainer's direction the author applied those dispositions on `main` through the SonarCloud +API, each with a reviewed comment (the seven S6505 and the S8707 as false positives, the S4036 +accepted), after re-verifying the recorded rationale against the repository at `6d2a7372` — the +pinned pnpm 9.12.3, the two-package `onlyBuiltDependencies` allowlist, no workspace install hooks, +the S8707 path coming from a fresh owned scratch directory, and `RELAVIUM_PACKAGE_SOURCE_SHA` set by +both workflow steps that run the aggregator. SonarCloud then reported **zero open vulnerabilities +and an OK quality gate for `main`**. No rule, threshold or quality gate was changed. The failed +check-run attached to the merge commit itself is historical and unchanged. Recorded in +[deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18). + +#### Exit status and Step 9 as next work + +**Wave 3 Steps 1–8 are on `main`; Wave 3 remains in progress.** Open after this merge: Steps 9–19 +and whole-wave acceptance (Step 19: A/E/G and M2.5.5-3 evidence, promoting core's ≥90% line/branch +floor to required CI, the reconciled EXIT:4 reading, two new whole-wave cumulative reviews); +and the unactioned review findings. Phase 2.5.5 exit criteria 4–8, sub-stream I, later +product waves and the existing approved deferrals keep their canonical homes. The merge adds no +deferral, ADR or release. + +**Next is Step 9 — release map/custody substrate** (#46, ADR-0111): upstream TypeScript maps, a +sole prepack final build, exact tarball symbolication, the encrypt-once candidate through +discovery/reservation → pack/reuse → smoke → archive → npm → finalization, with a dedicated +archive/crypto security sitting. The maintainer-approved 9a–9d subdivision is recorded in the +[increment table](#proposed-implementation-increments), and 9a is first. Steps 10–19 follow the +approved plan in order. New implementation +commits on `development` now start a new publication checkpoint; PR #95 no longer bounds their review scope. + +### Independent review of `a3e164b5` and its follow-up corrections — 2026-10-19 + +The merged delta `19c99968..a3e164b5` had author verification only. After the merge it was reviewed by +independent reviewers, each in an isolated worktree at an exact tree, given the code and a neutral statement of +purpose, never the author's conclusions: three area reviewers (the scrubbing rule and its tests; the +causal-control pins and README; the CLI test changes and two import moves) and a separate security sitting on +the rule. Later rounds review the **cumulative** result, so the fixes made between rounds are reviewed too. Detailed +records stay local and Git-ignored. + +| Round | Source | Result | +| --- | --- | --- | +| 1 | `a3e164b5` (tree `415af966`) | All four NONCLEAN: no blocker, high or medium; low and nit findings. | +| 2 | `f597977a` (tree `7fd14b05`) | Rule and tests **CLEAN**, pins and README **CLEAN**, security sitting **CLEAN**; CLI tests NONCLEAN (one medium, three low). | +| 3 | `fcfe79b1` (tree `b45d510d`) | Rule and tests **CLEAN**, pins and README **CLEAN**, CLI tests **CLEAN**. | +| 4 | `fcfe79b1` | CLI tests **CLEAN**, a second fresh reviewer on the same bytes. | + +Each area therefore has two clean rounds on identical code bytes (rule and tests: rounds 2–3; pins and README: +2–3; CLI tests: 3–4), plus one clean security sitting (round 2). Differential corpora ran the pre-PR matcher, +`19c99968`, `a3e164b5` and the corrected tree side by side: the scrubbing reviewers used 0.8–1.1 million structured +inputs and up to 2.2 million random strings, the security sitting 48 million exhaustive and 31.5 million random +token sequences with a planted-token leak oracle (which did detect leaks in the three older versions). **Against +`19c99968` no input regressed, apart from accidents of that version:** the few inputs it redacted wholesale only +because its raw and normalized views disagreed (for example an ANSI-coloured URL whose credential is split by a +plain space) are now masked in place and keep the documented opaque-fragments-across-spaces limit that the +pre-PR matcher shares. **Against the pre-PR matcher** the only inputs it masked and the corrected tree does not +are the raw-`?`/`#` userinfo class recorded in deferred-tasks.md. Reviewers ran on macOS only, and no review ran SonarCloud +on the final code or exercised Windows. Three comment-only corrections in `scrub-secrets.ts` and two README +sentences were made after the last review of those files; they change no behaviour, anchor or count. + +| Finding (source) | Disposition | +| --- | --- | +| A glued scheme with an LF/TAB-split credential that also holds a raw `@` left a partial mask with part of the credential visible, e.g. `xhttps://user:p@ss\tword@host/v1` (security sitting, round 1; low). | **Fixed** in `c64f3c39`: the URL rule's `boundary` blocks no start, and the scanner's word-character shortcut applies only when a rule's boundary rejects word characters. The anchored scanner gate and pattern line are untouched, so no causal-control anchor moves. Two regression cases (reverting the gate fails exactly those) and CLI assertions. | +| The join of the owner reservation was pinned on the failed-spawn branch only; a mutant keeping it there alone passed every `process*` test (round 1; low). | **Fixed** in `f597977a` and `ebbec1c0`: one table over failed-spawn, natural exit, abort and timeout, mutation-verified branch by branch, against a stop-based join, against ungated publications up to about 100 ms late, and with the child reaped in a `finally` (removing it leaks a child). | +| The 4 s `waitFor` bound could not help its only caller (round 2; medium). | **Reverted** in `ebbec1c0`; `native-forwarding.test-support.ts` is byte-identical to its `19c99968` form. Examined and deliberately unchanged. | +| Stale pinned counts and README numbers (round 1; low). | **Fixed**: scrub-secrets set 89→111, sanitize + render-error 179→180, llm-error + sanitize + render-error 277→278; the README carries them and 25 files / 1,343 cases, measured independently by three reviewers. | +| Inaccurate or overclaiming comments (rounds 1–3). | **Corrected**: ws/wss/ftp/ftps were this PR's additions, not pre-PR behaviour; "never reveal" and "all fail-closed"; the `slice(2)` contract; the `'a'` proxy; the `mcp-orphan` skip reason. | +| Whole-diagnostic redaction of prose next to a path-less URL and a later `@`, and suffix matches such as `sftp://` (rounds 1–3; low). | **Characterised, not changed**: three tests and a docblock state the trade-off. It fails closed, equals the pre-PR matcher for http(s), and is new for the schemes this PR added. | +| Residuals that predate the follow-up and are not in this delta: an `api-key:Bearer ` token leak, userinfo with a raw `?`/`#` before the first `:`, URL schemes outside the finite rule, a constant-factor scan cost, two unchecked invariants, an equivalent-mutant coverage gap, the load-sensitive wall-clock test and join-table nits. | **Open, not fixed**: recorded in [deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18). | + +Author verification of the corrected tree: full local `pnpm run ci` passes end to end at `7913b7b7` (workspace lint, typecheck, tests and build; format; fences; bundle closure; smokes; the acceptance tools; and the wire, usage, filesystem and secrets mutation families); the affected suites pass with and +without `CI=true GITHUB_ACTIONS=true`. The `a3e164b5` subject line (`refactor(tests)`, a non-package scope on a +change that spans llm, core, cli and tools) is already published and is not rewritten. The follow-up commits are +`c64f3c39`, `f597977a`, `ebbec1c0`, `fcfe79b1` and `7913b7b7`. They are on `development`, pushed to +`origin/development` with the recording commit `9902e0a0`, and are **not on `main`**; they reach `main` with the next +publication checkpoint. [Push CI 39275223233](https://github.com/HodeTech/Relavium/actions/runs/39275223233) at +`9902e0a0` completed `success` on 2026-10-19; source CI certifies no later documentation commit. + +### Wave 3 Steps 9–19 preflight review and maintainer rulings — 2026-10-19 + +**Source:** `development` = `origin/development` = `9902e0a0`. Steps 9–19 had not started. Before implementation, +sixteen scoped auditors read the canonical documents for Steps 9–19 and the open PR #95 findings in full and checked +every requirement against the current source; fifteen independent adversarial verifiers re-checked each audit, and a +completeness critic compared all of them against the execution table and the current.md queue. Every finding ID in +rows 9–19 has an owner. The detailed reports stay local and Git-ignored. **No new ADR is required.** Three approved +amendments to Accepted ADRs will be appended when their increments land: ADR-0057 for Ctrl-C through the approval +swallow (11b) and for no hidden affirmative input below 80×24 (15c), and ADR-0054 for the `TERM=dumb` bare-invocation +gate (15c). Dated implementation notes follow the canonical landing checklist: ADR-0051's no-map clause (9a/9d), +ADR-0111 (9d) and ADR-0110's frame mechanism and static hint family (10a/10b). + +The maintainer approved every recommendation below on 2026-10-19. The increment table above records the resulting +plan amendment (row PR95) and the recorded splits; ID coverage is unchanged apart from PR95's new scope. + +| Area | Ruling | +| --- | --- | +| Acceptance | Each increment keeps the existing rule: implement, author gates, commit, then NEW independent complete cumulative review rounds until two consecutive rounds are clean after the last material fix. Comment- or documentation-only nits do not reset the count. Security-touching increments add a separate scoped sitting: PR95, 9a.1, 9a.2, 9b, 9c, 9d (whole Step 9 custody), 10a, 10c, 11b, 13c, 14b, 15c, 16a, 16b, 16c (whole Step 16), 18 and 19b (whole wave). PR95 instead follows the maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)). **Amended 2026-10-23 (maintainer, after Step 9a.1's twelfth round):** from Step 9a.1's round 13 on, every increment is accepted after one NEW independent complete cumulative round that reports no blocker, high or medium finding and no regression; that round's low findings are recorded in [deferred-tasks.md](../deferred-tasks.md) with their file, a minimal example and a re-check note rather than fixed in the loop, and nits are optional. A high or medium finding is still corrected and followed by a new round, and the separate security sittings above are unchanged. **Amended 2026-11-04 (D64):** tiered for the CLI MVP — see the [MVP priority reset](#mvp-priority-reset--2026-11-04). | +| Publication | `development` is pushed after each accepted increment, so its actual CI (including the four same-tarball platform consumers) is observed per increment. The Wave 3 PR follows Step 19. **Amended 2026-11-04 (D61):** a scoped `development`→`main` checkpoint follows the [MVP priority reset](#mvp-priority-reset--2026-11-04); the Wave 3 PR still follows Step 19. | +| Plan shape | PR95 + 9a.1, 9a.2, 9b, 9c, 9d + 10a–c + 11a–b + 12a–b + 13a–c + 14a–b + 15a–c + 16a–c + 17a–b + 18 + 19a–b (29 increments). Each split is recorded before its step starts. **Amended 2026-10-24 (maintainer):** 9a.2 is split into 9a.2a and 9a.2b, so the plan has 30 increments ([9a.2 preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)). **Amended 2026-10-24 (maintainer, E6):** the native-identity increment NI is added after 9a.2b, so the plan has 31 increments ([9a.2a acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)). **Amended 2026-10-25 (maintainer, L8):** 9b is split into 9b.1 and 9b.2, so the plan has 32 increments ([9b preflight decisions](#wave-3-step-9b-preflight-decisions--2026-10-25)). | +| Reassignments | G54 moves from row 17 to 11a; #311's implementation moves to 10b and its certification stays in 17a; G36/G46 are re-scoped in 10a to the branch that replaces the raw-stack branch and are certified in 17b; promotion of the Steps 6–10 commands.md sections moves from Step 19 to 10c. | +| Interim release guard | 9a.1 adds a fail-closed check to the `release.yml` publish job that refuses any non-release candidate. Consequence, acknowledged: no npm publication, including a v0.1.x patch, until 9d and Wave 4b production custody exist. **Amended 2026-11-04 (D59):** to be superseded by a pre-1.0 release ADR (option B, sunset at 1.0 or when 9c/9d land); the guard stays in force until that ADR is Accepted and its guard change passes a security sitting ([MVP priority reset](#mvp-priority-reset--2026-11-04)). | +| PR95 scrubber residuals | Fixed before 9a: the `api-key:Bearer`/`Basic` leak; raw `?`/`#` userinfo masked unless the text before it is host-shaped and an `=` assignment separates it from the authority `@`, in both the LLM leaf and core's twin predicate (core's other documented differences are recorded, not removed); an extended finite scheme list that masks colon-bearing userinfo for the new schemes; the scanner invariants, prefilter and witnesses; the load-sensitive 8 MiB timing case. LOW-1 and LOW-2 become approved deferrals with their recorded triggers. | +| Step 9 mechanics | The encrypted plaintext container is an in-house strict USTAR (regular files, repository-relative paths) recoverable with standard `gpg -d \| tar -x`; GnuPG ≥ 2.2.8 becomes a prerequisite of local `pnpm run ci` and of the CI Ubuntu runner, documented in local-dev-setup.md, and its controls fail rather than skip when it is absent; release-mode staging artifacts are retained 30 days, package-test artifacts 7; product-copy map negative controls run as `check:map-controls` locally and in a required uncached CI job; the private manifest keys final JS as `dist/.js`; the production recipient must be an OpenPGP v4 key with a cv25519 or RSA≥3072 encryption subkey, a Wave 4b prerequisite recorded in [deferred-tasks.md](../deferred-tasks.md#wave-3-approved-later-wave-ownership--2026-10-16). | +| Step 10 compatibility | Verbose frames use a same-build allowlist (a tsup `renderChunk` substitution of a fixed-length placeholder, so no second build or post-finalization write) and appear only on classified non-infrastructure diagnostics; `--quiet` treats live progress as decoration on TTY surfaces; excess positional arguments are refused on every non-variadic command; plain `note:` stderr lines become `{type:'diagnostic', code, message}` records in JSON mode; `help [path] --json` returns the discovery DTO; human `--help` keeps Commander behaviour for unknown grammar; deprecated aliases emit one structured JSON notice or one fixed human line kept under quiet and are hidden from human help listings. | +| Steps 11–12 | The raw-µ¢ conservative-hold disclosures move to USD with their wording kept (a recorded #307 extension). Ctrl-C while a tool approval is pending maps to `/cancel`, rejecting the approval. `commands/chat.ts` keeps only its command entry points and their argument/dependency types after Step 12; `chat.ts`-internal duplication is consolidated in Step 12 and Home's chat wiring moves onto the shared helpers in 13c. | +| Step 13 | `http_request` is omitted per turn while the live authority is empty or suspended; `[chat_http].allowed_domains` also rejects `.internal`. | +| Step 14 | The `[c]` reason-capture reducer is shared by Home and chat-ink (a chat-ink call-site swap only, consistent with D18). | +| Step 15 | `TERM=dumb` (exact match) selects the plain renderers and makes the bare invocation print help; below 80×24 only Ctrl-C, Esc, `n` (reject), Ctrl-Z and mouse-report consumption act on either chat surface and on the Home strip. | +| Step 16 | An explicit ephemeral model choice wins over project, workspace and global defaults for the invocation and the UI says when it shadows a project default; a successful `/models` reseat does not update that binding. Color-off applies to all four clack prompt sites. `ConfigError` and `/doctor` stop printing absolute configuration paths. | +| Step 17 | The one-shot cost line goes to stderr after the turn settles, also after an abort or infrastructure failure once a provider call was admitted (qualified known total). Bare-Home `/doctor` reports the `chat-read-write` host profile a new chat binds plus Step 13's live authority view. The #24 stdin hint and the new notices of Steps 13, 15 and 16 are preserved under `--quiet`. | +| Step 19 | The coverage job is renamed `engine coverage floor` when core is promoted; the maintainer updates branch protection. | +| Housekeeping | 54 merged local branches were deleted with `git branch -d` and local `main` was fast-forwarded to `6d2a7372`; the restore list is kept locally. | + +PR #95 item owners: archive `TAR_FORMAT` dead branches and the `worker.mjs` ancestor loop → 9a.1; the lazy-dispatch +cause-wrap and the purity-guard root set → 10a; `HomeDeps.exit` → 14a; the kitty `data` listener → 15c; the four +exit-code literal sites → 17a; test-evidence and join-table nits → 17b; the duplicate-heading nit (with its corrected +premise) → 19b. [deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18) +carries the same owners. + +### Wave 3 PR95 increment implementation checkpoint and review-round corrections — 2026-10-20 + +**Scope and source.** The PR95 increment implements the PR95 scrubber residuals ruling above in +`packages/llm/src/scrub-secrets.ts` (with its package-internal `split-credential-probe.ts` and +`fixed-point.ts`) and in core's twin projection, `packages/core/src/tools/bounding.ts`, together with +the CLI's authored-content detector, the secrets causal controls and the canonical documents. Base +`9902e0a0`; commits `38e10d42` (Docs-0, the rulings), `dda3b45e` (implementation) and the +corrections `ef3ded0d`, `bdd292a6`, `6fb7e9d9`, `b2c50e88`, `6b2e94dd` (round 5, which added this +section), `b89ac62e` (round 6), `082962e2` (round 7), `6f7dd342` (round 8), `59c64498` (round 9) and the +round-10 commit. The commits stayed local on `development` until the increment closed. + +**Review rounds.** Each round is a NEW independent complete cumulative review of `9902e0a0..HEAD`, every +reviewer in its own worktree detached at the reviewed head: scrubber correctness, tests and causal +controls, documentation, integration, a separate security sitting and, from round 3, generative +properties and a completeness critic. Every finding is adversarially verified. The detailed reports +stay local and Git-ignored. + +| Round | Reviewed head | Verdict | Verified findings | Main corrections | +| --- | --- | --- | --- | --- | +| 1 | `dda3b45e` | NONCLEAN | 15 (2 high) | the host-shaped prefix bounded at 260 characters before the RegExp, which had exhausted V8's stack at admitted size, with its anchors and port bound pinned by tests; stacked labels; already-masked userinfo kept | +| 2 | `ef3ded0d` | NONCLEAN | 15 | quote-closed labels, bounded label stacking, masked tokens as fixed points | +| 3 | `bdd292a6` | NONCLEAN | 17 | sequential rule application replaced by one merged, token-only masking pass | +| 4 | `6fb7e9d9` | NONCLEAN (integration clean) | 19 | the literal query names in the merged pass; concatenated label runs and the nine-label bound; one standalone label rule for both labels; a bounded fixed point in both maskers; the secrets CI job allowance becomes 60 minutes | +| 5 | `b2c50e88` | NONCLEAN | 20 (2 high) | core's round-4 authority extent erased the context its sequential rules read, so core now masks an ambiguous userinfo with exactly the query values its later rules mask whole; its split view masks URL userinfo before scheme labels; the LLM split scan retries a kept candidate glued across a line break; tested fixed-point helpers; untested pattern pieces gained discriminating tests | +| 6 | `6b2e94dd` | NONCLEAN | 21 (9 high) | round 5's value mirror was quadratic and still erased context, and its split view dropped the value mask; core now merges every rule's ranges on one text, as the scrubber does, reads a key or scheme label inside another value and a value through earlier masks, reads a split header label or a scheme label right after a credential, and scans PEM blocks linearly; the scrubber's nine-label bound also reads the raw text and its literal query names follow `#`; both fail-closed values are tested | +| 7 | `b89ac62e` | NONCLEAN | 17 (2 high) | only a userinfo's username owns its keys, never past an LF or TAB, so a key in its password or parameter position or on the next line masks its value through the last `@`; quoted values are found by a character scan that an admitted 8-MiB value cannot exhaust; a credential glued to a fixed-length one is masked in the same pass; summaries use the projection's fixed point; a tiered call whose journal projection does not settle is refused before its prepare; the secrets controls run as two CI shards so that each evidence export fits the unchanged 64-MiB bound, which the single export had exceeded; untested pieces gained discriminating tests | +| 8 | `082962e2` | NONCLEAN (integration and security clean) | 16 (1 high) | round 7's glued-credential chain cut wider credentials and swallowed key names, and is gone; a structured result's summary reads its JSON as one text once credential members are masked; quoted values are read one JSON string down, and one that does not close on its line is read as an unquoted run from after its quote, no further than that line's end (one JSON string down, its string's end), unless another mask of the pass takes its line break and it closes; a summary masks one stopped by a line break through a later quote instead; a header label at a line start inside a split continuation is read; the scrubber's split probe also reads the raw text; a password-position key's extent is documented as credential equality. Before the commit the author ran the reviewers' method on the candidate and six internal pre-reviews: the first found three of the draft's alternatives unsound (a trusted host-shaped tail, quoted values across lines, a quadratic line-start scan), and they were dropped; the second (17 findings, none high) led to the summary's through-the-closer masking, colliding member names refused as unsettled and escaped separators and opaque values before escaped quotes; the third (27 findings, two high) found that the round's own additions failed — masking a value that never closes to its line end merged journal identities, and the scrubber's escaped header separator and terminal final-byte view let a token leak or blanked coloured diagnostics — so such a value is read as an unquoted run, the two scrubber additions are withdrawn and documented as limits, and a structured summary's walk became linear at a small constant factor; the fourth (16 findings, two medium; the critic clean, with all 128 controls run) found that the run and the opaque mask's kept backslash showed values that earlier versions masked whole in their next pass, so such a value is decided once the pass's other masks are known and the opaque mask takes the backslash again, and gaps in tests and wording; the fifth (19 findings, one high; the critic clean) found that projecting a structured result's strings one by one removed the PEM markers and keys that reading its JSON needs across members, so a structured summary reads its JSON as one text again and the summary-only runs over line breaks are withdrawn; the sixth (four findings and a duplicate, one medium; the integration lens and the critic clean) found that the closing view and the summary's closer read a quote's escape after a nested key's value had taken its backslash, and that a structured summary's walk dropped the private-key markers a block across members needs, both regressions against base, and that a structured summary read no JSON-escaped line break or tab, a gap base shares, so a quoted value closes at the later of its readings with and without the masks' trailing backslashes, which also masks the tail an opaque value's mask unescaped (behind a URL userinfo, a further regression the author's oracles found), the walk keeps a marker whose partner lies outside its member, and every summary also reads its escaped separators; two low edge cases base shares went to deferred-tasks.md under the maintainer's 2026-10-21 bar | +| 9 | `6f7dd342` | NONCLEAN (integration clean) | 18 (2 high, 8 medium) | round 8's view comparisons and per-range backslash walks were quadratic at admitted size, so each view is built once and a shared backslash run is read once; a quoted value is also read with the scheme labels' tokens filled, and one that does not close on its line with and without the key rule's own masks; a summary masks a value that one reading stops at a line break through its later closer; a structured summary reads the unwalked JSON once with the walk's whole masks applied in that pass, which replaces round 8's kept PEM markers; a summary reads private-key markers that a line break, a tab or more than one space splits; three low findings went to deferred-tasks.md, and two private-key block gaps the author found afterwards, unchanged since base, are named as limits and recorded there pending the maintainer's confirmation | +| 10 | `59c64498` | NONCLEAN (critic clean) | 11 (7 medium and 4 low as verified; three of the mediums are one finding) | no high or blocker finding; the maintainer counted GEN-R10-1 as high on 2026-10-22 — a `cat .env` result whose password, after a query value at the end of a line, showed in its structured summary, unchanged since base — so a summary opens a key's value whose escaped opening quote ends an earlier mask's backslashes where that mask keeps them, and the journal keeps its reading, so `v5:` digests do not change; the docblocks and the canonical sentence that counted one summary difference, effect-journal.md §11's closing readings and the status records are corrected; the wrapped markers' whitespace bound, the U+E000 placeholder's fallback, a PEM END marker glued to another rule's match, two test gaps and the journal half of GEN-R10-1 are named as limits where a canonical document claimed otherwise and recorded in deferred-tasks.md under the round cap; the two private-key block gaps found after round 9 are approved deferrals | + +**Author gates and controls.** Every corrected head passed `pnpm run ci` and the affected suites +under `CI=true GITHUB_ACTIONS=true`, plus local differential, exhaustive and admitted-size harnesses +that found no new leak against the previous head beyond the classes a round documents (in round 8, +glued credentials that round 7's chain had masked in one pass, and a key that only such a +credential's mask exposes inside a value that does not close). Until round 7 no author gate ran the +evidence export, which only the dedicated CI job enables, so the 110-control family's single export, +about 77 MiB against the unchanged 64-MiB bound, would have failed that job. From round 7 the family +is 120 controls / 360 phases / 501 targets and witnesses, run as two 60-control shards that each +project 1,087 of the unchanged 2,000 retained files; both shards ran locally with +`RELAVIUM_ACCEPTANCE_EVIDENCE` set, and each export held 1,087 files and measured 45.50 MiB and +45.48 MiB, its final record included. From round 8 the family is 130 controls / 390 phases / 513 +targets and witnesses in two shards of 65 controls (1,177 files each), whose exports measured +49.83 MiB and 49.76 MiB (50.19 MiB and 50.09 MiB at the round-9 source, 50.24 MiB and 50.14 MiB at the round-10 source). From round 8 the author also runs the reviewers' own +generative harness (a planted-token oracle, idempotence, JSON round trips, prose preservation, +admitted-size timing and a search for projections that do not settle) and an internal multi-lens +pre-review on every correction before its commit. +[tools/secrets-controls/README.md](../../../tools/secrets-controls/README.md) records each round's +control changes and, from round 5, the rule that pieces inside a mechanism's pattern are pinned by +discriminating tests rather than by new controls. + +**Maintainer ruling — the masker acceptance bar, 2026-10-21.** After eight external rounds and six +internal pre-reviews, the maintainer narrowed this increment's acceptance bar. A pattern-based masker +cannot be complete, and each aggressive round found another corner case while Step 9 waited. A round +is clean for this increment when it reports no high or medium finding and no regression against base +`9902e0a0` (text that base masked and the reviewed head shows, or a guarantee base kept, such as no +throw, linear time at admitted size or a settled projection, that the reviewed head no longer keeps). Each low finding is recorded in +[deferred-tasks.md](../deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22) +with its file, a minimal example and where to re-check it, instead of being corrected in the loop. +The increment was to close with the sixth internal pre-review and one external round (round 9) under +that bar; the round cap below superseded that. The ruling covers only the open-ended masker work: `packages/llm/src/scrub-secrets.ts`, core's +twin projection in `packages/core/src/tools/bounding.ts` and their controls. Every other increment +keeps the approved bar of two NEW consecutive clean complete cumulative rounds after the last +material fix. + +**Maintainer ruling — the round cap, 2026-10-21.** Round 9 reported two high findings, in round 8's +own quadratic view comparisons, and eight medium regressions in five constructed shapes. The maintainer +ruled that round 9's high and medium findings are corrected (its three low findings were recorded under +the bar) and round 10 runs, and that the loop then stops: after round 10, whatever it reports, only its +high and blocker findings are corrected; its medium and low findings are recorded in deferred-tasks.md +with the file, a minimal example and where to re-check it, and the plan moves on to Step 9a.1. + +**Maintainer rulings after round 10, 2026-10-22.** Round 10 reported no high or blocker finding. The +maintainer counted GEN-R10-1 as high, since a `cat .env` run_command or MCP result whose +`POSTGRES_PASSWORD` line follows a URL ending in a query value showed that password in the structured +summary that the `agent:tool_result` event carries into history.db (unchanged since base), and ruled +that it is corrected in the summary only, so that `v5:` journal digests do not change. The maintainer +also approved the two private-key block gaps the author found after round 9 as deferrals. As the round +cap provides, the correction has author verification only: a test with five witnesses and a journal pin +(218 bounding cases), a mutation run of six mutants on the new code that kills four (the two survivors +are output-neutral, because a mask that the key rule's kept view keeps ends before a delimiter, where a +view that keeps the backslashes closes no later than the filled one), the eight controls next to the +changed code (`core-quoted-value-scan` and `core-escaped-quote` re-anchored), the full `pnpm run ci` +with both exporting shards of all 130 controls, and differentials against +`59c64498` and base: the journal and display text identical on 250,000 inputs, every summary that +differs on 320,000 inputs showing a subsequence of `59c64498`'s visible text, and no planted-token +regression against base or `59c64498` in the generators or in an exhaustive run of 837,606 sequences. + +**Closure, 2026-10-22.** The increment closes under the round cap with the round-10 correction. It is +not independently accepted in the sense of the approved bar: round 10 was NONCLEAN, and its correction +was verified by the author alone, as the round cap provides; every finding it did not correct is in +[deferred-tasks.md](../deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). +`development` is pushed and its CI observed, and the PR95 items in +[deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18) are +closed. Step 9a.1 follows. + +**Post-closure CI corrections, 2026-10-22.** `development` was pushed at `9eb38156` (run 39693684654). +Its separate engine coverage floor job, which `pnpm run ci` does not include, failed: under instrumented +coverage, round 9's admitted-size linear-views case ran ten 8-MiB projections in one case and passed its +15 s case deadline on the runner (3.7 s plain and 7.2 s under local coverage). The next push cancelled +that run before the filesystem controls and the second secrets shard finished, and its aggregating +`lint · typecheck · test` job failed at the step that requires the filesystem controls. `2a1f7359` (run +39698522813) makes the case one case per shape with the same inputs, assertions and deadline (1.1–2.8 s +each under local coverage; 221 core projection cases); its coverage job passed, and the local enforced +coverage gate passes at llm 97.79 % lines and 95.69 % branches, mcp 96.56 % and 91.29 %, core (measured) +96.71 % and 91.06 %. In that run the filesystem causal controls failed at export, and the aggregating +job failed with them: each phase retains a manifest of the whole source graph (203,421 bytes once PR95 +grew it), and the 330 manifests of one export passed the unchanged 64-MiB bound by 20,066 bytes, an +overflow already present at `9eb38156` because `2a1f7359` added no source file. `4c3cafe9` (run +39706480450) runs that family as two exporting shards, as PR95's round 7 did for the secrets family +([filesystem controls README](../../../tools/fs-policy-controls/README.md)), and that run passed all 17 +jobs, both shards of each control family, the coverage floor and the four installed-CLI consumers +included. The author now also runs the coverage gate before a push and checks each control family's +export arithmetic. + +### Wave 3 Step 9a.1 implementation checkpoint — 2026-10-22 + +**Scope and source.** Step 9a.1 (the 9a.1 row of the increment table; #46, ADR-0111) implements the +candidate build and finalizer on `development`, on top of `4c3cafe9`, the PR95 closure and its two CI +corrections. It changes +`tools/cli-package` (the owned candidate tree `candidate.mjs`; the producer `produce.mjs` and its entry +point `pack.mjs`; the prepack orchestrator `prepack.mjs`; the finalizer `finalize.mjs`; the seal; the +archive and smoke fold-ins; the interim publication guard `release-guard.mjs` with its rule +`release-guard-core.mjs`; the interrupt, corepack and Git-isolation substrate in `process.mjs`; the local +check `check.mjs`; the consumer lifetime `consume.mjs` and `consumer-lifetime.mjs`; `paths.mjs` and +`production-failure.mjs`), `tools/bundle-closure` (map-aware closure and a shared emitted inventory), +`apps/cli` (the prepack script and the candidate-mode tsup selection), `.github/workflows/release.yml` +(the guard before npm publication) and, through the review rounds, `tools/cli-bootstrap` (stopping the +bootstrap witness's bounded children and removing its live fixtures for an interrupted consumer), the +installed smoke's native fixtures and worker root, the `ci.yml` pack-step comment, the acceptance-tools +list in `package.json` and `tools/ci/require-packed-cli.test.mjs`, with the +[packaging contract](../../reference/cli/packaging.md#candidate-build-and-finalizer-implementation-boundary--step-9a1), +the release runbook, an ADR-0051 amendment, the tooling and smoke READMEs, the CLI reference index and the +`CLAUDE.md`, `AGENTS.md`, `current.md` and `deferred-tasks.md` status lines. It adds no runtime dependency, and +the receipt stays `compiled_package_test`; the contract section is the canonical description of the +mechanism. + +**Before the commit.** An internal read-only pre-review of the prepared change (five lenses, adversarial +verification of every finding and a completeness critic; the detailed record is local) found seventeen +issues, none high once verified. The publication guard refused only when it recognized itself as the +entry point, so a symlinked or otherwise non-canonical path would have let a tag push reach +`npm publish`; the producer's composition had no test; the candidate's file list ignored the developer's +global Git excludes, so a globally ignored private file entered the candidate; and smaller gaps in tests, +modes and wording. All are corrected: the guard's entry point exits 1 before it evaluates its rule, the +producer is a tested module, the candidate reads the developer's global excludes file and nothing else +of the global configuration, the candidate copy's modes are exact under any umask that lets the owner write, and each gap has a +killing test. A mutation run of twenty mutants over the new code kills all of them. The full local CI +then found one more defect in the change itself: `tools/bundle-closure/closure.test.mjs` built its +staged fixture with a top-level `await` after most of its tests were registered, so a slow build let the +runner finish those tests and run the file's `after` hook, which removed the scratch directory a later +test needed (intermittent under load; reproduced deterministically with a delayed fixture). Both +fixtures are now built before the first test is registered, as the file's first fixture already was. + +**Author gates.** `lint:tools` and `typecheck:tools` pass, the 259 cases of the changed tooling tests +pass, and the full `pnpm run ci` on top of `4c3cafe9` passes, both exporting secrets shards included +(65 controls each, 50.39 MiB and 50.28 MiB), with the producer running for real in `smoke:cli-package`. +Under `CI=true GITHUB_ACTIONS=true`, 709 of the 710 acceptance tooling tests pass; the other one, +unchanged by this step, installs through the corepack pnpm shim in an empty home, which needs the +registry, and corepack hung for 45 s on this host while the same tree had passed it in the full run an +hour earlier. + +**Review rounds.** Each round is a NEW independent complete cumulative review of `4c3cafe9..HEAD`, +every reviewer in its own worktree detached at the reviewed head: plan and contract conformance, a +separate security sitting, producer and finalizer correctness (running the real producer), tests and +causal adequacy, integration and documentation, and a completeness critic. Every finding is +adversarially verified; the detailed reports stay local and Git-ignored. + +| Round | Reviewed head | Verdict | Verified findings | Main corrections | +| --- | --- | --- | --- | --- | +| 1 | `a72c7ecd` | NONCLEAN (plan lens clean) | 9 (2 medium, 6 low, 1 nit) | an interrupted producer (SIGINT, SIGTERM, SIGHUP) left its private maps and candidate tree in the temporary directory while its detached build ran on, so the producer now stops the child trees `runCommand` tracks and removes the private root, the build home and the artifact directory's contents before exiting with the signal's code (a producer killed outright is a named limit); private staging must be canonical and outside the candidate by real path; the global-excludes wiring, the producer environment, the placement check's artifact half, the candidate's bounds and three finalizer refusals gained killing tests; the status lines, the CI pack-step comment and the post-closure CI record were corrected | +| 2 | `8ddb7ec2` | NONCLEAN (plan and integration lenses clean) | 12 (4 medium, 7 low, 1 nit; 9 distinct) | the producer removed its interrupt handlers before the final removal, so an interrupt during it ended the process with the private root half removed: the handlers now stay until everything private is gone, and a deterministic test interrupts that removal; SIGHUP, the handlers' removal and a whole build tree (a grandchild in the build's process group) are tested; path containment is decided by segments in one shared helper (`..private` is inside); private staging and the producer's placement compare the real paths and letter case the filesystem stores (`realpathSync.native`); the local `pnpm smoke:cli-package` forwards an interrupt to the producer and the consumer it started (`forwardInterrupts`); the finalizer creates its maps directory with an exact mode and its staged and public modes are tested under a umask that removes the owner's write bit; further finalizer and staged-closure refusals gained killing tests; the ADR-0051 note links ADR-0111 | +| 3 | `184562bd` | NONCLEAN (security lens clean) | 11 (2 medium, 8 low, 1 nit) | round 2 still dropped an interrupt that arrived after the build's last child exited, during the seal or the final removal, because Node delivers a caught signal only when the event loop next polls and the handlers were removed first; round 2's test asserted exit 0 and so pinned the dropped interrupt. Every tool now keeps its handlers until the loop has polled once more, which takes two turns when the work ran from an I/O callback (one turn, the suggested fix, was measured to drop it), and the producer then empties the artifact directory and exits with the signal's code, tested from an I/O-callback tail. An interrupted consumer stops its own and the bootstrap witness's children and removes its installation and fixtures, and the local `pnpm smoke:cli-package`, now tested end to end, starts no later stage once an interrupt arrived. The build's isolated environment, the producer's native real path of the checkout and the prepack catch-all gained killing tests. Git reading the checkout uses no home, global or system configuration, a skip-worktree or assume-unchanged file makes the identity dirty, and an empty upstream mapping is accepted only for `export {};` | +| 4 | `aa5c994a` | NONCLEAN (integration and critic lenses clean) | 10 (3 medium, 5 low, 2 nits) | an interrupted consumer killed the installed smoke's worker before its own removal, so the worker's directory (42 MB) stayed under the report directory and the native fixture target the installed CLI had started in its own process group ran on as an orphan: the fixture's processes now end once the smoke worker that owns their case is gone, and the consumer lifetime, given the report directory as a required `supportRoot`, abandons the bootstrap witness's children and fixtures and the smoke's worker roots itself before removing the installation. The local check's ordinary failures, its successful summary and its hand-off of the store and excludes variables, a dirty identity's path to the seal, the producer's removal of engine build state and the bounded children's process group gained killing tests. "Owner-only" is qualified where POSIX modes are not observable (Windows); the candidate copy's umask wording and the checkpoint's scope list were corrected | +| 5 | `d51a2437` | NONCLEAN (plan, correctness and integration lenses clean) | 7 (1 medium, 6 low) | `prepareCandidate`'s link verification had no composed test, so deleting the call passed: a preparation whose install resolves an engine package or the root dependency directory to the original checkout now refuses after the install. A tracked directory replaced by an ignored link let the copy read bytes from outside the checkout through the linked parent, so every parent directory below the checkout must now be an ordinary directory; a `core.excludesFile` that is present but empty disables Git's global excludes and no longer falls back to the default ignore file. Settled children leaving both live-child sets, the exact `export {};` rule, each field of a facade's empty map, an interrupt the consumer outlives and the smoke-root prefix limit gained killing tests | +| 6 | `3c6c5dce` | NONCLEAN (plan and correctness lenses clean) | 6 (1 medium, 4 low, 1 partial nit; 4 distinct) | nothing pinned that a leader-exit descendant watches the smoke worker rather than its own parent, so a descendant watching its exited parent would have ended on its own and left the product's group reap unproven: a test now keeps the descendant alive past its parent's exit until the owner is killed. The staging directory's ownership check gained a killing test (a different `process.getuid`); every wait for the local check's exit is bounded; and five live status statements that still said Steps 9–19 had not started now say that Step 9a.1 is under review | +| 7 | `0e59f5ff` | NONCLEAN (plan, security, correctness and integration lenses clean) | 3 (3 low) | a replace ref on HEAD let Git's status compare the working tree with a substituted commit, so the identity recorded the real HEAD as clean over other bytes: Git reading the checkout now ignores replace refs (`GIT_NO_REPLACE_OBJECTS`), with a real-repository test. The linked-parent refusal is tested for links above the listed file's own directory and below the root, and the fixture owner tests bound every wait and clean up in an after-hook, so a regression fails within the test and leaves no process | +| 8 | `c691ef9f` | NONCLEAN (tests lens only; plan, security, correctness, integration and critic clean) | 4 (2 low, 2 nits) | two of the finalizer's pre-write refusals (a runtime output that is not a flat `dist/.js`, and one whose own map entry was swapped for another) and the excludes lookup's forwarded configuration-location variables had no killing test; they now do, with real-Git cases for an excludes file set only in an XDG configuration directory or a `GIT_CONFIG_GLOBAL` file. The non-material gaps were closed too: the upstream map's name type, a metafile without outputs or inputs, the metafile and map byte bounds, the pinned pnpm version, the producer's exact modes under a restrictive umask and the reference index row | +| 9 | `08f296cd` | NONCLEAN (plan, security, correctness and integration lenses clean) | 4 (3 low, 1 nit) | a failed production printed its build children's captured output word for word, and esbuild quotes a malformed upstream map in its warning, contrary to the rule that maps are excluded from every diagnostic: the producer's entry point now prints only prepack's own fixed `✗ CLI …` lines and one fixed code (`production-failure.mjs`), tested through the real entry point with a quoted-map marker. The local check's first stage guard gained a killing test (an interrupt during its own discovery), and the real-checkout excludes test no longer reads the host's system Git configuration | +| 10 | `954d6a62` | NONCLEAN (plan, correctness, integration and critic clean) | 4 (3 low, 1 nit) | with a corepack-managed pnpm, the producer's pnpm ran from the isolated build home under the shared temporary directory, so its optional module lookups reached a `node_modules` there that another local user could create (present since Step 7b; shared-`/tmp` POSIX hosts only, never CI): isolated children now carry the developer's own corepack cache (`COREPACK_HOME`, computed as corepack computes it), and a test plants such a module and shows it never runs. A failed removal during an interrupt escaped the signal handler with a raw stack and exit 1: each cleanup step is now attempted on its own, a failure leaves one fixed line, the exit code stays the signal's, and an ordinary failed removal no longer skips the other directory. A source root's sibling sharing its prefix gained a killing test, and an untracked nested repository refuses under its own code | +| 11 | `d890abf0` | NONCLEAN (plan, security, correctness and integration lenses clean) | 4 (1 medium, 3 low) | nothing pinned that the producer looks up the developer's global excludes in the caller's own environment rather than the isolated build environment, which would have dropped them silently: the composition test now requires the very environment object. The local check's deadline killed only a stage's own process group, so a producer's detached build child could recreate the removed root with a map: a stage past its deadline is now interrupted first (`runCommand`'s `graceMs`) and killed only after a grace period, tested with a late-writing build child. The check's wait for an interrupt caught during its own root removal gained a killing test, and `dependencyStore`'s docblock says where corepack runs | +| 12 | `78623964` | NONCLEAN (plan, security and correctness lenses clean) | 7 (1 medium, 2 low, 4 nits; 5 distinct) | Git reading the checkout got Node's null device, `\\.\nul` on Windows, which Git for Windows refuses (it maps only `nul` and the literal `/dev/null`), so every isolated Git call would exit on a Windows host — the local producer and a local consumer; the CI producer runs on Linux and every CI consumer names its source: it now gets the literal `/dev/null`. The consumer stage's deadline grace and a bounded grace test gained killing tests, and the store discovery's exception is stated. The author gate statements of rounds 3–11 said the filesystem and secrets families' captured sources were unchanged, but both capture the root `package.json`, which round 3 changed (one acceptance-tools entry); this round runs the full `pnpm run ci`, every family included | +| 13 | `cf97b7f1` | **ACCEPTED** under the 2026-10-23 bar (security sitting, tests and integration lenses clean; no blocker, high or medium finding and no regression) | 10 (3 low, 7 nits; 6 distinct) | none in the loop. The contract's list of the caller variables that reach store discovery omitted `NPM_CONFIG_STORE_DIR`, which the code forwards (low, reported by four lenses), and three wording nits (the deadline's group kill, the guard's exit code, deferred-tasks.md's freshness line) are documentation only and corrected in the acceptance commit; `forwardInterrupts`' group-wide forwarding has no test (low) and the map-source bound is pinned on its admitted side only (nit), both recorded in deferred-tasks.md | + +**Internal rigor pass, 2026-10-24.** After the twelfth round the maintainer asked for maximum care before +any further round, so an internal read-only pass covered the whole increment at `1f247772` before round +13: five mutation sweeps (the candidate and paths, the finalizer, the producer and seal, the process and +interrupt lifetime, and the closure and guard; each mutant applied in place, run in its own process group +and restored by digest) and claims, platform and security lenses; the detailed record is local. It +reported 84 raw findings (6 medium, 69 low, 9 nits). Code defects corrected: past its deadline, a group +member that ignored SIGTERM outlived a leader that exited on it, so the group is now killed when the leader +closes inside the grace; the excludes lookup ran Git from the checkout when the caller had no `PATH`, and +missed command-scope `GIT_CONFIG_*` configuration; Git reading the checkout still read the system +attributes and, on Windows, default attribute and ignore locations under the null-device home, so it now +sets `GIT_ATTR_NOSYSTEM` and names both files; the store discovery dropped `npm_config_store_dir`; the +candidate copy treated a group or other execute bit as executable, unlike Git; the consumer compared its +placement by real paths that keep the given letter case; the producer removed the private maps last on an +interrupt, and an ordinary failure let a removal error replace the primary refusal and left a refused +tarball in the artifact directory; two finalizer refusals echoed an unvalidated metafile string as their +subject; and one bootstrap fixture that could not be removed stopped the others' removal. Each test gap +gained a killing test, among them a composed `sealPackage` run on a candidate that is not the repository, +a behavioural evaluation of the tsup configuration, exact finalizer bounds and refusal subjects, the +release guard's handling of a rule that returns, exits or throws a foreign error, the publish job's exact +keys, and the completeness of the acceptance-tools list. The 260 mutants that had survived the five sweeps, +and new mutants over the corrections, were then run against the corrected tests. All are killed except the +ones the sweeps judged equivalent or defensive and the consumer and worker composition lines that no node +test reaches by design; two of the old survivors needed further killing tests first (the producer's default +temporary directory, and a deadline interrupt that reaches every member of the process group, not only its +leader). One of those runs left a test's detached build stand-in behind under a mutant that hung the test, +so every wait for a spawned tool's exit in the interrupt tests is now bounded and a regressed tool fails its +test instead. Five low findings that need a change of scope are recorded in +[deferred-tasks.md](../deferred-tasks.md#wave-3-step-9a1--recorded-low-findings-2026-10-24). + +**Remaining gates.** None for acceptance: see the acceptance record below. `development` is pushed and +its CI observed after the acceptance commit. + +### Wave 3 Step 9a.1 acceptance — 2026-10-24 + +**Step 9a.1 is independently accepted on `development` at `cf97b7f1` (2026-10-24)** under the Acceptance +row as amended on 2026-10-23. Round 13, a NEW independent complete cumulative review of +`4c3cafe9..cf97b7f1` with every reviewer in its own worktree detached at that head, reported no blocker, +high or medium finding and no regression, and its separate security sitting, run at the same head as its +own reviewer and report, was clean. The plan reviewer ran the real producer at the reviewed head: it exited +0 with a `compiled_package_test` receipt, and the tarball held no map, map link or `sourcesContent`. The +round's ten verified findings (three low, seven nits; six distinct) are recorded in +[deferred-tasks.md](../deferred-tasks.md#wave-3-step-9a1--recorded-low-findings-2026-10-24). Those that +concern documentation only — the contract's store-discovery list, the deadline's group-kill wording, the +guard's exit-code wording and deferred-tasks.md's freshness line — are corrected in the acceptance commit, +which changes no code; the two test gaps stay open there. The PR #95 tooling nits owned by 9a.1 (the +unreachable `TAR_FORMAT` branches and the smoke worker's ancestor loop) are accepted with it; that item's +duplicate headings stay with 19b. Author evidence at `cf97b7f1`: the full `pnpm run ci` with every +causal-control family, and the acceptance tools (835 of 835) with and without `CI=true GITHUB_ACTIONS=true`. +The receipt stays `compiled_package_test`; Step 9a.2 is next. + +**Push CI.** `development` was pushed at `b0579580` (the acceptance commit on top of `cf97b7f1`). Every +required job of [run 40050981969](https://github.com/HodeTech/Relavium/actions/runs/40050981969) passed: the +originating pack, the four same-tarball installed-CLI consumers (current Linux, macOS and Windows and Node +22.0.0 Linux) and their aggregate, lint, typecheck and test, the engine coverage floor and the causal-control +families. The advisory Windows job failed: one `packages/db` test timed out at vitest's default 5 s on that +runner (7.7 s). 9a.1 changed no database or CLI source; the cause and the runner's history are recorded in +[deferred-tasks.md](../deferred-tasks.md#wave-3-step-9a1--recorded-low-findings-2026-10-24). + +### Wave 3 Step 9a.2 preflight decisions — 2026-10-24 + +A read-only preflight (five readers and a synthesis; the detailed record and a Turkish decision register are +local and Git-ignored) mapped every place Step 9a.2 touches before any code was written. The maintainer +approved each recommendation on 2026-10-24; no new ADR or runtime dependency follows. + +| Decision | Ruling | +| --- | --- | +| Split | 9a.2 becomes 9a.2a (decoder, finalizer hardening, private manifest, frame capture and symbolication before the receipt write; the receipt stays `compiled_package_test`) and 9a.2b (`check:map-controls`, the required uncached CI job and its pins, then the atomic `package_test` flip). `package_test` is never claimed before its negative controls exist. | +| Frames | "Actual packed CLI and engine frames" are captured by a tooling-only `node --import` preload while the packed CLI runs an offline `relavium import` of a canary document: the preload wraps the external `yaml` package's `parse` and records one synchronous stack of structured call sites for that document alone, crossing the CLI and `@relavium/core`. Engine coverage is `@relavium/core` (at least two core and three CLI anchors). No product surface is added and no native module is loaded. | +| Where they run | The finalized `apps/cli/dist`, which the archive inspection has proven byte-equal to the tarball entry by entry; each anchor frame's file is re-hashed at capture against the manifest and the tarball inventory. | +| Manifest | The private manifest also carries a per-source digest inventory (repository-relative path and SHA-256 of every original source a final map names; no content) and the commit's Git tree object id with the dirty flag. This reconciles ADR-0111's "private per-source manifest" with packaging.md's field list; per-source inventories stay out of every public receipt. | +| Source revision | A clean candidate's anchor sources are read from the original checkout's Git objects at the recorded commit through isolated Git and must equal the manifest's digests; a dirty candidate (local only — a GitHub origin with a dirty source is already refused) uses the candidate copy's bytes, with `dirty` recorded. | +| Post-receipt rebuild | ADR-0111's "post-receipt rebuild fail" is closed in 9a.2b for the package-test producer: a second prepack into sealed staging, a re-seal into the artifact directory and a rebuilt `dist` that differs from the manifest are refused. The release-mode no-rebuild retry rules stay with 9c and 9d. | + +Engineering defaults recorded with the rulings: an in-house strict VLQ decoder in tooling, with Node's +built-in `node:module` source map used only as an independent test oracle; anchor frames must land on an +exact mapping segment, and the expected 1-based line and column come from the source revision's TypeScript +syntax tree, never from a pinned position table; the finalizer reuses the decoder for strict mapping and range +checks; the manifest is a separate owner-only `staging/manifest.json`, removed with the staging; the +`package_test` receipt keeps version 1 and its fields; consumers and the aggregate verify identity through the +shared validator and change no schema; `check:map-controls` is a standalone runner in `tools/cli-package`, +outside the causal source roots; a tag push now produces a package-test artifact that the interim guard refuses. +The inert `--verbose` raw-stack branch the preflight found is recorded for Step 10a, which replaces it. + +### Wave 3 Step 9a.2a implementation checkpoint — 2026-10-24 + +**Scope and source.** Step 9a.2a (the 9a.2a row; #46, ADR-0111) implements, on `development` after the 9a.1 +acceptance and the 9a.2 preflight record (`6277e375`), the strict source-map decoder (`tools/cli-package/source-map.mjs`), +the finalizer's strict mapping and range checks (`finalize.mjs`), the private manifest (`manifest.mjs`), the +tooling-only frame-capture preload with its keyring stub (`frame-capture.mjs`, `frame-capture-hooks.mjs`, +`frame-capture-keyring.mjs`), symbolication against the matching source revision (`symbolicate.mjs`), the seal's +composition (inspection → manifest → symbolication → receipt, `seal.mjs`), the producer's private work directory +(`produce.mjs`) and the entry point's summary (`pack.mjs`), with their tests and the +[packaging contract](../../reference/cli/packaging.md#private-manifest-and-symbolication-implementation-boundary--step-9a2a), +the tooling README and the status lines. No runtime dependency is added, Node's own source map decoder is only a +test oracle, and the receipt stays `compiled_package_test`. On the real product the decoder agrees with that +oracle at all 245,806 final-map and 330,759 upstream-map segment starts, every position lies inside its text, and +the real producer seals a 25-output, 404-source manifest and reaches all five anchors in about a second. + +**Internal rigor pass, 2026-10-24.** Under the maintainer's rigor-before-review rule, a read-only pass covered +the whole increment at `1ab4c134` before its first round: claims, platform, security, integration and critic +lenses and three mutation sweeps (the decoder and finalizer, the manifest, seal and producer, and symbolication and +the preload); the detailed record is local. It reported two high findings. On a clean checkout with converted +line endings (`core.autocrlf`, the Git for Windows default) Git status stays clean while the copy holds CRLF bytes, +so every clean production refused `SOURCE_REVISION`: candidate preparation now confirms a clean status byte for +byte, comparing each copy's Git blob id with the commit tree's entry, and records a differing copy as `dirty`. And +`relavium import --cwd` walks up for a project, so an ancestor `.relavium/` of the temporary directory (a +developer's real project on Windows) could receive the canary: the capture now creates the project's own +`.relavium/` first and refuses `CAPTURE_PROJECT` unless the canary landed there. Medium findings corrected: the +manifest inputs missed the configurations the engine build configurations extend; the two-core, three-CLI anchor +ruling was enforced by nothing; zero-payload VLQ padding decoded as NaN, which passed every comparison, so an eighth +digit now refuses; and many claimed guarantees had no killing test (a new `manifest.test.mjs`, a rewritten +`symbolicate.test.mjs` on a shared test-only fixture whose CLI walks up for its project like the real one, and +extended decoder, finalizer, seal and candidate tests). Lows corrected include a native-addon refusal in the +preload beside the keyring stand-in, which now exports every export the package declares, owners without a body +no longer counted as anchor owners, and the Git, re-hash, README and status wording. Two lows are recorded in +[deferred-tasks.md](../deferred-tasks.md#wave-3-step-9a2a--recorded-low-findings-2026-10-24): the capture child is not +sandboxed (also a named limit in the contract), and two packs of one source can order the packed manifest's +`@relavium/*` dependencies differently, so their digests differ (for Step 9c). All 554 mutants of the three sweeps +and of the corrections were then run against the corrected tests: 509 are killed, 4 targeted code the pass +changed and are replaced by new mutants, and the 41 survivors are equivalent or defensive (unreachable defaults, +caches, a redundant clause, parser options, constants and the binary search's arithmetic). + +**Author evidence at `fd6cd1cd`.** The full `pnpm run ci` (exit 0, about 70 minutes, with the real producer +and consumer smoke and every causal-control family) and `test:acceptance-tools` (881 of 881, plain and under +`CI=true GITHUB_ACTIONS=true`); on the clean tree, the package smoke and a direct production whose receipt +records `dirty: false` and whose anchors were read from the commit's Git objects. + +**Round 1 at `fd6cd1cd` — NONCLEAN.** Five lenses and a critic, every finding adversarially verified. The security +sitting reported one confirmed medium (SEC-9A2A-R1-1): the capture child ran the candidate CLI from the producer's +root in the shared temporary directory, so ws's optional `require('bufferutil')` resolved a module another local +user could plant in `/node_modules`, which ran beside the private staging while the production still +succeeded. Corrected: the preload registers a synchronous resolution hook (`module.registerHooks`, which sees +`require` as well as `import`) that refuses any module resolving outside the candidate; a test plants such a module +and shows that Node loads it without the preload and refuses it, by `require` and by `import`, with it. The test +fixture now installs `yaml` inside the candidate in pnpm's layout. Lows corrected in the same commit: the anchor +ruling counts distinct call sites (PLAN-9A2A-R1-1 = DMCS-9A2A-R1-1); the contract's 9a.2a section carries the +Windows caveat of "owner-only" (PLAN-9A2A-R1-2); the manifest admits exactly the package versions a receipt admits, +through one shared rule (CRIT-9A2A-R1-1); the triggered 9a.1 PLAT-9 deferral records its re-check (CRIT-9A2A-R1-2); +and this checkpoint records the author's CI evidence (CRIT-9A2A-R1-3). Of thirteen mutants over the corrections, ten +are killed and three are defensive: a prefix-sibling comparison that no resolution path reaches, a real-path call +on paths the resolver already returns real, and the hook API check on a Node that has the API. + +**Round 2 at `990a5d16` — not accepted.** The plan lens was clean; the round reported no blocker, high or medium +finding, and the security sitting one low. But three lenses independently found a regression the round-1 +correction introduced (DMCS-9A2A-R2-1 = TCA-9A2A-R2-1 = INT-9A2A-R2-1, rated low): `module.registerHooks` exists +from Node 22.15, the workspace admits Node 22.13, and on 22.13 or 22.14 every local production now refused with the +generic `CAPTURE_FAILED` code, because the preload's named import failed at link time and its guard never ran. +Since the amended bar requires no regression, the increment is not accepted at round 2. Corrected: the producer +refuses an unsupported Node with `PACKED_CLI_PRODUCER_NODE` before anything exists (`captureSupported`), the +preload reads the API from the module object so its own guard runs, and the contract and README name the minimum. +The security sitting's low (SEC-9A2A-R2-1) is corrected as wording: the keyring, native-addon and module refusals +hold in the capture process itself, not in a thread or subprocess the candidate starts without the preload; the +contract, the preload's docblock and the SEC-9A2A-2 record say so, and that record notes its re-check +(CRIT-9A2A-R2-1). + +Round 3 accepted the increment; see the acceptance below. + +### Wave 3 Step 9a.2a acceptance — 2026-10-24 + +**Step 9a.2a is independently accepted on `development` at `a92a7602` (2026-10-24)** under the Acceptance row as +amended on 2026-10-23. Round 3, a NEW independent complete cumulative review of `6277e375..a92a7602` with every +reviewer in its own worktree detached at that head, reported no blocker, high or medium finding and no +regression, and its separate security sitting, run at the same head as its own reviewer and report, was clean; +so were the plan, correctness and integration lenses and the critic. The plan reviewer ran the real producer at +the reviewed head: it exited 0 with a `compiled_package_test` receipt and `dirty: false`, the receipt's keys +were unchanged, neither the receipt nor the tarball held a map, the manifest, a frame, the per-source inventory or +a host path, and the sealed manifest (25 outputs, 404 sources) and the symbolication of all five anchors (about a +second) matched this checkpoint. The round's three verified findings (one low, two nits) are recorded in +[deferred-tasks.md](../deferred-tasks.md#wave-3-step-9a2a--recorded-low-findings-2026-10-24); the wording nit is +corrected in the acceptance commit, which changes no code, and the two test gaps stay open there. The receipt +stays `compiled_package_test`; Step 9a.2b is next. + +**Push CI.** `development` was pushed at `f65999a6` (the acceptance commit on top of `a92a7602`). All fourteen +required jobs of [run 40105286543](https://github.com/HodeTech/Relavium/actions/runs/40105286543) passed, among them +the originating pack, the four same-tarball installed-CLI consumers and their aggregate, lint, typecheck and test, +the engine coverage floor and the causal-control families. The advisory Windows job failed on a security assertion +unrelated to 9a.2a: `apps/cli/src/engine/tool-host/fs-policy.test.ts`, "revokes a selected subdirectory +replacement even when its enclosing grant is unchanged", read the replacement instead of refusing it. It was the +first failure of that test in the last sixty CI runs. The cause is in the code: the filesystem policy pins a +directory by `dev` and `ino` read without `{ bigint: true }`, and on Windows `ino` is a 64-bit NTFS file ID that a +JavaScript number above 2^53 cannot hold exactly, so a directory created next to the renamed original can round to +the same identity. Four other identity checks share the pattern. The maintainer approved a separate increment for +them on 2026-10-24, after 9a.2b and before 9b (row NI; +[deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24)). + +### Wave 3 Step 9a.2b preflight decisions — 2026-10-24 + +A read-only preflight (five readers and a synthesis at `f65999a6`; the detailed record and a Turkish decision +register are local and Git-ignored) mapped every place Step 9a.2b touches. The maintainer approved each +recommendation on 2026-10-24; no new ADR or runtime dependency follows. The earlier 9a.2 rulings stand: the +three post-receipt rebuild refusals, the receipt keeping version 1 and its twelve fields with only `mode` changing +to `package_test` and `compiled_package_test` refused, the standalone runner `tools/cli-package/map-controls.mjs`, +and a `map-controls` CI job shaped like `pack-cli` (uncached, no upload, at most 20 minutes, inside `ci`'s needs +with a success guard; the required contexts stay `ci` and `coverage`). + +| Decision | Ruling | +| --- | --- | +| Sealed product | One real `producePackage` run, driven through its existing ports: the controls run inside the seal callback while the private staging still exists, mutate the owned candidate and staging copies in place and restore each file by digest; every private copy stays under the staging, which is removed first on success, failure and interrupt. `produce.mjs`'s cleanup and interrupt handling stay the one owner. | +| Re-seal | `sealPackage` refuses with `PACKED_CLI_ALREADY_SEALED` at its entry, before the inspection and any capture, when the artifact directory holds anything besides the one tarball or the staging already holds a manifest; a receipt-write race maps to the same code. | +| Rebuilt `dist` | A new `verifySealedDist` compares the whole `dist` inventory, every output's digest and the tarball's digest with the in-memory manifest (`PACKED_CLI_MANIFEST_DIST_DRIFT`, `PACKED_CLI_MANIFEST_TARBALL_CHANGED`); the seal calls it immediately before the receipt write, and the control calls it after a real rebuild. | +| Release workflow | `release.yml` gets no map-controls job: the interim guard refuses every release-workflow artifact until 9d and Wave 4b, and 9d owns the workflow graph. A manual dry-run's `package_test` artifact is backed by the required `ci` run at the same commit, and the contract says so. | +| Increments | One increment with ordered commits — the seal refusals, the runner, the CI job and its pins, then the flip, then the documents — under one acceptance gate and one security sitting; the flip never exists in a tree without the controls before it. | + +Engineering defaults recorded with the rulings: each control asserts its exact refusal code and its stage markers +(so a refusal from the wrong layer cannot pass); wrong source runs both clean (`SOURCE_REVISION`) and dirty +(`ANCHOR_UNMAPPED`); truncation covers a byte cut and a cut inside a VLQ value; the JS and map controls run on every +anchor-bearing chunk; permuted sources rotate with no fixed point; the shifted upstream map is checked to be +strictly valid before use; the runner prints one JSON summary with codes and outcomes only — no path, frame, map +byte or child output — and compares bytes by digest; it runs in `pnpm run ci` right after `smoke:cli-package`. + +### Wave 3 Step 9a.2b implementation checkpoint — 2026-10-24 + +**Scope and source.** Step 9a.2b (the 9a.2b row; #46, ADR-0111) implements, on `development` after the 9a.2a +acceptance and the 9a.2b preflight decisions (`436d7c67`), four ordered commits and this record: the seal-side +rebuild refusals (`PACKED_CLI_ALREADY_SEALED` at `sealPackage`'s entry and `verifySealedDist` before the +receipt, `seal.mjs` and `manifest.mjs`, with a prepack test of a second prepack into a sealed staging); the +product map controls runner (`tools/cli-package/map-controls.mjs`, `pnpm check:map-controls`, with tests of its +pure helpers); the required `map-controls` CI job inside `ci`'s needs with its success guard, `pnpm run ci`'s +step after `smoke:cli-package` and the workflow pins; and only then the `package_test` flip (one +`PACKAGE_TEST_MODE` shared by the seal and both receipt validators, `compiled_package_test` refused), with the +[packaging contract](../../reference/cli/packaging.md#product-map-controls-and-the-package-test-receipt--step-9a2b), +the tooling README and the status lines. No runtime dependency or ADR is added; `release.yml` gets no map +controls job, as ruled. At the implementation head (`77ee5d0a`) the runner's 24 controls passed in about 30 seconds +on one real production, `smoke:cli-package` passed with the flipped receipt through the real producer and +consumers, and `test:acceptance-tools` passed 911 of 911; after the rigor pass the runner records 25 (the +seventeen fixed controls plus four for each of the two anchor-bearing chunks) and the suite runs 932 tests. + +**Internal rigor pass, 2026-10-24.** Under the rigor-before-review rule a pass covered the whole increment at +`77ee5d0a` before its first round: three mutation sweeps (the seal refusals and the flip; the runner, including +removing each production refusal a control is meant to prove and running the real runner; the CI pins) and +claims, platform, security and integration lenses with a completeness critic; the detailed record is local. It +confirmed one high finding: the re-seal-receipt control re-sealed against the sealed staging, so the staging half +of `PACKED_CLI_ALREADY_SEALED` refused for it and the artifact half it claimed to prove could be removed with the +runner still passing. Corrected: it now re-seals against a copy of the staging without its manifest. Medium +findings corrected: nothing tested the seal's default `verify` port (a seal test now runs the real check, and a new +`seal-verify` control changes a chunk during the seal's own run); the runner accepted whatever controls ran and +its exit gate was untested (it now requires the seventeen fixed controls plus four per anchor-bearing chunk, each +exactly once, through a tested `verdict` and `report`); its entry guard compared two differently resolved paths +and could exit 0 on Windows without running (the entry point is now an unconditional `map-controls.mjs` over +`map-controls-core.mjs`); the per-chunk controls were not checked to corrupt their own chunk in their own way (each +re-recorded copy must now differ from the sealed record in exactly its chunk, and a line or column shift must +change the line count as it names); the workflow pins did not pin the step shells, the job environment or several +properties' negative cases (they do now); and the contract's statement that a manual dry-run's `package_test` +artifact is backed by the required `ci` run at the same commit, recorded in the preflight table above, did not hold +for every dispatch — corrected in the contract and the workflow comments, with Step 9d's binding to a successful +`ci` run recorded in deferred-tasks.md. Lows corrected include an early refusal of a half-configured CI +environment, removing a refused control build's `dist`, tests of the already-sealed refusal's order, of a failed +receipt write, of the version's start anchor and of `verifySealedDist`'s every output and read bounds, and the +summary, checkout and already-sealed wording. Four lows, and one medium for Step 9d, are recorded in +[deferred-tasks.md](../deferred-tasks.md#wave-3-step-9a2b--recorded-low-findings-2026-10-24). The five mutants the +corrections target now fail the real runner, each on its own control or self-check. + +**Author evidence at `3e6ae33c`.** The full `pnpm run ci` (exit 0, about 73 minutes, with `check:map-controls` +recording 25 of 25 expected controls on a clean tree, the real producer and consumer smoke and every causal-control +family). + +**Round 1 at `3e6ae33c` — NONCLEAN.** Five lenses and a critic, every finding adversarially verified. Four lenses +reported a regression in the author's own record (medium): commit `9f8b0890`, which recorded the native-identity +increment, replaced the start of deferred-tasks.md's "Phase 2.6.5 deferrals" heading instead of inserting before it, +so the approved CR-70, CR-93 and CR-95 deferrals fell under the new section; the heading is restored exactly, and an +audit of every line any 9a.2b commit removed found no other loss. The tests lens reported that nothing tested the +entry point's exit code (medium): a test now runs the real `map-controls.mjs` beside a stand-in core and requires +exit 0 for a passing run, exit 1 naming the control for a failed or missing one, and exit 1 with only the fixed code +for a run that throws. Lows corrected: the contract's earlier sentence that the same commit's `ci` runs the controls +now says when an artifact is backed; the runner no longer aborts without a summary when a refusal breaks (an +`aborted` entry and every missing control are reported, and the broken control is named); every refused control +build that ran the bundler has its `dist` removed; the seventeen fixed controls are pinned in a test; the "restores +the file" wording excepts `dist`; the checkpoint's counts (25 controls, 932 tests, four lows and one medium) and the +9a.2a push-CI job count (fourteen required jobs) are corrected; and the status lines name the NI increment. + +Round 2 accepted the increment; see the acceptance below. + +### Wave 3 Step 9a.2b acceptance — 2026-10-24 + +**Step 9a.2b is independently accepted on `development` at `0f73c253` (2026-10-24)** under the Acceptance row as +amended on 2026-10-23. Round 2, a NEW independent complete cumulative review of `436d7c67..0f73c253` with every +reviewer in its own worktree detached at that head, reported no blocker, high or medium finding and no regression; +its separate security sitting, run at the same head as its own reviewer and report, reported one low; the plan and +correctness lenses were clean. The plan reviewer checked the commit order (the flip never exists in a tree without +the controls, the required job and its guard before it), ran the real runner at the reviewed head (25 of 25 +expected controls on a clean tree) and showed that removing the artifact half of `PACKED_CLI_ALREADY_SEALED` fails +both re-seal controls. The round's seven verified findings (five low, two nits) are recorded in +[deferred-tasks.md](../deferred-tasks.md#wave-3-step-9a2b--recorded-low-findings-2026-10-24) or corrected in the +acceptance commit, which changes no code: the plan's increment count now includes NI, deferred-tasks.md's freshness +line names it, and the contract and the interrupted-runner record say that directory can still hold the baseline +public tarball and receipt. The receipt now records `package_test`; the native-identity increment NI is next, then +Step 9b. + +**Push CI.** `development` was pushed at `10993bb8` (the acceptance commit on top of `0f73c253`), after a full local +`pnpm run ci` at that head passed with all 25 map controls. All fifteen required jobs of +[run 40137915608](https://github.com/HodeTech/Relavium/actions/runs/40137915608) passed, among them the first CI run +of the required product map controls job, the originating pack, the four same-tarball installed-CLI consumers and +their aggregate, lint, typecheck and test, the engine coverage floor and the causal-control families. The three +advisory jobs passed too, including the Windows job whose earlier failure opened NI. + +### Wave 3 NI preflight decisions — 2026-10-25 + +A read-only preflight (three readers and a synthesis at `10993bb8`; the detailed record and a Turkish decision +register are local and Git-ignored) mapped every file and directory identity comparison in the tree. The maintainer +approved each recommendation on 2026-10-25; no new ADR or runtime dependency follows, and ADR-0105 and ADR-0106 +receive dated correction notes after the precedent of ADR-0106's 2026-10-13 exact temporary claim identity +correction. The sites are the row's five files: the filesystem policy's canonical and lexical root pins; the +workflow catalog's open check, the only swap guard on Windows, where `O_NOFOLLOW` is undefined; and the media +store's root pin, its file snapshot and owned-file check (the authority to delete), its read check, its publish +handle and the temporary sweep's claim directory, child, marker and moved-blob checks with the `Number` cross-check +inside `nativeFile`. One test precondition reads number identities as well. No persisted format changes: +`media-claim-v2` names already carry exact base-36 tokens, and the v1 reader admits only safe tokens, whose bigint +form is identical. + +Three facts were verified after the preflight. Node 22 fills bigint stats from a signed `BigInt64Array`, so a 64-bit +file ID with its top bit set (an NTFS sequence number of 0x8000 or more) reads negative; exact equality is +unaffected, but the claim token refused it and such a temporary was retained forever. The defect is not +Windows-only: Linux overlayfs's `xino` feature "uses the high inode number bits for fsid" (kernel documentation). +And on ReFS, including Dev Drive, Microsoft documents that the 64-bit file identifier "is not guaranteed to be +unique" (ReFS uses 128-bit identifiers). + +| Decision | Ruling | +| --- | --- | +| Snapshot shape | The media store reads each identity with one bigint stat and projects it once into an internal record: bigint `dev` and `ino`, number `nlink`, `size` and `mtimeMs` (from `mtimeNs`, so today's sub-millisecond values stay), and the type predicates of the same stat. A path is never stat'ed twice for one snapshot. The CLI sites keep `BigIntStats`, since no other field there meets an identity check. | +| Permanent proof | Five filesystem causal controls, one per site: the policy pins, the catalog open check, the root pin, the owned-file check and the temporary sweep's `same`. Each mutant returns that site's whole comparison to `Number` and is witnessed by a real replacement under injected identities above 2^60, where adjacent numbers are 256 apart. The read check, the `nativeFile` cross-check and the per-read variants are covered by the recorded mutation sweep and the type checks. | +| Windows evidence | Acceptance rests on the deterministic proof. `fs-policy.test.ts` gains a Windows-only probe (about 200 rename-and-`mkdir` pairs, each distinct as bigints, logging how many collide as numbers or read negative) and `{ repeats: 20 }` on its real replacement tests. The Windows clause closes after three consecutive green advisory Windows runs on pushed heads, with the logged counts recorded; the existing Windows steps are unchanged. | +| Tooling sites | The three same-class tooling checks are fixed in NI after the `tools/cli-package/files.mjs` pattern, each with an injected-identity check: the overflow capture's destination check, the budget-replay evidence retention's directory identity (it deletes recursively) and that tool's lifecycle-smoke precondition. | +| Read check | Converted for uniformity. Its second-stat comparison is tested with a faked second stat and recorded as a consistency check that a real filesystem cannot fail; its `dev` comparisons are exact on every platform. No permanent control. | +| Sign and ReFS | `nativeIdentityToken` maps each value with `BigInt.asUintN(64, …)`, in one place: tokens compare with tokens, and every value without its top bit set keeps a byte-identical token, so existing names stay compatible. A test shows that a negative identity can be claimed. ReFS is recorded in deferred-tasks.md as a scoped limit, since a fix needs native code or a dependency. The claim becomes exact identity on filesystems whose 64-bit file indexes are unique, such as NTFS, ext4 and APFS. | + +Engineering defaults recorded with the rulings: +- No identity helper is shared across packages, and the shards get no pin. +- `nativeFile` keeps its second `lstat`. +- The temporary sweep's token comparison line stays byte-identical, since its precision control anchors on it. +- New witnesses go into new test files, so no pinned count moves. +- `CAS_COOPERATIVE_READ` is the one anchor that drifts; it is updated in the same commit. +- No identity value enters an error, event, log or JSON. +- ADR-0105's cooperative-parent assumption is unchanged. + +### Wave 3 NI implementation checkpoint — 2026-10-25 + +**Scope and source.** The native-identity increment (register row NI; ADR-0105, ADR-0106) is implemented on +`development` after the 9a.2b acceptance and the [NI preflight decisions](#wave-3-ni-preflight-decisions--2026-10-25) +(`451223f7`), in five ordered commits and this record: + +1. **CLI fix.** The filesystem policy's canonical and lexical root pins and the workflow catalog's open check read + with `{ bigint: true }` into bigint fields. A witness file has four injected-identity real-replacement cases, and + the fs-invocation precondition reads exactly. +2. **Windows evidence.** A Windows-only probe in `fs-policy.test.ts` makes 200 rename-and-`mkdir` pairs, asserts + each pair is distinct as exact identities and logs how many collide as Numbers or read negative. Its real + replacement tests repeat twenty times. +3. **Media-store fix.** One internal `ExactStats` projection holds bigint `dev` and `ino` and number `nlink`, `size` + and `mtimeMs`; the last comes from `mtimeNs` by Node's own formula and was bit-identical on 307 checked files, + including fractional and negative times. It serves file snapshots, the owned-file check, the read's + descriptor stats, the publish handle and the temporary sweep's claim directory, children, marker and moved + blob; the root pin keeps its own bigint stats. `nativeFile` loses its `Number()`, and the claim token encodes the unsigned value + (`BigInt.asUintN(64, …)`). A witness file has seven cases, and the exact-claim vectors gain two negative + identities. `CAS_COOPERATIVE_READ`'s anchor follows the new parameter type. +4. **Tooling fix.** The overflow capture's destination check, the budget-replay evidence retention's directory + identity and its lifecycle-smoke precondition, each with a substituted-identity mode in its smoke. +5. **Permanent controls.** Five filesystem causal controls: `fs-root-identity-precision`, + `catalog-identity-precision`, `cas-root-pin-precision`, `cas-owned-identity-precision` and + `media-temp-same-precision`. Each passed its fixed, removal and restored phases through the real runner. The + declaration grows to 115 controls / 345 phases, in shards of 58 and 57. +6. **Documents.** The policy and physical storage paragraphs of the + [built-in tools reference](../../reference/shared-core/built-in-tools.md#physical-media-storage-and-limits), + dated corrections to ADR-0105 and ADR-0106, the NI entry and two scoped limits in + [deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24), and + the status lines. + +No runtime dependency or ADR is added, and no persisted format changes. + +**Proof.** The witnesses substitute every inode the code reads, keyed by its real device and inode, with +2^60 + k; device numbers stay real. Adjacent substitutes there are distinct bigints but one Number. Each witness asserts that its fixture collides as Numbers only, keeps the original +alive so the platform cannot recycle its inode, and asserts its outcome once with a named marker. + +A recorded sweep at the implementation commits applied 28 mutants, and each failed: + +- every comparison returned to a Number, which failed its named witness — the canonical and lexical reads and the + root pin's canonical and entry reads each have their own case; +- every read without its option, which failed its witness and also fails typecheck and the projection; +- the token's signed refusal restored, which failed on the negative vectors; +- each capture and retention read or comparison reverted, which failed its smoke (the lifecycle smoke's + precondition reads are test-side and were not swept; see the acceptance below). + +Number mutants on `dev` alone are equivalent on every supported platform and are not claimed: Windows `dev` is a +32-bit volume serial, and POSIX device numbers are small. The read's second-stat comparison is tested with a faked +second stat; it is a consistency check that no real filesystem fails. By decision, one number comparison remains: +the causal-control source graph's hard-link check, which can only refuse falsely. It is recorded as a scoped limit. + +**Windows evidence.** The advisory Windows job runs the probe and the repeated replacement tests after the push. The +Windows clause closes after three consecutive green advisory runs on pushed heads, with the logged counts recorded. + +**Author evidence at `209e7193`.** The full `pnpm run ci` passed (exit 0), including all 115 filesystem causal +controls with the five new ones, the product map controls and both changed smokes. + +**Internal rigor pass, 2026-10-25.** Under the rigor-before-review rule, an internal pass at `209e7193` ran claims, +platform, security, tests and integration lenses with a completeness critic, and every finding was adversarially +verified; the detailed record is local. It confirmed one medium finding: nothing pinned that a negative identity's +claim token is the unsigned value. Two encodings that alias a negative identity onto another value's token +(`asUintN(63, …)` and an absolute value) therefore passed the whole suite on the deletion-authority path. Corrected: +the exact-claim vectors now assert each claim name's token against a literal, and both mutants fail on +`MEDIA_TEMP_NATIVE_IDENTITY_TOKEN`. + +Low findings corrected: +- The recovery-marker fixture builds claim names with production's unsigned token. +- The scratch descriptor's real replacement test repeats like the others. +- The catalog's byte precheck gets two tests, each failing its own mutant: one refuses an oversized file before + any read, the other admits a dense multibyte file within the character cap. +- The claim-directory guard gets a test that retains a regular file under a claim name as an unsafe entry. +- The ADR-0106 note now names `nativeFile`'s Number cross-check, not the token, as what refused a top-bit identity + before NI (the message of commit `ed690d04` still says the token). +- The scoped limit now says FAT and exFAT can fail open, since their file IDs derive from directory-entry positions. +- The status lines and the controls' comment now say every *product* identity check, since one tooling check + remains by decision. +- Also corrected: the row's status, current.md's freshness line, the budget-replay README's replacement-control + count, and this checkpoint's commit count and root-pin wording. + +The pass measured the export capacity: one new control's phase manifest is 203,728 bytes, so a 58-control shard's +174 manifests come to 35,448,672 bytes, 52.8 % of the 64-MiB bound. + +### Wave 3 NI acceptance — 2026-10-25 + +**The native-identity increment NI is independently accepted on `development` at `fc5af701` (2026-10-25)** under +the Acceptance row as amended on 2026-10-23. Round 1 was a NEW independent complete cumulative review of +`451223f7..fc5af701`, with every reviewer in its own worktree detached at that head. It reported no blocker, high or +medium finding and no regression. Its separate security sitting, run at the same head as its own reviewer and report, +was clean. It verified that a replacement fails closed at the policy pins, the catalog open check, the media root pin, +the owned-file check, the temporary sweep's checks and the tooling retention check, and that the unsigned token is a +bijection on Node's reachable identities. It also verified that no identity value reaches an error, event, log, JSON +or IPC. The plan, correctness and integration lenses were clean. The plan reviewer found every commit's anchors valid +and its packages typechecked, with `CAS_COOPERATIVE_READ` moving in the same commit as its target. It also applied the +five control removals, and each failed its named witness. + +**Author evidence at `fc5af701`.** The full `pnpm run ci` ran alongside the round and passed (exit 0). Every causal-control family completed, among them all 115 filesystem controls with the five NI controls. All 25 product map controls and both changed smokes passed. + +The round's verified findings were four lows and four nits: +- Three lows are recorded in + [deferred-tasks.md](../deferred-tasks.md#wave-3-ni--recorded-low-findings-2026-10-25): the lifecycle smoke's + precondition reads can revert unnoticed; nothing tests the projection's `mtimeMs` formula; the catalog's + dense-multibyte case catches only a byte bound below 1.5 times the character cap. +- The fourth low is corrected in the acceptance commit, which changes no code: the fs-policy-controls README's + 110-control shard sentence is now dated. +- Of the nits, the docs' wording now says the witnesses substitute inodes keyed by device and inode, not whole + identities, and the checkpoint's proof bullet no longer claims a lifecycle-smoke sweep. +- The witness files' comments keep their wording: two nits, that devices stay real and that overlayfs inodes can + exceed 2^53. +- The CLI witness stays off the pinned Windows list: one nit. Its proof uses injected identities and does not + depend on the platform. + +The NI entry stays open until its Windows clause closes: three consecutive green advisory Windows runs on pushed +heads, with the probe's logged counts recorded. Step 9b is next. + +**Windows clause, run 1 of 3 (2026-10-25).** Push CI +[run 40151763770](https://github.com/HodeTech/Relavium/actions/runs/40151763770) at `c8c0d41b` passed all eighteen +jobs, the advisory Windows job among them. Its probe logged `1 of 200 replacement pairs collide as Numbers; 0 read +negative`: on the runner itself a replaced directory's file ID rounded to its original's once in 200 pairs, the +diagnosed mechanism, while every pair stayed distinct as exact identities, and the repeated replacement tests passed. + +**Windows clause, run 2 of 3 (2026-10-25).** Push CI +[run 40211801913](https://github.com/HodeTech/Relavium/actions/runs/40211801913) at `56ccae62` passed all eighteen +jobs, the advisory Windows job among them. Its probe logged `0 of 200 replacement pairs collide as Numbers; 0 read +negative`, and the repeated replacement tests passed. + +**Windows clause, run 3 of 3 (2026-10-26) — closed.** Push CI +[run 40276471445](https://github.com/HodeTech/Relavium/actions/runs/40276471445) at `5a328bca` passed all eighteen +jobs, the advisory Windows job among them. Its probe logged `0 of 200 replacement pairs collide as Numbers; 0 read +negative`, and the repeated replacement tests passed. With three consecutive green advisory Windows runs on pushed +heads, the Windows clause and the NI entry in [deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24) +are closed; NI's scoped limits stay recorded there. + +### Wave 3 Step 9b preflight decisions — 2026-10-25 + +A read-only preflight (four readers and a synthesis at `c8c0d41b`) mapped Step 9b's obligations, the code it meets and +GnuPG's mechanics on this machine and the CI runners; the detailed record and a Turkish decision register are local +and Git-ignored. The maintainer approved each recommendation on 2026-10-25. No new ADR or runtime dependency follows: +GnuPG is release tooling (packaging.md's recipient custody), the container is the in-house strict USTAR dialect the +2026-10-19 C1 ruling requires, and SHA-256 comes from `node:crypto`. The 2026-10-19 C2 ruling stands: GnuPG ≥ 2.2.8 is +required by local `pnpm run ci` and the Ubuntu runner, and the controls fail rather than skip. + +| Decision | Ruling | +| --- | --- | +| Placement | A library plus synthetic controls only. `producePackage`, `sealPackage`, the package-test receipt, the interim release guard and both workflows stay unchanged; encryption in the originating job is Step 9d's. SEC-9A2A-2's trigger (sandbox the capture child when 9b designs the custody job) is re-recorded to 9d with a dated note. | +| Release binding receipt | Step 9b defines its schema v1, its validator and a sealer that writes only after the ciphertext exists. Step 9d adds the origin, map-check and `ci`-run binding and replaces the guard. | +| Recipient selection | `--recipient-file` with `--no-keyring`: the file holds exactly one key, whose only usable encryption subkey is the pinned one, so there is no keyring, import, trustdb or agent. The single-usable-subkey rule is also the unexpected-recipient rule; Wave 4b's export becomes `gpg --export '!'`. | +| Ciphertext profile | SEIPDv1 with MDC and AES-256 (`BEGIN_ENCRYPTION 2 9`) is required and an AEAD profile refused, so `gpg -d \| tar -x` with GnuPG ≥ 2.2.8 stays true. A recipient key must not advertise AEAD, a new Wave 4b key clause. | +| Round-trip key | An ephemeral key per run, in an owner-only short-path home whose agent is stopped and whose socket directory is removed afterwards, with a test that no tracked file holds a PGP private-key block. No private key enters the repository or CI. | +| Key policy | Every recipient, fixtures included, must be OpenPGP v4 with a cv25519 or RSA ≥ 3072 encryption subkey. Expiry refuses at `now` with no margin; Wave 4b may add a margin with the real key. | +| Container layout | Staging-relative: `manifest.json`, `finalized.json`, `metafile-esm.json` and `maps/.js.map`. The 2026-10-19 ruling's "repository-relative paths" is read as relative POSIX paths with no host path; a decrypted, extracted archive is a staging directory that the manifest reader and symbolication consume unchanged. | +| Split | 9b.1 (USTAR, container, GnuPG runner, pre-check, fixtures, `encryptOnce`, prerequisite documentation) and 9b.2 (reuse, binding receipt, production loader, scope separation), each with its own round and security sitting; the plan has 32 increments. | + +Engineering defaults recorded with the rulings: +- Every GnuPG call goes through the existing bounded, shell-free `runCommand` in its own process group. +- The call passes `--no-options --no-autostart --no-keyring --no-encrypt-to --no-auto-key-locate --no-auto-key-retrieve --trust-model always --compress-algo none --status-fd 1 --set-filename`. +- It never passes `--yes`, `--hidden-recipient`, `--throw-keyids`, `--faked-system-time`, `--ignore-*` or `--allow-*`. +- Recipients are classified from a colon listing, because `INV_RECP` reasons do not tell the cases apart. +- Success requires one `BEGIN_ENCRYPTION`, one `END_ENCRYPTION`, exit 0 and exactly one `ENC_TO` for the pinned subkey on the ciphertext. +- An existing ciphertext refuses with zero GnuPG calls, and the ciphertext is published with `wx`. +- Plaintext lives in an owner-only work directory that is removed on every path. +- Errors are fixed `PACKED_CLI_CUSTODY_` codes, and GnuPG's stderr never surfaces. +- The tests join `test:acceptance-tools`, so they run in local `pnpm run ci` and the four Ubuntu jobs already inside `ci`'s needs, with no workflow edit. +- Windows has no GnuPG on its image, and the 2.2.8 floor is enforced only by the version check; both are recorded limits. +- Before the first external round, the custody tests run in a local `ubuntu:24.04` container with the distribution's GnuPG, to gather 2.4.4 evidence. + +### Wave 3 Step 9b.1 implementation checkpoint — 2026-10-25 + +**Scope and source.** Step 9b.1 (the 9b.1 row; #46, ADR-0111) is implemented on `development` after the +[Step 9b preflight decisions](#wave-3-step-9b-preflight-decisions--2026-10-25) (`12883df4`), in five ordered commits +and this record: + +1. **One strict USTAR dialect.** `ustar.mjs` extracts the archive inspector's reader with a path rule, a mode rule, + an entry hook and a refusal reporter, and promotes the writer. The inspector delegates with its refusal codes and + order unchanged, and `archive.test.mjs` and `seal.test.mjs` pass unchanged. +2. **The private container.** `container.mjs` builds and reads the owner-only container in the approved + staging-relative layout, every map checked against the manifest and the finalizer's record. +3. **The isolated GnuPG runner.** `gnupg.mjs`, with GnuPG ≥ 2.2.8 added to the local prerequisites. +4. **Synthetic fixtures.** Committed armored public keys for the recipient classes GnuPG can produce, + their pins and a maintainer-only generator; crafted listings cover the other classes, and the round-trip key + is generated per run. +5. **Encrypt once.** `custody.mjs`: pins, the colon pre-check, `encryptOnce` and its tests. + +The [packaging contract](../../reference/cli/packaging.md#recipient-custody-implementation-boundary--step-9b1) records +the mechanism, the tooling README and deferred-tasks.md (SEC-9A2A-2 re-recorded to Step 9d, two Wave 4b key +clauses, two named limits) the rest. Product paths, the receipt, the guard and both workflows are unchanged; no +runtime dependency or ADR is added. + +**Measured before writing the parsers.** One script listed every fixture, encrypted to each and listed the +ciphertext's recipients on GnuPG 2.5.21 (local) and 2.4.4 (`ubuntu:24.04`): +- `BEGIN_ENCRYPTION 2 9` for a key without the AEAD feature, `0 9 2` (OCB) for one with it, on both versions. +- `--list-only` exits 0 with one `ENC_TO`; `--list-packets` exits 2. +- `INV_RECP` reasons do not tell expired, revoked and sign-only apart, and the `FAILURE` numbers differ between + calls, so neither is keyed on. +- An `INV_RECP` line carries the recipient file's absolute path, which is why status output is never surfaced. +- GnuPG 2.4.4 cannot list an expired or revoked key under `--trust-model always` without a trust database (`Fatal: + can't open trustdb.gpg`), so the listing runs in its own home with the default trust model. +- With `--trust-model always`, the encryption home gains only `random_seed`. + +**Author evidence at `85a9aebe`.** +- The four custody test files passed locally: 32 tests on GnuPG 2.5.21. +- They also passed in a clean clone in `ubuntu:24.04`: 32 of 32 with none skipped, as a non-root user, on its + GnuPG 2.4.4 after a fresh frozen install, with no GnuPG process left afterwards. +- The archive, seal, packed-CLI pin and tooling typecheck and lint suites passed. + +**Internal rigor pass, 2026-10-25.** Two checks preceded the first round: +- an author mutation sweep at `0bd1953f`, 44 mutants; +- an internal pass at `45bc5654`, with claims, platform, crypto/custody pre-sitting, tests and integration lenses + and a completeness critic, every finding adversarially verified (the record is local). + +Commit `45bc5654` fixed the sweep's survivors. Its message says three survived; a fourth, the container's pre-write +total check, had been counted as killed only because a flaky interrupt test failed under it. + +The pass confirmed seven medium findings, all corrected: +- **Host errors escaped with their path.** Errors raised before `encryptOnce`'s main try block carried their + path: a missing destination or root, a failed listing or path resolution. The whole function now sits inside + the fixed-code boundary. A missing destination refuses `PLACEMENT`, a missing root `PRIVATE_ROOT`, and a work + directory that cannot be removed `CLEANUP`. +- **The AEAD control was version-blind.** It asserted `PROFILE`, which holds only on GnuPG 2.3 and later. It now + takes the expectation from the installed version, never skipping: GnuPG 2.2 has no AEAD, encrypts SEIPDv1 and + succeeds. +- **The interrupt test raced the reaping.** It probed the stand-in's process once, racing the reaping of its + killed group, and failed about once in 30 runs. It now polls with a deadline and failed 0 of 30. +- **The success test checked the record against itself.** The record's SHA-256 and GnuPG version are now + computed independently. +- **Three guards had no isolating test.** The MDC half of the profile check, the pinned-primary-as-subkey guard + and the finalizer record's map binding (path, digest and size) each have one now. +- **The checked recipient copy and its bound were unproven.** Nothing proved that the copy is the one encrypted + to, or that the 256 KiB bound holds. One test now changes the caller's file mid-run, and another exceeds the + bound. + +Low findings corrected: +- The publish is atomic: a temporary sibling, hard-linked to the final name. +- The version call carries the isolation flags. +- The metafile must parse as a JSON object, and the contract says what binds each record. +- The pre-write total refuses with its own code. +- More USTAR header, status-keyword and classification cases are tested. +- The contract's wording on the modification-time field, listed keys, future creation, the fixture classes and + Windows is corrected (the Windows image ships Git's MSYS2 `gpg`, and the tests use POSIX tools, so Windows + developers use WSL), and so is deferred-tasks.md's freshness line. + +Two lows are recorded in deferred-tasks.md: an interrupted test run can leave the ephemeral key's home, and the +pre-check counts only the keys GnuPG lists. A rerun sweep of 52 mutants left no survivor. + +**CI at `45bc5654`.** The full `pnpm run ci` failed one test outside the increment. The usage-coverage +inventory's `vitest list` collection exceeded its 30-second spawn timeout while the rigor pass's agents held the +load average at 40–66. On a quiet machine it passed three of three runs in 10–12 seconds. The full CI reruns at +the corrected head, alone, before the first round. + +**Author evidence at `f0555a61`.** The full `pnpm run ci` passed alone (exit 0): `test:acceptance-tools` 969 of +969, all 25 product map controls, and every causal-control family complete. + +**Round 1 at `f0555a61` — NONCLEAN.** Five lenses and a critic ran, every finding adversarially verified. They +reported three medium findings, all corrected: +- **Global configuration.** GnuPG reads its installation's `/gpg.conf` even under `--no-options`, so + the "no options file" claim was false. The reviewer showed that a global `log-file` with debug options wrote + the AES session key outside every home while the encryption passed every check. The installation's `gpgconf` + now names that directory before any key is handled, and a `gpg.conf` or `common.conf` there refuses + (`GPG_CONFIG`). The contract names the installation as trusted. +- **Container host errors.** `buildPrivateContainer` still let host errors carrying the staging path escape; the + rigor pass had named it with `encryptOnce`, and only `encryptOnce` was corrected. They now refuse + `CONTAINER_UNREADABLE`. +- **Algorithm policy.** Nothing tested the algorithm-ID half of the policy: an ElGamal subkey would pass once + that half was removed. ElGamal cases now refuse. + +Lows corrected: +- **Overlap.** The destination and root are compared by device and inode, not spelling; a case-variant test runs + on case-insensitive volumes and is skipped where the volume is case-sensitive. +- **Publish order.** The listed ciphertext file must still be the bytes published, and the plaintext work + directory is removed before anything is published, so a `CLEANUP` refusal publishes nothing. +- **Tests.** New cases cover revocation before expiry, an expired second subkey, `ssb`-only secret material, the + default clock's unit, the record items' exact keys, the isolated environment, the C locale and owner-only homes, + and the forbidden `--ignore-*` family. +- **Wording.** Corrected in the fixture comment, the contract's secret-material wording, the Wave 4b clause (an + AES256 preference) and this checkpoint's fixture item. + +Two lows are recorded in deferred-tasks.md: secret material GnuPG neither lists nor reads is admitted, and the +atomic publish has no fault-injection test. Every corrected guard's mutant now fails a test. + +**Author evidence at `dea32ea6`.** The full `pnpm run ci` passed alone (exit 0): `test:acceptance-tools` 975 of +975. On Ubuntu 24.04 with GnuPG 2.4.4 (in Docker), the 41 USTAR, container, GnuPG and custody tests ran: 40 +passed and the case-insensitive-volume case skipped, as it does on a case-sensitive volume. + +**Round 2 at `dea32ea6` — NONCLEAN.** Five lenses and a critic ran, every finding adversarially verified. The plan +and integration lenses were clean. One medium, corrected: +- **The outer `CLEANUP` override was untested.** Round 1 added a removal before publishing, and the only `CLEANUP` + test then reached that inner removal alone. Deleting the outer override, which turns an earlier refusal into + `CLEANUP` when its work directory cannot be removed, failed no test, so a caller could be told `STATUS` while + plaintext remained. A refusal whose work directory then cannot be removed is now tested to report `CLEANUP`. + +Lows corrected, none deferred: +- **gpgconf's answer.** `gpgconf --list-dirs sysconfdir` prints a named directory raw; only the unnamed listing + is percent-escaped. The percent-decoding turned a literal `%41` in the path into another directory, so a global + configuration there was admitted. The answer is now taken as printed, and anything but exactly one absolute + path ended by one newline refuses (CRIT-9B1-R2-1; checked against the real gpgconf 2.5.21). +- **Windows.** The owner-only check admitted any directory on Windows, where modes say nothing about access. No + private root qualifies there now, so `encryptOnce` refuses before any GnuPG call, and the named limit says so + (SEC-9B1-R2-2). +- **Pins.** A pin that only converts to a fingerprint, such as a one-element array, passed. Pins must now be + strings (CRIT-9B1-R2-2). +- **Record bounds.** The container admitted a `manifest.json` of up to 16 MiB, which the manifest reader refuses + over 4 MiB. Each record is now held to its own reader's bound, one `RECORD_BYTES` map shared by the manifest + readers, the metafile read in `seal.mjs` and the container (CRIT-9B1-R2-3). +- **Fixture.** The ephemeral key's home was created before its path-length check, so a failed check left it + behind (T-9B1-R2-8). +- **Tests.** New cases cover: + - a duplicated or second `KEY_CONSIDERED`, and a `sec` record without `ssb`; + - the environment and bounds of every GnuPG call; + - the private root's group bits, a linked root and another user's root; + - the work directory's, homes' and plaintext files' modes; + - the ciphertext name as the contract spells it; + - every container bound met exactly, and the entry bound on reading; + - the build's read-back self-check, through a chunk name too long for the dialect's name field; + - empty and block-aligned USTAR entries; + - a planted ambient environment. + + (COR-9B1-R2-1/2, T-9B1-R2-2 to T-9B1-R2-7, SEC-9B1-R2-1.) +- **Wording.** The contract's secret-material sentence now names the recorded limit, its "every call" is + narrowed to every `gpg` call, and its runbook citation is a relative link. The runbook names the + global-configuration requirement. The 9b row carries its split note, and the CLI reference index names 9b.1 + (PLAN-9B1-R2-1/2/3, INT-9B1-R2-1/2, COR-9B1-R2-3). + +A 30-mutant sweep over the corrected guards and the round's named mutants left no survivor. That includes the +owner check's uid half, which no earlier test covered. + +**Author evidence at `ade62b7e`.** A first full run was started with `CI=true GITHUB_ACTIONS=true` set for the +whole chain. `check:map-controls` refuses such a half-configured GitHub environment before it builds anything +(`PACKED_CLI_MAP_CONTROL_ORIGIN`), as designed, so that run certified nothing. Rerun plain and alone, the full +`pnpm run ci` passed (exit 0): `test:acceptance-tools` 979 of 979, all 25 product map controls, and every +causal-control family. On Ubuntu 24.04 with GnuPG 2.4.4 (in Docker), the 45 USTAR, container, GnuPG and custody +tests ran: 44 passed and the case-insensitive-volume case skipped. The commit recording this evidence also +corrects one contract phrase and changes no code. + +**Round 3 at `b3eaebaf` — NONCLEAN.** Five lenses and a critic ran, every finding adversarially verified. The plan, +security and integration lenses were clean. Two mediums, both in tests, corrected: +- **The environment assertion did not name the work directory.** Round 2's check that every GnuPG call runs in the + work directory's environment accepted any directory named `custody-*`, and the test's own private root and + destination are named that way, so an environment built on either passed. Round 2's sweep had tried a leaked + process environment, never a wrong directory. Every call's environment must now be that of the one work + directory holding every home, in `mkdtemp`'s name shape, the gpgconf query's included. +- **A hidden recipient beside the pinned one was untested.** The cases covered a duplicated pinned line, a wrong + ID, a hidden ID alone and no recipient, so a filter that ignored anonymous `ENC_TO` lines passed. The pinned + subkey with a hidden or another recipient beside it now refuses `RECIPIENTS` in tests. + +Lows corrected, none deferred: +- **Additional decryption subkeys.** The pre-check did not classify an ADSK (capability `r`, GnuPG 2.4.1 and + later), which GnuPG adds as a further recipient, so a key carrying one passed it and only GnuPG's own refusal + and the recipient check stopped it. A usable ADSK now refuses `RECIPIENT_UNEXPECTED` before encryption, checked + against a real key carrying one on GnuPG 2.5.21 (COR-9B1-R3-1). +- **Tests.** New cases cover the finalizer record's output paths, a same-count map rename, the read-side entry + count one past its bound, the creation time to the second, a prefixed status line, gpgconf beside a linked + gpg's real binary, a directory named `gpg` on PATH, an empty USTAR name, a regular-file private root and a + missing configuration directory (T-9B1-R3-3 to T-9B1-R3-6, CRIT-9B1-R3-1). +- **Wording.** The 9b.1 row says which record is bound to what, round 2's record names the readers that share + `RECORD_BYTES`, the contract and its JSDoc claim only the size bound, and the runbook and a test comment no + longer say Homebrew and Ubuntu ship an empty configuration directory (PLAN-9B1-R3-1/2, CRIT-9B1-R3-2/3). + +**Sweeps.** The 48 named mutants (round 2's 30 and round 3's 18) all fail a test. A generated sweep then mutated +every one-line refusal, condition clause and comparison of `custody.mjs`, `gnupg.mjs`, `container.mjs` and +`ustar.mjs`: 182 mutants, 168 killed at first. Five survivors were real gaps and gained cases: +- a fingerprint record before any key; +- a ciphertext that can no longer be read after its listing; +- a `null` record item and a `null` finalizer record; +- a one-character USTAR name. + +The other nine are equivalent: +- two distinct status lines never share an index; +- the record items' type checks refuse with the same code at the comparison that follows; +- two sort comparators never meet a tie that matters; +- the build's map-name check refuses with the same code as the name-set check; +- the self-check's comparison cannot differ with the current writer; +- the checksum loop's extra index reads `?? 0`; +- twelve octal digits are always a safe integer. + + +### Wave 3 Step 9b.1 acceptance — 2026-10-25 + +**Step 9b.1 is independently accepted on `development` at `147c2788` (2026-10-25)** under the Acceptance row as +amended on 2026-10-23. Rounds 1–3 were not accepted, each with mediums corrected as the +[checkpoint](#wave-3-step-9b1-implementation-checkpoint--2026-10-25) records: +- round 1, a global GnuPG configuration the "no options file" claim missed; +- round 2, round 1's correction left the outer `CLEANUP` override untested; +- round 3, round 2's environment assertion accepted the wrong directory, and no case put a hidden recipient beside + the pinned one. + +Round 4 was a NEW independent complete cumulative review of `12883df4..147c2788`, with every reviewer in its own +worktree detached at that head. It reported no blocker, high or medium finding and no regression. Its separate +crypto/custody security sitting, run at the same head as its own reviewer and report, was clean. The plan lens was +clean too. It mapped every clause of the 9b.1 row and rulings L1–L8, C1 and C2 to code, found the inventory pin and +every import valid at all twelve commits, and confirmed that nothing outside the tests imports the library. + +**Author evidence at `147c2788`.** The full `pnpm run ci` passed plain and alone (exit 0): `test:acceptance-tools` +980 of 980, all 25 product map controls, and every causal-control family. On Ubuntu 24.04 with GnuPG 2.4.4 (in +Docker), the 46 USTAR, container, GnuPG and custody tests ran: 45 passed and the case-insensitive-volume case +skipped. + +The round's verified findings were five lows: +- Four are recorded in + [deferred-tasks.md](../deferred-tasks.md#wave-3-step-9b1--recorded-low-findings-2026-10-25): + - a temporary sibling that cannot be removed after a successful link reports `UNEXPECTED`, although the ciphertext + is published; + - no test pins the metafile's 4 MiB value; + - no test builds the container from a real finalizer's staging; + - the overlap check misses containment through a bind mount of a directory inside the other. +- The fifth is corrected in the acceptance commit, which changes no code: the contract and the Windows named limit + no longer call Git for Windows' build the image's only `gpg`. The same commit narrows the contract's overlap + sentence to what the check catches. + +Step 9b.2 is next. + +### Wave 3 Step 9b.2 preflight decisions — 2026-10-25 + +A read-only preflight (four readers and a synthesis at `56ccae62`) mapped Step 9b.2's obligations, the code it builds +on and what 9c, 9d and Wave 4b will need from it; the detailed record and a Turkish decision register are local and +Git-ignored. The maintainer approved each recommendation on 2026-10-25. No new ADR or runtime dependency follows: the +receipt identifies files by SHA-256 from `node:crypto`, with no signature or HMAC (packaging.md's public binding +receipt), and every module is release tooling. All four parts are new modules, so no accepted 9b.1 file changes and +none of 9b.1's "next change to the file" lows fire. + +| Decision | Ruling | +| --- | --- | +| Schema v1 | Carries every field the contract lists now, `origin` and `mapCheck` included, checked for shape only. 9d adds trust checks against these fields (the origin against the GitHub run, `mapCheck` against a successful `ci` run at the source commit) and no new field; there is no `ciRun` field. | +| Release-smoke inventory | Not embedded. 9d derives the public file inventory and package metadata from the digest-verified tarball with the inspector that produces the package-test receipt, a recorded 9d obligation. | +| Toolchain identity | The manifest's per-file `inputs` (lock and configuration files only), the five tools plus the GnuPG version, and the target: checkable file by file against the commit, and not the per-source inventory ADR-0111 forbids. | +| Scope enforcement | A new `encryptCandidate` gate in front of the unchanged `encryptOnce`, with a module-private registry of the records it returns. Production scope requires the loader's brand and a clean, non-synthetic source; synthetic scope requires the synthetic source, so the per-run ephemeral key can never encrypt a product staging. The sealer accepts only a record that gate returned, and only once. | +| Production loader | A fixed committed path, `tools/cli-package/release-recipient/`, holding `recipient.asc` and a `pins.json` with an append-only `current`/`retired` history: exactly one `current`, no entry ever removed. In Wave 3 the directory is absent and the loader refuses `PRODUCTION_RECIPIENT_ABSENT`; there is no environment variable, flag or fallback, and a fixture fingerprint refuses. Rotation controls are built now (ADR-0111's acceptance 5); Wave 4b only adds the files. | + +Engineering defaults recorded with the rulings: +- **Files.** `release-recipient.mjs` (the loader and its brand), `release-binding.mjs` (schema, validator and reuse, + whose import closure cannot start a process, so reuse's zero GnuPG calls are structural), `release-custody.mjs` (the + gate and the sealer, the only module that reaches GnuPG) and the test-only `release-fixture.mjs`. Each test file + joins `test:acceptance-tools` in the commit that creates it. +- **Receipt bytes.** The receipt is written once in canonical bytes as `release-binding-receipt.json`, the name the + interim guard already keys on. Verifiers parse the exact bytes and never re-serialize, so a duplicated key, a + reordering, a BOM or trailing data refuses. +- **Sealer.** It refuses an existing receipt before reading anything, admits exactly the tarball and the ciphertext + in the destination, and re-reads the ciphertext's size and digest against the record and the tarball's digest + against the manifest. It then publishes once, by `wx` and a link. A sibling that cannot be removed after the link + is not a refusal, and a write port allows fault injection: the lessons of COR-9B1-R4-1 and T-9B1-R1-3. +- **Reuse.** It admits exactly the three files and checks each asset's size, then its bounded bytes and SHA-256, + against the receipt and the expected recipient history. It returns the verified bytes rather than paths and writes + nothing. Re-checking `ENC_TO` would need GnuPG and is a named limit. +- **Static controls.** Only `release-custody.mjs` imports `encryptOnce`; no product path or workflow names a release + module until 9d; the release-binding closure excludes every subprocess module; no recipient directory is tracked + until Wave 4b. The interim guard stays unchanged and still refuses a valid synthetic candidate. +- **Gate.** A rigor pass and a generated mutation sweep before the first round; the full `pnpm run ci` alone; the + custody tests on Ubuntu 24.04 with GnuPG 2.4.4 in Docker; one new complete cumulative round and a separate + crypto/custody security sitting. + +### Wave 3 Step 9b.2 implementation checkpoint — 2026-10-25 + +Step 9b.2 is implemented on `development` in the commits from `84b8e455` through `4d49176b` and this record, under the +[preflight decisions](#wave-3-step-9b2-preflight-decisions--2026-10-25). Its canonical description is packaging.md's +[Step 9b.2 boundary](../../reference/cli/packaging.md#recipient-custody-implementation-boundary--step-9b2). No Step +9b.1 module changed, and nothing outside the new tests calls the new modules. +- **The production recipient** (`release-recipient.mjs`). The loader reads the committed + `tools/cli-package/release-recipient/` directory and refuses `PRODUCTION_RECIPIENT_ABSENT` in Wave 3, because no + recipient is committed. A linked or malformed directory refuses `PRODUCTION_RECIPIENT_INVALID`, and a history naming + a committed fixture key refuses `FIXTURE_RECIPIENT`. The loaded value is branded through a module-private set. +- **The receipt, its validator and reuse** (`release-binding.mjs`). Schema v1 carries every field the contract lists, + with a required expectation and canonical bytes. Reuse admits exactly the three files and returns the verified bytes. + The module's import closure, pinned by a test, holds only `archive.mjs`, `custody-error.mjs`, `files.mjs`, + `release-recipient.mjs` and `ustar.mjs`. +- **The gate and the sealer** (`release-custody.mjs`). Every scope refusal comes before any GnuPG call, and a + module-private registry ties the sealer to the records the gate returned. The receipt is published once through a + hard link, and a sibling left after the link is not a refusal. +- **Tests.** Four new files join `test:acceptance-tools`, with 34 tests before the rigor pass and 39 after it: `release-recipient`, `release-binding`, + `release-custody` and `release-scope`. They include a production round trip, in which an ephemeral key under a + rotated history decrypts to the container whose manifest the receipt binds, and fault injection after the link and + after encryption. +- **Docs.** packaging.md's Step 9b.2 boundary is the schema's canonical home. deferred-tasks.md records the named + limits (as of round 1: an orphaned ciphertext, no recipient re-check on reuse, a sibling left after the link, the + fixture denylist's coverage, the test ports and Windows not exercised), the Step 9d handoffs and a Wave 4b clause on + the recipient's committed form. + +**Sweeps before the rigor pass.** A generated sweep mutated every refusal statement, every guarded one-line refusal, +every clause of a multi-line condition and every comparison of the three modules; an inner clause of a one-line +compound condition is not mutated on its own: +131 mutants, 119 killed at first. Five survivors were real gaps and gained cases: +- a `pending` status beside the current one; +- a one-entry inputs map; +- a receipt passed as a `Uint8Array`; +- a host failure in the gate, and one after encryption. + +The other seven are equivalent: +- the history's type and read checks: removing one leads to a host error that the loader's wrapper maps to the same + code, or for an empty history to the one-current rule; +- the inputs sort comparator, whose keys are unique; +- the receipt bound's `>=` flip, since no canonical receipt reaches 64 KiB. + +24 named semantic mutants, which the generator cannot produce, all fail a test. They cover the brand, the fixture +denylist, the root's shape, the current pins, the registry's consumption, the best-effort removal, the `EEXIST` +mapping, the canonical bytes, the sealer's fields and the link-aware presence check. + +**Rigor pass at `d94ed705`.** Internal lenses (claims, platform, crypto/custody pre-sitting, tests and integration) +and a critic ran, every finding adversarially verified. The platform lens was clean. Three mediums, corrected: +- **The loader's seams minted a production recipient.** The exported `root` and `fixtures` parameters let any caller + brand any key, a fixture key or the per-run key included, with the denylist switched off; only a comment and an + interim control stood in the way. `loadProductionRecipient()` now takes no argument, and tests load temporary roots + through a separately named `loadRecipientForTests`. Two permanent controls keep every call of the loader + argument-free and its test seam out of product and tooling code (CLAIM-9B2-1, SIT-9B2-4). +- **The import control matched three spellings.** A `../custody.mjs` import from `smoke/`, a workflow's `./tools/…` + path or a query suffix passed unseen. The control now resolves every quoted specifier against its file, or a + workflow's against the repository root, and has its own positive and negative self-test (SIT-9B2-1, T-9B2-R1-1). +- **The primary half of the recipient pair was untested.** A pair matching only by its subkey now refuses in both + scopes (T-9B2-R1-2). + +Lows corrected: +- the gate reads its scope once and encrypts its own copy of the container, and the validator snapshots its + expectation (SIT-9B2-2, T-9B2-R1-3); +- the gate resolves the destination before encrypting (`PLACEMENT`) and checks the tarball against the manifest + (`TARBALL_DRIFT`), so nothing after publication can fail on the destination and the likeliest orphan causes refuse + first (CRIT-9B2-3, SIT-9B2-5); +- an integer-like input name refuses, so the canonical bytes list inputs by path (CLAIM-9B2-5, SIT-9B2-8); +- the fixtures' pins file is parsed strictly (SIT-9B2-3); +- the closure control also reads `export * as`, forbids `cluster` and `module`, and scans the closure's files for + `require`, `createRequire`, `getBuiltinModule` and `process.binding` (SIT-9B2-7, T-9B2-R1-5, INT-9B2-R1-2, + CRIT-9B2-1); +- the guard control uses a complete, valid synthetic candidate (T-9B2-R1-4, INT-9B2-R1-1); +- new tests cover the origin's length bounds, a genuine invalid UTF-8 byte, a reused asset swapped for a link after + its size check, and the reuse refusal after a sibling is left (T-9B2-R1-6, T-9B2-R1-7, T-9B2-R1-8, SIT-9B2-6); +- wording: the contract's diagnostics, canonical-bytes and orphan sentences, the module header, the README and this + checkpoint's commit range (CLAIM-9B2-4, CLAIM-9B2-6, INT-9B2-R1-3 to INT-9B2-R1-6, CRIT-9B2-4). + +Recorded in deferred-tasks.md: the fixture denylist misses an unlisted second subkey, a left sibling blocks reuse +until it is removed, the append-only rule needs a Wave 4b control, and the release job's checkout must include the +fixtures' pins file (CLAIM-9B2-2, CLAIM-9B2-3, SIT-9B2-6, CRIT-9B2-2). + +**Sweeps at the corrected head.** The generated sweep ran 137 mutants and killed 128 at first. One more gap, a +malformed fixture subkey, gained a case. The other eight are equivalent: six history, fixture and read checks whose +removal the loader's wrapper maps to the same code, the inputs comparator, and the receipt bound's `>=` flip. Of 29 +named mutants, 28 are killed. The survivor, the synthetic recipient's snapshot, is equivalent: synthetic scope admits +any recipient, and `encryptOnce` checks the file it reads against the pins it parsed. + +**Author evidence at `91d08a65`.** The full `pnpm run ci` passed plain and alone (exit 0): `test:acceptance-tools` +1,019 of 1,019, all 25 product map controls, and every causal-control family. On Ubuntu 24.04 with GnuPG 2.4.4 (in +Docker, a fresh clone, a non-root user), the 85 tests of the 9b.1 custody files and the four release files ran: 84 +passed and the case-insensitive-volume case skipped. + +**Round 1 at `91d08a65` — NONCLEAN.** Five lenses and a critic ran, every finding adversarially verified. Three +mediums, corrected: +- **The rigor pass's destination fix dropped a 9b.1 refusal.** Resolving the destination in the gate and handing + `encryptOnce` the resolved path let a linked destination through, which `encryptOnce` alone refuses. The gate now + refuses a link itself (`PLACEMENT`, before anything is read through it) and passes `encryptOnce` the caller's + destination, so its refusals apply unchanged (PLAN-9B2-R1-1, COR-9B2-R1-1, SIT-R1-1). +- **A stray entry led to an unsealable ciphertext.** The gate checked the tarball but not the destination's other + entries, which the sealer refuses. It now requires exactly the tarball: an existing ciphertext refuses + `ALREADY_ENCRYPTED`, and anything else, including a tarball name in another case, refuses `CANDIDATE_ENTRIES`, all + before any GnuPG call (SIT-R1-2, PLAN-9B2-R1-2, COR-9B2-R1-2). +- **A bare `custody.mjs` specifier escaped the import control.** The control now reads every quoted literal, nested + ones included, decodes percent-escapes, ignores a query or fragment, and resolves a bare name as Node resolves + `new URL(…, import.meta.url)`; package scripts join its scan (INT-9B2-R1-A, TEST-9B2-R1-7, COR-9B2-R1-4, + PLAN-9B2-R1-3). + +Lows corrected: +- the validator reads each field once, and the public validator, parser and reuse refuse a malformed call or a + throwing accessor with a fixed code (SIT-R1-6, COR-9B2-R1-3); a vanished asset refuses `CANDIDATE_ASSET` + (COR-9B2-R1-5); +- `pins.json` must be its canonical serialization, so a duplicated key refuses, and the checks that this subsumes are + removed (CRIT-9B2-R1-2); +- the closure control is an allowlist of its own files and listed builtins, and refuses a non-literal dynamic import + and `process.execve` (SIT-R1-3, INT-9B2-R1-B); +- the fixture control covers every `*-fixture.mjs` name (SIT-R1-4); +- the overhead constant's parity reads custody.mjs's own declaration (TEST-9B2-R1-6); +- new tests cover anchored patterns, a non-string whose string form matches, the forwarding of the clock and the + GnuPG installation, the reuse snapshot and a directory under the tarball's name (TEST-9B2-R1-2, TEST-9B2-R1-3, + TEST-9B2-R1-5, CRIT-9B2-R1-1); +- wording: this checkpoint's named-limit list and its sweep description (INT-9B2-R1-C, CRIT-9B2-R1-1). + +Recorded in deferred-tasks.md: the release entry points accept the test ports (SIT-R1-5), and the production loader's +own denylist wiring is exercised only once Wave 4b commits the files (TEST-9B2-R1-4). The oversized-receipt case of +TEST-9B2-R1-1 stays as it is: no canonical receipt reaches the 64 KiB bound, which only spares the parser an +oversized input. + +**Sweeps at the round 1 corrections.** The generated sweep ran 131 mutants and killed 127. The four survivors are +equivalent: the fixture entry's record check and the recipient files' read refusal, whose removal leads to a host error +that the loader's wrapper maps to the same code, the inputs comparator, and the receipt bound's `>=` flip. Of 38 named +mutants, 36 are killed. The two survivors are equivalent: passing `encryptOnce` the resolved destination, because the +gate's own `lstat` refuses every link first and `encryptOnce` resolves the path itself, and the synthetic recipient's +snapshot, as above. + +**Author evidence at `5cda6459`.** The full `pnpm run ci` passed plain and alone (exit 0): `test:acceptance-tools` +1,022 of 1,022, all 25 product map controls, and every causal-control family. On Ubuntu 24.04 with GnuPG 2.4.4, the +88 tests ran: 87 passed and the case-insensitive-volume case skipped. + +**Round 2 at `5cda6459` — NONCLEAN.** Five lenses and a critic ran, every finding adversarially verified. Two +mediums, corrected: +- **A nested package script escaped the import control.** pnpm runs a package's scripts in that package's directory, + but the control resolved every manifest against the repository root, so a script in `apps/cli/package.json` could + reach `custody.mjs` unseen; a workflow step's working directory left the same gap. No single base is right, so the + control now refuses any quoted reference, outside the gate, to a file named `custody.mjs`, the one tracked file of + that name, with separators, queries, fragments, percent-escapes and case normalized (PLAN-9B2-R2-1, COR-R2-1, + T-R2-3). +- **A trailing separator bypassed the link refusal.** `lstat('link/')` follows the link, so a linked destination + spelled with a trailing `/` or `/.` was admitted. The gate now requires the destination and the private root in + normalized absolute form, and checks the destination before probing it for a receipt, so a link refuses + `PLACEMENT` before anything is read through it (COR-R2-2, SIT2-R2-1, PLAN-9B2-R2-2, INT-9B2-R2-1). + +Lows corrected: +- the sealer reads its options inside its fixed-code boundary (SIT2-R2-2, COR-R2-4, T-R2-1); +- the validator reads the origin and the inputs once (COR-R2-5); +- a dynamic import in the receipt's closure must be exactly one string literal, a rule with its own self-test + (COR-R2-3); +- a recipient root whose parent is a file refuses `PRODUCTION_RECIPIENT_INVALID`, never absent (T-R2-2); +- the Step 9d handoff widens the gate's, the sealer's and reuse's directory admission together (CRIT-9B2-R2-1). + +Recorded in deferred-tasks.md: `encryptOnce` called directly still follows a link spelled with a trailing separator. +Nothing outside the gate and the tests calls it, and the gate requires normalized paths (the 9b.1 half of COR-R2-2). + +**Sweeps at the round 2 corrections.** The generated sweep ran 135 mutants and killed 131; the four survivors are the +equivalents named above. Of 40 named mutants, 38 are killed; the two survivors are the equivalents named above. + +**Author evidence at `463a047c`.** The full `pnpm run ci` passed plain and alone (exit 0): `test:acceptance-tools` +1,023 of 1,023, all 25 product map controls, and every causal-control family. On Ubuntu 24.04 with GnuPG 2.4.4, the +89 tests ran: 88 passed and the case-insensitive-volume case skipped. + +**Round 3 at `463a047c` — NONCLEAN.** Five lenses and a critic ran, every finding adversarially verified. The plan and +integration lenses were clean, and the security sitting reported lows only. The tests lens reported four mediums, all +in tests or static controls; the code they cover was correct: +- **A non-`EEXIST` link failure was untested.** Turning that `throw` into a `return` passed every test, so a seal could + have reported success with no receipt. An injected `EPERM` link now refuses `UNEXPECTED` with no host text, publishes + nothing and leaves the record sealable (T4-1). +- **The gate's refusal of a linked tarball was untested.** A tarball that is a link to a file with the right bytes + now refuses `TARBALL_DRIFT` before any GnuPG call (T4-2). +- **No test pinned the patterns' trailing anchors.** A suffix after each valid value now refuses (T4-3). +- **A template literal escaped the `custody.mjs` control.** A substitution containing a space hid + `` `${join(here, base)}/custody.mjs` ``, and whitespace-bearing scripts, unquoted arguments and JavaScript escapes + hid other spellings. The control no longer looks for quoted specifiers: after JavaScript and percent escapes, + separators and case are normalized, it refuses any mention of the name outside the gate, comments included + (T4-4, PLAN-9B2-R3-1, SIT3-L2). + +Lows corrected: +- the closure scan reads every specifier after `from` or `import`, whatever the spacing, so spaced, compact and + multi-line imports are seen (SIT3-L1, T4-5); +- new tests cover single reads of the gate's recipient, the validator's origin and inputs, throwing options on the gate + and on reuse, same-count renames for the sealer and reuse, the toolchain's length bounds, and the receipt bytes + returned by reuse while the file changes (COR3-R3-1, COR3-R3-2, T4-6, T4-7, T4-8); +- the contract's diagnostics bullet names every path's code (CRIT-9B2-R3-1). + +**Sweeps at the round 3 corrections.** The generated sweep gained the operator classes the rounds had found by hand: a +`throw` turned into a `return`, each regex anchor removed, each `||` and `&&` swapped inside one-line conditions, each +integer bound raised by one, and each single-line call statement deleted. It ran 233 mutants. Three survivors were +real gaps, the `typescript`, `tsup` and `esbuild` length bounds, and gained cases. Ten are equivalent: +- the four named above; +- `isRecord`'s two operator swaps and the history guard's `||`, which let through only values that still refuse with + the same code; +- `string()`'s default maximum, used only by fixed-length patterns; +- deleting `writePublicFile`'s `fsync` or `close`, which guard durability and descriptor hygiene, not anything a test + observes. + +Of 40 named mutants, 38 are killed; the two survivors are the equivalents named above. + +### Wave 3 Step 9b.2 acceptance — 2026-10-26 + +**Step 9b.2 is independently accepted on `development` at `563402e6` (2026-10-26)** under the Acceptance row as +amended on 2026-10-23. An internal rigor pass and rounds 1–3 were not accepted, each with mediums corrected as the +[checkpoint](#wave-3-step-9b2-implementation-checkpoint--2026-10-25) records: +- the rigor pass: the loader's exported seams could brand any key, the import control matched three spellings, and + half the recipient pair was untested; +- round 1: the rigor fix's resolved destination dropped a 9b.1 refusal, stray destination entries led to an unsealable + ciphertext, and a bare specifier escaped the import control; +- round 2: a nested package script escaped the import control, and a trailing separator bypassed the link refusal; +- round 3: four untested paths or control gaps, in the link failure, a linked tarball, the trailing anchors and a + template literal. + +Round 4 was a NEW independent complete cumulative review of `819f8cbb..563402e6`, with every reviewer in its own +worktree detached at that head. It reported no blocker, high or medium finding and no regression. Its separate +crypto/custody security sitting, run at the same head as its own reviewer and report, found lows only. The +integration lens and the critic were clean. The plan lens confirmed the inventory pin at every commit and that no 9b.1 +module, product path, receipt, guard or workflow changed. + +**Author evidence at `563402e6`.** The full `pnpm run ci` passed plain and alone (exit 0): `test:acceptance-tools` +1,024 of 1,024, all 25 product map controls, and every causal-control family. On Ubuntu 24.04 with GnuPG 2.4.4 (in +Docker, a fresh clone, a non-root user), the 90 tests of the 9b.1 custody files and the four release files ran: 89 +passed and the case-insensitive-volume case skipped. + +The round's verified findings were twelve lows and one nit: +- Two are corrected in the acceptance commit, which changes no code. The contract's diagnostics bullet no longer + says every host failure on the destination is `PLACEMENT`: a destination that cannot be listed refuses `UNEXPECTED` + (PLAN-9B2-R4-3, SIT9B2-R4-L2, COR-R4-1). The `custody.mjs` control's claim now names the escapes it decodes + (PLAN-9B2-R4-1, COR-R4-2). +- The rest are recorded in + [deferred-tasks.md](../deferred-tasks.md#wave-3-step-9b2--recorded-low-findings-2026-10-26), each with its example + and re-check. + +Step 9c was next; Wave 3 was paused on 2026-10-26 behind the CLI MVP ([MVP priority reset](#mvp-priority-reset--2026-11-04)). + +## MVP priority reset — 2026-11-04 + +**Wave 3 is paused after Step 9b.2.** On 2026-10-26 the maintainer stopped the Step 9c preflight and redirected the +project to an MVP: features and bugfixes that a CLI user feels, in `apps/cli` and `packages/core` first. Steps +9c–19 (25 of the 32 increments; the plan grew from 29 with the 9a.2, NI and 9b splits) are **pending, not +declined**: each keeps its row, its recorded ADR amendments and its security sittings. The live order is +canonical in [current.md](../current.md#next-work--the-cli-mvp); this section records the evidence, the decisions +and the amendments to this register. + +**Evidence.** A read-only multi-agent review at `development` `5a328bca` read every roadmap document and the code: +eight scoped readers (661 open items), eight adversarial verifiers (three re-run on 2026-11-04 after a spend +limit), a synthesis and a completeness critic. The detailed reports and a Turkish decision register stay local and +Git-ignored. Its main findings, each re-checked against the code before the decisions: + +- The engine is solid; the agent and presentation layers are not. The built-in chat agent grants only + `read_file`, `list_directory` and `git_status` (`apps/cli/src/chat/default-agent.ts:35`) and starts in `ask` + (`apps/cli/src/chat/chat-mode.ts:31`); no edit, search or find tool exists; `run_command` is refused before any + approval unless allowlisted (`packages/core/src/tools/registry.ts:762-770`). +- Run results are invisible on the human surfaces (no outputs, no run id, no finished-run listing, crashed runs stay + `running`); replies are raw markdown; a resumed chat opens empty; Anthropic chat turns default to a 4,096-token + output estimate (`packages/llm/src/output-cap.ts:37`). +- Nothing newer than `0.1.1` can be published: the Step 9a.1 guard sets `process.exitCode = 1` before evaluating + (`tools/cli-package/release-guard.mjs:13`), and it exists only on `development` — `main` has no guard. +- The published `0.1.1` prints unsanitized model output in its `relavium run` view (#56); `LICENSE` §2 grants no + usage right without a written agreement. + +**Decisions (maintainer, 2026-11-04).** D58–D72 are recorded in the +[decision table](../current.md#wave-gating-decisions) with their rulings. In short: the larger MVP (v0.2.0 after +MVP-1…MVP-10, including conversational authoring, web, the Home hub and MCP management; milestone M3.5); release +path B (D59); the license unchanged (D60); a scoped `development`→`main` checkpoint now (D61); the default agent +may write, with `[chat].default_mode` (D62); a `run_command` approval card with remembered grants outside the +repository (D63); a tiered acceptance bar (D64); six recorded splits plus #61 (D65); `current.md` reduced to live +status (D66); the CR-71 residual un-deferred and CR-70 after the release with ADR-0096 §5 (D67); first-run defaults and one +keymap ADR (D68); authoring → web → Home → MCP (D69); no pre-release (D70); the parity items in the MVP (D71); no +action on `0.1.1`'s #56 (D72). + +**Recorded splits (D65), ID coverage unchanged.** Each split keeps its row's recorded ADR amendment and security +sitting, and lands in the named MVP milestone: + +| From | Split | Lands in | Order amendment | +|---|---|---|---| +| 10b | Excess positional arguments refused on every non-variadic command | MVP-2 | — | +| 15a | #65: tabs expanded to 8 cells before width measurement | MVP-2 | Lands before #58/#62/G40/G51 (row 15's strict arrow amended) | +| 15c | #63: 80×24 on both chat surfaces with no hidden affirmative input there or on the Home strip (approved ADR-0057 amendment, approval-path sitting); with it, the Home strip row budget (2.6.F's open obligation) | MVP-3 | Lands before 15a/15b's remaining items | +| 11b | Ctrl-C at a pending approval rejects it | MVP-4 | Lands before 11a. The 2026-10-19 "maps to `/cancel`" wording is superseded by the keymap ADR (D68): reject first, then the ordinary interrupt semantics; the approval-path sitting stays | +| 16b | #51: the persistent, current-state keyless nudge (keychain/config sitting) | MVP-2 | Lands before 16a | +| 17a | #24: the one-shot stdin hint | MVP-2 | — | +| 13a–13c | G30: chat HTTP authority under ADR-0112 (egress/config-trust sitting) | MVP-7 | Row 13's "after decomposition" no longer holds: 13a/13b land before Step 12; 13c is re-scoped when MVP-7 starts because 12b's shared helpers do not exist yet | +| Wave 5a | #61: full-screen word-wrap | MVP-2 | Its Wave 5a ownership ([later-wave ownership](#existing-later-wave-ownership-and-approval-gate)) ends | + +**Amendments to the 2026-10-19 rulings** (dated, the original text kept): + +- *Acceptance (D64):* tiered. An increment touching an approval or consent path, egress, keys or secrets, + filesystem writes, a new terminal sink for untrusted content, the lease or resume paths, or release and + publication keeps the 2026-10-23 bar — a rigor pass, one NEW complete cumulative round with no blocker, high or + medium finding and no regression, and a separate security sitting. Other UX, rendering and command increments are + reviewed per MVP milestone: author self-review, `pnpm run ci` plus `pnpm coverage:enforced`, and one independent + scoped round with no high or medium finding; lows go to [deferred-tasks.md](../deferred-tasks.md). +- *Publication (D61):* a scoped `development`→`main` checkpoint follows this reset; the Wave 3 PR still follows + Step 19 when the wave resumes. +- *Interim release guard (D59):* to be superseded by a new pre-1.0 release ADR (option B) with a sunset at 1.0 or + when 9c/9d land, whichever comes first. The guard stays in force, in code and procedurally, until that ADR is + Accepted and its guard change passes a security sitting. + +**Recorded risks.** The published package grants no usage right (D60). The published `0.1.1` keeps #56's +unsanitized `relavium run` output; the maintainer accepted it because the product has no users yet (D72). + +**NI's Windows clause is closed** (2026-10-26): run 3 of 3 is recorded in the +[NI acceptance](#wave-3-ni-acceptance--2026-10-25). + +## Execution record moved from current.md — 2026-11-04 + +> Moved verbatim from `docs/roadmap/current.md` on 2026-11-04 (decision D66) when that page was reduced to live +> status and order; only relative links were rewritten for this directory and heading levels lowered by one. +> It keeps the 2026-07-26 execution plan's baseline, axes, Waves 0–1 and the PR #81 closing list — the only +> definition of `#W15-1`…`#W15-24`, which code comments cite. The live order is in [current.md](../current.md). +> This first pass moved only that plan; the records it left out follow in +> [the PR #96 review correction](#moved-by-the-pr-96-review-correction--2026-11-04). + +### Execution order — Phase 2.5.5 + Phase 2.6 (temporary) + +> **Temporary section — delete it when both phases close.** It owns **ordering only**. Every item's +> definition, size, file list and finding ids live in its phase file — +> [phase-2.5.5](phase-2.5.5-hardening-and-remediation.md) (176 work items, sub-streams A–I) and +> [phase-2.6](phase-2.6-conversational-authoring.md) (15 open workstreams) — and nothing below +> restates them. Items are cited as ` · ()`, which +> greps directly against the phase file. +> +> Derived 2026-07-26 from a full pass over both phase files, their own sequencing sections, and the live +> tree. Where it departs from a phase file's own prose, see [Departures](../current.md#departures-from-the-phase-files-own-sequencing). + +#### Ordering root — one baseline ✅ **discharged 2026-07-26** + +The plan opened on a divergence neither phase file modelled: Phase 2.5.5's whole backlog is written against +`f88b0e8`, which existed only on `development` behind PR #76, so every `file:line` citation in it resolved +against a tree nobody was branching from. **PR #76 and #77 are both merged** — `f88b0e8` is on `origin/main`, +`origin/development` and `origin/main` held identical trees with no open PRs as of 2026-07-26. Every +citation in this plan now resolves. (PR #79 and PR #80 have opened since.) + +One consequence survives and is worth keeping until the phases close: + +> **Branch from `origin/development`, never from a local `main`.** A long-lived `development` merged into +> `main` by merge commit leaves a stale local `main` behind after every PR, silently and without conflict. +> Run `git fetch` and cut from `origin/development`. + +Also settled while discharging this: `main` now carries branch protection (PR required, `lint · typecheck · +test` required, force-push and deletion blocked) and a `v*` tag ruleset restricting tag creation to the +maintain/admin roles — closing the Phase-0 obligation that made `ci.yml`'s own "REQUIRED status check" +comment untrue. `release.yml` asserts its ref is reachable from `main` before `pack`, so **2.5.5.H's `G26` +is closed ahead of Wave 0**. + +#### The three axes this order is built on + +1. **Irreversibility** — a defect whose damage a later fix cannot undo jumps every queue. That is Wave 1: + a credential already written into a durable `--json`-visible row, a billed turn lost after the reply + rendered, model-controlled ANSI inside the consent card, and the states where the cost cap is silently off. +2. **File ownership** — whoever edits a file next inherits everyone else's rebase. Every god-file + decomposition and naming/convention ruling lands *before* the 2.6 workstream that piles onto it, and + eleven items are pulled across the phase line because they edit the exact function another item restructures. +3. **Certification** — a wave that discharges a phase obligation certifies it while the evidence is fresh, + which is why 2.5.5's exit criteria 1–3 close in Wave 2 rather than at the end. + +Waves are readable single-branch sequences. Concurrency is expressed as *"these queues are file-disjoint, +any order"* — never as headcount. + +```mermaid +flowchart TD + W0["Wave 0 — One true baseline
baseline ✅ · CI truth · numbers"] + W1["Wave 1 — Stop the bleeding ✅
3 CRITICALs · cost cap · ADR-0074"] + LEDGER["#W15-1 — realized-cost ledger ✅
ADR-0076 + ADR-0077"] + P265["Phase 2.6.5 — Core reliability
51 CR items · 8 P0 blockers behind ADR-0078–0084
W0–W8 complete · W3 blocker closed in W8
W4 merged — the hostile-MCP class, ADR-0088
W5 merged — media correctness, ADR-0089+0090
W6 merged — authoring correctness, ADR-0091–0094
W7 merged — PR #90 · W8 merged — PR #92
49/51 closed · approved deferrals retained"] + W2["Wave 2 — Shut the doors ✅
PR #94 merged · fs jail · secrets · config trust
2.5.5 EXIT 1–3 satisfied"] + W3["Wave 3 — Clear the ground
god-file decomposition · CLI net"] + W4a["Wave 4a — The spine
2.6.A/D/H/K + 2 ADRs"] + W4b["Wave 4b — Money floor & close-out
2.5.5.B/C/F/H drain · v0.2.0"] + W5a["Wave 5a — Paint before you build
render-v2 · theme · config"] + W5b["Wave 5b — Home becomes the product
2.6.G/I/J · then i18n sweep"] + W6["Wave 6 — Hands, voice, lineage
2.6.M/B/E + 2.6.N foundation"] + W7["Wave 7 — Orchestration & the gate
2.6.O/P · go/no-go → Phase 3"] + W0 --> W1 --> LEDGER --> P265 --> W2 --> W3 --> W4a --> W5a --> W5b --> W6 --> W7 + W3 --> W4b + W4b -.->|v0.2.0 release| W7 + W4a -.->|file-disjoint lanes| W4b +``` + +#### Wave 0 — One true baseline + +Collapse the divergence, make the required gate execute the artifact it ships, and fix the review +checklists before ~30 security-gated PRs are reviewed against them. + +1. ✅ **Merge PR #76** (2.6.Q P1–P5, ADR-0071/0072, the three picker bugs, the param self-heal) — **done + 2026-07-26**, along with #77 (this plan + the Phase-2.5.5 opening + `release.yml`'s `G26` ancestry gate). +2. ✅ **Reserve numbers** — **done**, recorded here: migrations end at `0012` and ADRs at `0072` (verified), + so `0013` = **ADR-0074 §4's conservative money columns** (`agent_sessions.total_conservative_microcents` + `session_costs.conservative_microcents`) — the number was freed when D8 ruled against the approval-preview scrub and is RE-USED here rather than left as a hole · `0014` = 2.5.5.C's enum CHECKs (#101) · `0015` = 2.6.H · + `0016` = 2.6.G's pins · `0017` = 2.6.N lineage; **ADR-0075+** for the nine unwritten 2.6 ADRs (`0073` = the history.db migration lock, `0074` = durable conservative budget commitments — both landed in Wave 1). +3. ✅ **One `ci.yml`/`turbo.json`/`tsup.config.ts` PR** — **done 2026-07-29 (PR #80)**, all eight items: + 2.5.5.H · the required gate never runs the compiled binary + undeclared `drizzle` output (#294, #315) → + local `pnpm run ci` vs `ci.yml` divergence (#312 — and `pnpm ci` is shadowed by a pnpm builtin, so the + script was unreachable by the name everyone types) → the coverage-floor **ruling and its implementation** + (#296, #152) → the `(advisory)` labels (#320) → `THIRD_PARTY_EXTERNAL` (G27, #248) → bundle-closure + single-chunk assert (#314) → `sync:models:check` (#317). *(`release.yml`'s ancestry check, `G26`, landed in + #77; its tag-protection half is configured.)* The PR also closed the four blockers from the 2026-07-29 + roadmap review, and its own CI caught two real defects in it — a TS4111 that `turbo run typecheck` does not + cover, and `pnpm ci` being unreachable because pnpm reserves `ci` as a builtin. +4. ✅ **2.5.5.F · three skills' hardcoded foreign-project paths** (#162) — **done 2026-07-29.** Filed as docs, + actually a functional bug: they pointed at `/Users/dev/…/Agent-Organizer/`, so `add-package`'s `mkdir -p` + would have created a tree outside the repo. All three now anchor on `$(git rev-parse --show-toplevel)`. +5. ✅ **One markdown PR** — **done 2026-07-29**: `packages/mcp` added to all five inventory sites (#128, #129, + #153, #163, #254), the reviewer agent's secrets item extended to the shipped CLI floor (#164), the + security-review skill's SSRF step now requires reusing the shared guard rather than re-deriving one (#167), + and the i18n clause settled by the D5 ruling (#74, #260, #134). +6. ✅ **The locale-bar amendment** (D5, `en`+`tr`) and the **snapshot regen** (D3) — **done**, propagated to + all seven locale sites and merged as PR #79 (two price rises taken, nine retirements accepted). + +> ✅ **Wave 0 is COMPLETE (2026-07-29).** The required gate now executes the binary it ships, +> `apps/cli/drizzle/**` is a declared turbo output, and `lint · typecheck · test` + +> `engine coverage floor (llm, mcp)` are both required checks under branch protection. In phase terms this +> closes **8 of 2.5.5.H's 14** tasks and **3 of 2.5.5.F's 20**. **Wave 1 followed and is also complete +> (2026-07-30, PR #81) — see below; Wave 2 is next.** + +#### Wave 1 — Stop the bleeding + +All three CRITICALs, plus the states where `max_cost_microcents` is silently not a cap. Four file-disjoint +lanes, any order between them, plus the money hole they left open. + +**✅ Wave 1 is COMPLETE — lanes (a)–(d) on 2026-07-30 (PR #81), lane (e) on 2026-08-09 (`#W15-3`).** + +> **Superseded 2026-08-09 — kept because the correction is the record.** Lane (e) closed with `#W15-3`: +> `/cost --release` makes §1's escape hatch reachable, and `/cost` renders the durability state. The +> retraction below was accurate when written and is no longer; it stays so the history of the false +> completion claim is not quietly erased. +> +> **Retraction (2026-07-30).** This section previously said Wave 1 was complete and ADR-0074 fully implemented. +> That was wrong on lane (e) and I am correcting it rather than leaving it: **§1's release escape hatch is +> exposed on `GovernorWiring` but reachable from no command, REPL action or Home affordance**, and `/cost` never +> renders `conservativeDurabilityBroken`. §1 names this exact state as forbidden — "a single usage-less response +> would permanently shrink a long-lived chat's cap with no way out" — and this PR REMOVED the accidental escape +> a restart used to provide. A section whose compensating bound is unreachable is not implemented, and marking +> it done made four other documents inherit the error. Two independent reviews flagged it; both were right. +> +> Lane (e)'s remaining work, and two defects it exposed that ADR-0074 does not cover, are tracked in +> **[PR #81's closing list](#pr-81s-closing-list-and-the-two-items-that-need-an-adr)** below. Every lane below is verified against the shipped code, not +against its own task text — the check for each is stated inline. Two gaps are recorded rather than closed; both +are named in their lane and marked in the source, and neither is a shipped defect: + +1. the engine-level composition test for ADR-0074 §3's abort-breakable media-job hold (`engine.ts`, at the abort + listener) — the governor's own state machine is mutation-verified, the composition with `#countRunning`'s + termination gate is not; +2. the CLI-layer test that `restoreConservativeCost` is wired through `buildSessionRuntime` → `GovernorWiring` + (`session-host.test.ts`) — `AgentSession.resume` calling the hook IS mutation-verified in `packages/core`; + only the spread that supplies it is unproven. + +- ✅ **(a) `registry.ts`** — 2.5.5.A · **redact tool-approval previews like `sanitizeInput()` already does** + (#91). *(The originally-planned migration 0013 scrubbing already-persisted previews was ruled OUT — see D8.)* *Closed: `previewFor()` now scrubs the DISPLAY copy while `rawPreviewFor()` keeps an + `unredactedPreview` for in-process classification — splitting the two uses, because a whole-string scrub + flipped the protected-path decision and a per-segment one reopened the leak. Verified: `registry.ts` and + `tools/types.ts` both carry `unredactedPreview`.* +- ✅ **(b) `packages/db` chain, strict** — 2.5.5.C · `SQLITE_BUSY_SNAPSHOT` (#100) → async backoff replacing + the blocking `Atomics.wait` (#226) → **the chat-persister safety net** (#228, CRITICAL) → + serialize `runMigrations` (#99) co-landed with the unconditional 0600 self-heal (#28, #33) → + `createClient` typed error + PRAGMA docstring (#104, #105). `deprecationDate` crash guard (G1) is a + standalone one-liner, droppable anywhere. +- ✅ **(c) render/signals** — 2.5.5.I · **terminal-control sanitization to the four boundaries it never + reached** (G34, G44, #56, #57, CRITICAL — one PR at the shared helper, its own security review) → + `SIGTSTP`/`SIGCONT` + onboarding signal-registration ordering (G0, #50). *Closed at all four boundaries the register names, verified by call count: + `RunApp.tsx` (#56) 9, `final-summary.ts` (#57) 6, `render-error.ts` (G34) 4, `clack-prompter.ts` (G44) 4. G0's + run/gate `SIGTSTP`/`SIGCONT` is `render/run-job-control.ts`; #50 is closed by ordering — + `drive-home.tsx` subscribes signals at `:808`, before `runOnboardingWizard` at `:855`.* +- ✅ **(d) money, split across two PRs** — `budget-governor.ts` only: 2.5.5.A · reservation ledger under + `max_parallel` (G38) → re-armed `budget:warning` + projected `thresholdPct` (G39, G47, G49) → **2.6.Q's + strict-cap half**. Then `fallback-chain.ts`/`cost-tracker.ts` only: 2.5.5.B · `maxRetries: 0` + jitter + + `Retry-After` (#276, #279) → narrowed `#emitSuccess` catch + `CostTracker` bounds (#194, #198) + + **2.6.Q's realized-cost half**. + +- 🔶 **(e) ADR-0074 — durable conservative budget commitments (§2–§5 shipped; §1 INCOMPLETE)** · the money hole the other four lanes leave open: + a bounded estimate retained when a provider may already have billed but returned no trustworthy usage was + kept only in memory, so a crash or a resume reopened a strict cap against money that may already be owed. + §5 (a forward-compatible stored-event READ, the precondition for emitting anything new) → §2 (the + `budget:estimate_committed` contract, the governor's durability barrier, the checkpoint fold and the resume + restore) → §3 (freeze an async media job's accepted units + price) → §4 (the session's conservative + aggregate + per-model attribution + migration + transactional persister write). + +> 2.6.Q's cost-cap item is **split**: its task text spans both `budget-governor.ts` and `fallback-chain.ts`'s +> `#emitSuccess` — the same catch 2.5.5.B narrows. The two halves join two different PRs, not one. +> **Closes M2.5.5-1.** +> +> **Outstanding inside lane (e) — restated at close, because §4 changed the facts.** ADR-0074 §1 ties a +> commitment's release to the surface that renders it. +> +> **Rendering: done on all three surfaces.** The run TUI and `relavium logs` shipped with §2; `/cost` shipped +> with §4's fold and reads *estimated, possibly billed*, with `(no completed call)` for a model that has one +> without a completed call. All three are mutation-verified. +> +> **Release: reachable, as of `#W15-3` (2026-08-09).** `/cost --release` on the chat surface — the one that +> renders the amount, which is what §1 ties it to — clears the durable per-model and aggregate columns first +> and the live governor second, so a released commitment does not return on the next `chat-resume`. `/cost` +> also renders the durability state. This note previously said no affordance called it; that was true and no +> longer is. +> +> The reserved `budget:estimate_released` event stays **reserved and unemitted**, and that is a decision, not +> an omission: the session path restores its conservative total from COLUMNS rather than by replaying events, +> so zeroing them is what makes the release durable there. The event is needed only by a workflow-run release +> surface, which does not exist — a run is not long-lived, and `on_exceed: pause_for_approval`'s per-node +> bypass or raising the YAML cap are §1's own analogy. §1's dated note in +> [ADR-0074](../../decisions/0074-durable-conservative-budget-commitments.md) carries the same statement. + +#### PR #81's closing list, and the two items that need an ADR + +Two independent adversarial reviews of PR #81 (2026-07-30) found that the cap's core safety claim does not yet +hold across a process boundary. + +**Almost all of this belongs to PR #81 itself, not to a later wave** — a first framing filed it as one, which +was wrong. The test is simple: this PR either ADDED the code a finding is in (`background-failure.ts`), or +claimed to have closed the area it sits in (the terminal-safety boundaries, ADR-0074 §1), or REMOVED a +protection without replacing it (§1's escape hatch, which a restart used to provide). Shipping a regression and +deferring its fix is not a schedule decision. + +**Only §A is genuinely separable.** Both of its items need an append-only ADR before any code — one changes what +"spent" means across a crash, the other supersedes a decision ADR-0074 §5 already recorded. Those earn their own +PRs. Everything from §B down is #81's to close. + +Ordered by whether the repo currently states something untrue, then by blast radius. + +**Every item names the check that would close it, not just the defect.** + +> **Status — 24 of 24 closed and MERGED, across two PRs.** Everything marked ✅ below is fixed and +> break-verified with the mutation confirmed applied; the two PRs are recorded separately because the closure +> dates differ: +> +> - **23 items merged to `main` via PR #81** (2026-08-09). +> - **`#W15-1`, the last one, closed 2026-08-10** behind ADR-0076 + ADR-0077 (see §A) and **merged to `main` +> via PR #82** (2026-08-11), together with the first Phase 2.6.5 batch. +> +> §E's six coverage gaps are all closed — `#W15-16`'s composition test fails by TIMING OUT when the abort +> listener is removed, which is the unkillable run reproduced exactly rather than an assertion standing in +> for it. +> +> **`#W15-2` closed 2026-08-09** as [ADR-0075](../../decisions/0075-fail-closed-resume-on-an-unreadable-event-log.md), +> amending ADR-0074 §5: a read that feeds a REPLAY refuses when any row was skipped, a read that feeds a +> DISPLAY stays tolerant. It landed first on purpose — `#W15-1` adds a new durable event type, and an older +> binary reading it falls straight into §5's skip path, so this decision governs how that event degrades. +> +> **One gap this closing deliberately left open**, named here rather than implied: ADR-0075's refusal is +> pinned at `gate.ts`'s own catch (where the blocker was) and at `toUserFacing`, but NOT through +> `dispatch.ts` / `specs.ts` to the process exit. A re-wrapping `try/catch` added in either layer reproduces +> the same blocker and leaves both tests green. Closing it needs a `run(argv, io)` drive with a redirected +> HOME and a seeded on-disk `history.db`; no such harness exists for `gate` yet, and the marker sits at the +> test it concerns. +> +> **`#W15-1` is what remains, and its DECISION is now made**: +> [ADR-0076](../../decisions/0076-durable-per-attempt-realized-cost-ledger.md) (Accepted 2026-08-09) — an +> additive durable run event recording one settled provider attempt, emitted through ADR-0074 §2's barrier and +> folded into a `run_costs` row at persist time. It landed after ADR-0075 on purpose: a new durable event type +> is exactly the input that decision governs. +> +> **The implementation is staged, and starts here.** In order, because each step is a precondition for the +> next: +> +> 1. `packages/shared/src/run-event.ts` — `cost:attempt_settled`'s schema, its union arm, and +> `parseStoredRunEvent`. Until this lands nothing else compiles. +> 2. `docs/reference/contracts/sse-event-schema.md` — the canonical shape + which of the two cost events is +> authoritative (ADR-0076's last Negative names that drift risk). +> 3. `packages/db/src/run-history-store.ts` — the `applyDerived` arm writing the `run_costs` row in the SAME +> transaction as its event. The `node:completed` delta then telescopes to zero on its own; a test must pin +> that `SUM(run_costs) == runs.total_cost_microcents` still holds, since that is the invariant carrying the +> no-double-count claim. +> 4. The attempt boundary and its barriers — **corrected by +> [ADR-0077](../../decisions/0077-realized-cost-ledger-uses-the-conservative-commitment-barrier.md)**. ADR-0076 +> §1 asked for an `await` at the settle point; there is no `await` to be had there. The seam's observer is +> `onAttempt?: (record) => void` — synchronous, void — and BOTH money events are emitted from that one +> callback a few lines apart, which `budget-governor.ts` already states outright ("which cannot await"). So +> the ledger takes ADR-0074 §2's shape: **start** the write at the settle instant on a chained in-flight +> promise, **join** it at three barriers — before the next egress admission, **before tool dispatch** +> (`agent-turn.ts`'s `await dispatchToolCalls(...)`, the barrier this adds beyond §2), and at the turn/node +> terminal. Every barrier awaits AND observes the failure, because `#emitDurable` is total for store faults +> and resolves. +> 5. `packages/core/src/engine/checkpoint.ts` — **corrected during implementation**. The instruction here was +> "fold it as a SUM of deltas", borrowing ADR-0074 §2's reasoning, and the wrong half of that cost two +> rewrites. Summing into the realized total double-counts (a node terminal's snapshot already contains its +> attempts); summing into a separate accumulator and maxing the two families UNDER-counts (a media node +> writes a snapshot and emits no attempt row, so every attempt after the last boundary vanishes). What is +> correct is the fold already two arms above it: `Math.max` over every durable ABSOLUTE total. §2's +> rejection of `Math.max` does not carry — it was rejected there only because a release can DECREASE a +> conservative total, and realized spend is monotonic. +> +> **STATUS: all five steps are landed** (`8d7ffcf` … `50c60bd`), each with an Opus and a Sonnet round folded. +> The break-verify that matters most was step 4's ordering, and it now holds: with the tests draining to +> quiescence before asserting, deleting B2 reddens the tool-dispatch test and deleting B3 reddens the node- +> terminal test. B1 is masked by B2 on the current fixture (the second egress only comes after the tool), so +> it is covered by construction rather than by a red — named rather than implied. +> +> **This is now the prerequisite of Phase 2.6.5, not a parallel track.** Its five steps touch the same four +> files `CR-10` (ordered append tail) and `CR-12` (effect journal) restructure, so it lands first — see the +> 2.6.5 section below. +> +> **What ADR-0076 explicitly does NOT close**, so it is not read as more than it is: the cost of a TOOL EFFECT. +> A resumed run can still re-execute an `http_request` POST, a `run_command` or an MCP mutation — the +> duplicate EFFECT is the harm, and no cost bookkeeping prevents it. That needs a durable effect journal with +> a prepare/receipt pair, which is a different decision about a different failure and earns its own ADR. It is +> the natural next one after this chain (forward-compatibility → conservative commitment → realized cost). +> +> Two decisions inside the closed set are worth surfacing here rather than leaving in a commit message. +> `#W15-4` **changed a previously-tested behaviour**: #228 pinned that a session survives a failed durable +> write and the next turn is clean; it now stops instead, because continuing to spend above a transcript that +> has fallen behind is the defect. And `#W15-23` is scoped to `run-history-store` only — the other +> `packages/db` stores share the same outer-`db`-inside-a-transaction convention, which remains open, is +> stated here rather than implied closed, and has its own PR queued after this one. + +##### A. Needs an ADR first — the only part that is NOT PR #81's to close + +- ✅ **`#W15-1` (blocker) — a workflow's REALIZED cost is not durable at the provider-attempt boundary.** + *Closed 2026-08-10 by ADR-0076 + ADR-0077 and the five staged steps; the barrier is a chain-and-join, not + the inline await ADR-0076 §1 first specified. See the staged plan below for what each step landed and the + three carried-forward gaps.* + `engine.ts` folds `cost:updated` into memory and streams it; the store documents that it is never persisted, + and the checkpoint can only recover cost from a LATER `node:completed`/gate snapshot. So: a paid call + succeeds → the model asks for a tool → the process dies during the tool → the realized cost is gone and the + resumed node re-runs the same paid call, against a cap that has forgotten the first one. ADR-0074 makes only + the CONSERVATIVE commitment durable; a call that returned trustworthy usage has no equivalent barrier. + *Needs an ADR: an idempotent per-attempt durable ledger, written before the next tool side-effect, the next + egress and the turn/node terminal. Not a local fix — it changes what "spent" means across a crash.* +- ✅ **`#W15-2` (blocker) — a tolerant read is fail-OPEN on the resume path.** ADR-0074 §5 drops an event whose + `type` an older binary does not know, and `checkpointer.ts` builds resumable state from what remains. An + older binary cannot know whether the dropped row was a node terminal, a job submission, a gate decision or a + cost commitment — so it may re-run completed or already-submitted work. Preserving the sequence high-water + mark prevents a UNIQUE collision; it does not restore lost state. Tolerance is right for `logs`/`status` + (read-only); it is wrong for a replay that changes the world. + *Needs a superseding ADR (§5 is append-only): fail closed on the resume path with "a newer Relavium is + required" when any row was skipped, while read-only surfaces stay tolerant — or classify events as + state-bearing vs not, and tolerate only the latter.* + +##### B. PR #81 — ADR-0074 §1, the half that makes the rest safe + +- ✅ **`#W15-3`** (blocker) — the release is exposed but unreachable, and nothing renders `durabilityBroken`. + `GovernorWiring.releaseConservativeCommitments` and `conservativeState()` exist and are tested; no command, + REPL action or Home affordance calls either. §1 requires the surface that RENDERS the amount to also expose + clearing it, and this PR removed the accidental escape a restart used to provide. + *Closed by `/cost --release`: `/cost` renders the durability state, and the release clears the per-model AND + aggregate columns before the live governor, so it survives `chat-resume`. The durable + `budget:estimate_released` named here was NOT needed — the session path restores from columns, not from an + event replay — so it stays reserved; see the lane-(e) note above.* + +##### C. PR #81 — data integrity, each with a concrete failure + +- ✅ **`#W15-4`** (blocker) — session persistence failures are invisible to the producer. `RunEventBus` isolates + listener errors by design, so a persister that cannot write still lets the turn complete. Two consequences: + `persister.ts` advances the in-memory cost total BEFORE the DB write, so a failed `recordSessionCost` leaves + the durable total low and a resume restores a cap that has forgotten real spend; and a failed `commitTurn` + leaves the staged turn uncleared, so the next `beginUserTurn` overwrites the previous user message and the + durable transcript loses an exchange the in-memory session still has. + *Closes when: cost and transcript persistence move to an awaited durability port (UI subscribers may stay + passive), and a persistent write failure puts the session in a durability-degraded state that refuses new + egress rather than reporting success.* +- ✅ **`#W15-5`** (high) — stored payloads are not checked against the authoritative columns. `readEventLog` + selects `seq`/`event_type` alongside the payload but never compares them to the parsed event's own `runId`, + `sequenceNumber` and `type`. A row with `seq = 2` carrying `sequenceNumber: 1` is accepted, leaving the + high-water mark wrong and the resume colliding on the next write. + *Closes when: every denormalized projection is compared after parse and any mismatch is a + `CorruptRunEventError`.* +- ✅ **`#W15-6`** (high) — terminal cost snapshots on failed/cancelled runs are dropped by the read models. + `node:failed`, `run:failed` and `run:cancelled` all carry cost in the shared schema, but the store folds + none of them, the checkpoint folds none, and the run TUI's summary ignores the terminal figure. A sibling's + paid media cleanup after a root failure exists ONLY on `run:failed.cumulativeCostMicrocents`, so replay and + history lose it. Pre-dates this PR, but the PR touches all three files and claims cost safety. +- ✅ **`#W15-7`** (medium) — `media_job:submitted` accepts a half-frozen basis. `units` and + `acceptedCostMicrocents` are independently optional, so a row with a frozen cost and no frozen units resumes + with the old reservation and a re-derived volume. The invariant is not "both or neither" (the approved-bypass + case legitimately has units only) but **`acceptedCostMicrocents` present ⇒ `units` required**. + +##### D. PR #81 — security and correctness + +- ✅ **`#W15-8`** (high) — the new failure-reporting paths can leak credentials. `background-failure.ts` prints an + arbitrary rejection message and, under `RELAVIUM_DEBUG`, a raw stack; `sanitize.ts` strips ANSI/control/bidi + but performs no secret redaction. Same class in the session listener notice, `render-error.ts`'s verbose + stack and `index.ts`'s fatal debug stack. The rejection can be an MCP error, a provider response body or a + tool result — all of which carry keys, `Bearer` headers or credentialed URLs. Existing tests plant ANSI, never + a secret. + *Closes when: the shared secret-shape redactor runs BEFORE terminal sanitization on every one of those paths + (stacks included), with tests planting `sk-*`, `Bearer`, `Basic`, credentialed URLs, query tokens and PEM.* +- ✅ **`#W15-9`** (high) — the stream path can throw out of a contract that says it never throws. + `#runStreamAttempt` calls `#emitSuccess` OUTSIDE its try/catch (the `generate()` path has it inside), so a + provider returning non-integer usage makes `assertAccountableUsage` throw raw out of the async generator — + and `streamOneTurn`'s `for await` has no try/catch, so it crashes the turn unclassified, AFTER the content was + produced and paid for. +- ✅ **`#W15-10`** (high) — `--json` output is not sanitized. `status --json` / `gate list --json` carry + model-controlled text; `JSON.stringify` does not escape Unicode bidi overrides (CVE-2021-42574). The human + render path is sanitized; the JSON branch is not. `createJsonRenderer` is the same gap on the NDJSON path. +- ✅ **`#W15-11`** (high) — `createPlainRenderer` sanitizes AFTER splitting on newlines, so an embedded newline + still forges a line. Every other boundary this PR touched sanitizes first; `renderer.ts` is the exception. +- ✅ **`#W15-12`** (high) — `isBlankPreview` treats a fully-redacted preview as informative (`chat-mode.ts`), so a + user pressing "always" on `Approve write to [redacted]?` auto-approves every `write_file`/`run_command`/ + `http_request` for the session, having been shown nothing. +- ✅ **`#W15-13`** (medium) — the config reader chmods PROJECT configs to 0600. `reassertOwnerOnly` runs on every + layer, but `config-spec.md` says project files are committed and shared. Reading a config mutates repo file + permissions, and `statSync`/`chmodSync` follow symlinks. Self-heal belongs to the canonical global config + only. +- ✅ **`#W15-14`** (medium) — a provider `Retry-After` sleep is not abort-aware. `fallback-chain.ts` passes the + delay straight to an injected `sleep` with no signal, and the real hosts use a plain `setTimeout`. A cancel + during a 60 s clamped wait hangs until the timer fires. The existing abort test resolves `sleep` immediately, + so the window is untested. The docblock also says an oversized value becomes `undefined`; the code clamps. +- ✅ **`#W15-15`** (medium) — corruption is reported to a multi-run view as "no data". `readPerRunOrDegrade` + returns only the fallback, so `gate list` can print "No pending human gates." for a run whose gates were lost + to corruption. It must return the degradation alongside the value, with a warning in human output and a + machine-readable marker under `--json`. + +##### E. PR #81 — coverage gaps it shipped knowingly + +All six are now CLOSED and their in-source markers removed. None had been a shipped defect — each had +mutation-verified coverage one layer down — but closing them surfaced one that was: `onLegacyMediaJobHold` +reached `WorkflowEngine` from a real `gate.ts` sentence and the constructor never read it, so ADR-0074 §3's +hold notice was dead and a `relavium gate` resume of a legacy parked media job stalled silently. Found by +the review of `#W15-16`, fixed with it, and now the signal that test drives. + +- ✅ **`#W15-16`** — the engine-level composition test for §3's abort-breakable media-job hold; deleting the abort + listener leaves the whole suite green. +- ✅ **`#W15-17`** — the CLI-layer test that `restoreConservativeCost` is wired through `buildSessionRuntime`. +- ✅ **`#W15-18`** — the `coalesce` backfill of `session_costs.model_catalog_id` (needs a `model_catalog` FK row + the block does not seed). +- ✅ **`#W15-19`** — that a chat turn's terminal is emitted only AFTER `flushBudgetCommitments` resolves. +- ✅ **`#W15-20`** — the H3 approved-bypass `acceptedCostMicrocents` omission; reverting it leaves the suite green. +- ✅ **`#W15-21`** — `agent-run.ts`'s `attachConservativeWriter`; deleting it silently breaks every capped + `agent run`. + +##### F. PR #81 — standards and documentation drift + +- ✅ **`#W15-22`** (low) — the PR's "no unsafe `as`" conformance claim is false. Two assertions in new production + code: `shared.ts`'s `readRetryAfter` casts an arbitrary `headers.get`, and `openai.ts`'s `mapOpenAiApiError` + casts a `headers` field back onto a parameter type that lacks it. A structural `headers?:` plus runtime + narrowing removes both. +- ✅ **`#W15-23`** (medium) — `persistEvent`'s `applyDerived` uses the outer `db`, not the transaction handle. + Harmless under `better-sqlite3`, but it breaks ADR-0005's Postgres-portability promise on the highest-volume + money write. +- ✅ **`#W15-24`** (low) — canonical docs describe code that was not shipped. `tool-registry.md` says path + redaction is per-segment (it is whole-string, and the same file says so correctly three paragraphs later); + `llm-provider-seam.md` still claims model-discovery/media-poll keep a small SDK retry; ADR-0028's amendment + note still says ADR-0074 §2–§5 have not landed. Each could send a future reader back toward a #91-class hole. +### Moved by the PR #96 review correction — 2026-11-04 + +> The first pass above moved only current.md's 2026-07-26 plan (its lines 502–919 at `5a328bca`). A PR #96 review +> found that the rest of its unique records had left the docs, against D66's ruling to relocate them, so they are +> moved here verbatim too: the Wave 2 and Wave 3 sections of that plan, the rows the reset edited in place (as they +> read before), and current.md's, CLAUDE.md's and AGENTS.md's status blocks. Only relative links were rewritten and +> heading levels lowered by one; a heading in brackets is added. The Phase 2.6.5 interlude and the W7/W8 records are +> in [Phase 2.6.5's record](phase-2.6.5-core-reliability-remediation.md#execution-record-moved-from-currentmd--2026-11-04), +> Phase 2's status in [Phase 2's](phase-2-cli.md#status-record-moved-from-currentmd--2026-11-04) and the +> "Not started yet" summary in [Phase 2.6's](phase-2.6-conversational-authoring.md#status-record-moved-from-currentmd--2026-11-04). + +#### Wave 2 — Shut the doors + +**Complete on `main`, 2026-10-15, merged as PR #94 (`ab08209e`).** The [merge checkpoint](#wave-2-merge-checkpoint--2026-10-15) joins all forty queue IDs, the exit/D sign-off and final correction evidence. Hostile-MCP work is reused from W4/PR #87; the remaining filesystem, secrets/config, persistence and media work is complete within its recorded limits. The queue below preserves scope and traceability. + +1. ✅ **Completed in W4/PR #87:** 2.5.5.D · project-layer MCP name collision redirecting a global secret + (G19) and MCP tool-definition poisoning (#202); see the + [W4 closing register](phase-2.6.5-core-reliability-remediation.md#w4-closing-register). +2. **MCP queue, strict — EXECUTED AS PHASE 2.6.5 `W4`, and COMPLETE there (2026-09-01, [ADR-0088](../../decisions/0088-the-mcp-boundary-is-hostile.md)).** The queue below landed in five steps, plus `#202` and `G19` from step 1 of this wave and the MCP half of `#21` from 2.5.5.E — all the same threat class and the same security sitting, so splitting them would have booked the reviewer twice. Certify the 2.5.5 ids from the `W4` closing register in [phases/phase-2.6.5-core-reliability-remediation.md](phase-2.6.5-core-reliability-remediation.md). + *The original queue, for the record:* Connect-phase timeouts on every transport + (#35, G32, #205) → transport/discovery/result size bounds against a hostile server (G33, #201, #209, #288) → + structured `serverId`/`reason` discriminants (#203, #204) → the two fail-loud violations (#206, #207) → + per-file transport-adapter tests (#297) → `CLIENT_INFO.version` (#208). These finding ids keep their home in + [phases/phase-2.5.5-hardening-and-remediation.md]() and + certify from there in step 6 below; the *work* happens in 2.6.5 alongside `CR-16` (consent before a stdio + spawn) and `CR-41` (the DNS/redirect SSRF hole), because they are the same code and the same security + sitting. Splitting them across two phases would book the hostile-MCP reviewer twice. +3. ✅ **The fs jail, ONE reviewed change set — complete on `main`, PR #94** — 2.5.5.D · sensitive-credential floor for `.kube`/`.azure`/gcloud + (#36, #39) → extract `deepestExistingReal` to `packages/shared` (#235) → **2.6.M's `project`-tier + `extraRoots`** and **2.6.N's fs-floor home-anchoring**, both *pulled forward*. `~/.relavium/tmp/` (G21) + is resolved under the same general predicate (decision D12) — otherwise `fs.ts` takes two security reviews. +4. ✅ `discoverCatalog` realpath containment (#1, accepted in Step 2); the secrets-layer test pair — OS-keychain bridge (#27) and + `readSecretFromStdin` + stdin cap (#31, #293); secret-typed `run --input` rejection (#25); the unredacted + workflow-snapshot doc + lint (#30); 2.5.5.B · `scrubSecrets` param-name broadening (#196). +5. ✅ Persistence-security: 2.5.5.C · `assertRelativePath`'s Windows drive-relative form (#113), media-store + hardening (#111, #108, #112), media `mimeType` verification (#107, #109, #110), and certification of + W7's structural session/tool-content controls (#115, Wave D15). 2.5.5.A · chat-export continuity/slug + completion and existing tool-content exclusion (#80, #81, #83, D14), plus `analyzeSecretTaint`'s + context sites (G22). 2.5.5.D · the dormant `default_headers` admission/custody gate (G20, D13). +6. ✅ **Phase 2.5.5 exit criteria 1, 2 and 3 and sub-stream D acceptance are satisfied on `main` by PR #94.** The D half of M2.5.5-2 is complete; the I half remains open. + +> Book **four security-review sittings by threat class** (fs/path-jail · secrets-at-rest · secrets-input · +> config-trust), not ~30 per-PR passes. The **hostile-MCP** sitting moved to Phase 2.6.5 with the queue-2 work; +> it covers `CR-16` and `CR-40`–`CR-42` there. Reviewer availability, not code, is this wave's critical path. + + +#### Wave 3 — Clear the ground + +**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1, 9a.2a, 9a.2b, NI, 9b.1 and 9b.2 accepted on `development` (2026-10-24/26), Steps 9c–19 NOT STARTED — 2026-10-26.** +ADR-0110–0113, their linked contracts and the nineteen-increment plan were approved on 2026-10-16. +[Steps 1–8 were independently accepted](#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) +on `development`, with whole Step 8 reviewed at `c83c8575`, and published as the scoped checkpoint +[PR #95](https://github.com/HodeTech/Relavium/pull/95); its [merge checkpoint](#wave-3-steps-18-merge-checkpoint--2026-10-18) is the canonical record of +what merged. The phase register owns each increment's source, tests, cumulative review rounds, security sittings and +platform/publication evidence, including historical NONCLEAN R1–R24 records, the +[previous corrective review completion](#pr-95-corrective-review-completion--2026-10-18), +the [second correction register](#pr-95-second-systematic-review-corrections--2026-10-18), +the [second corrective source-review completion](#pr-95-second-corrective-source-review-completion--2026-10-18) +(accepted at `ec0f3e10`), the [native Windows admission preparation](#pr-95-native-windows-admission-preparation--2026-10-18) +and the [supplemental fixture acceptance](#pr-95-supplemental-native-fixture-source-acceptance--2026-10-18). +Those scopes are not widened: the delta from `ec0f3e10` to the merged head is two independently accepted test-fixture +increments, documentation and the independent review's corrections at `a3e164b5`, which merged with author +verification only and were then independently reviewed after the merge +([review record](#independent-review-of-a3e164b5-and-its-follow-up-corrections--2026-10-19)); +the follow-up corrections are on `development`, not yet on `main`. Whole-wave acceptance remains Step 19. No new decision, credential, billing or provider capture gate is introduced. + +Decompose, guard and normalize every hot file a 2.6 workstream is about to land on. +**The multi-file queue is Step 1 and runs first.** The approved +[nineteen-step execution plan](#proposed-implementation-increments), +recorded as 29 increments since 2026-10-19, owns the full order and review gates: finish each increment and its fresh reviews +before advancing; no parallel edits of shared hot files. + +The groups below retain the original scope inventory and within-file dependency +arrows, not independent scheduling permission or current completion claims. The +[source-corrected preflight dispositions](#wave-3-preflight-and-proposed-execution-plan--2026-10-15) +own whether/how each original finding needs implementation or preservation. + +Two orderings here are not this plan's invention — they are the intra-phase serializations +[2.5.5 records under Positioning](#positioning): **E before I** +on `home-controller.ts`/`chat-ink.tsx` (the decomposition moves already-guarded code), and **C before E** on +`packages/db/src/client.ts` (C types the error path in Wave 1; E's `openLocalDb` rewrite lands last, here). + +- ✅ **Multi-file Step 1 accepted — 2026-10-16** ([evidence](#wave-3-step-1-acceptance--2026-10-16)) — 2.5.5.E · `closeQuietly()` to its four siblings (#22) → `displayPath` + discipline to `resolve.ts`/`catalog.ts`/`chat-export.ts` (#4, G11) → `const exhaustive: never` across the + six sites (#300, #301, G3, which also makes `relavium run` render `agent:reasoning`). +- **`chat.ts` (strict)** — 2.5.5.G · the four money/token reimplementations (#236, #237, #242, #307) → + `swapAgentModel` → `reseatAgentModel` (#218) → 2.5.5.E · SIGINT actually calling `session.abort()` (#21, #23) + → `budgetWarningText` → **2.5.5.I · decompose `chat.ts`** (#220, #244). Then **2.6.M's `allowedDomains` + chat gap (G30)**, pulled forward into `session-host.ts`. +- **`home-controller.ts` (strict)** — 2.5.5.E · `selectGatePrompter` test (G37) → the swallowed palette + rejection (#281) → **2.5.5.I · decompose `createHomeController` + the overlay discriminated union** + (#215, #243) → 2.5.5.G · the `Capability`/`Port` vocabulary split (#217, #222). + Settle #125/D26 before decomposition: the approved preflight certifies the planned scope of the real + in-Home-chat cost path and retaining chat-only scope. If a new bare-Home cost + surface is selected, wire its explicit session target before decomposition. +- **`chat-ink.tsx`** — 1,906 lines at preflight (the original queue recorded 1,795), decomposed by **neither** phase doc. + **Recommendation: decline** a new decomposition (decision D18); take only the exhaustiveness guard and the + min-terminal-size guard (#63), and hand restructuring to 2.6.M's render-v2 in Wave 5a. +- **Viewport (strict)** — 2.5.5.I · share the ADR-0069 grapheme segmenter (#58, #62, G40, G51) → + 2.5.5.G · tab expansion before width measurement (#65) → per-cluster width cache + resize debounce + (#64, #66), refined to frame coalescing in the proposed plan. Alt-screen word-wrap + (#61) retains the approved Wave 5a ownership; it is not an unapproved Wave 3 prerequisite. +- **`packages/shared` (strict)** — the `.code` ruling (G50, D10) → `findDuplicates`/`superRefine` implementation or preservation at the + schema dispositions (#89, #90, G16, G17, G18, #210, #211, #212) → `BaseEventSchema` disposition (#213) → + the `RelaviumError` base class (#219). *The ruling must precede the base class, or it is baked in permanently.* +- **`registry.ts` / `builtins.ts` / `engine.ts`** — declared-source context repair and honest unsupported-mapping disposition (#92/Accepted ADR-0113) → `ctx.fsScope` (#93) → + `ok`/`success` (#216, #221) → identifier-sort `localeCompare` pinning (#258, #259) → the `git_commit` + denial copy (#308); then the four schema length caps (#95), the `llmVisibleParams` rewrite (#96, #98), + stale doc comments (#78, #97, #214); then `#armRunTimeout` (#75), `#dispatch` dead branch (#79), + `AbortSignal` into `resolvePrompt` + the human-gate recheck (#76, G9). +- **Process/CLI boundary (strict head)** — the revised plan serializes pure flag/descriptor/owner substrate → lazy guarded runtime graph → native crash integration → map/archive custody → command behaviour. It covers 2.5.5.E · the five crash/signal net gaps (#40, #41, #42, #224, + #269), the `--help`/`--version` historical ~400 ms tax (#43, #44, #271), and the flag-system ruling (#45, #264, + #265, #268), the exit-code and `--help --json` rulings (G35, #13), durable-only logs preservation (G15 refuted), the `--input` + namespace ruling (#18, #310). **`openLocalDb` wrap-and-redact (#278, #280) lands LAST** — it touches 8+ + call sites the other queues are still moving. +- The numbered plan assigns the remaining 2.5.5.A/E/G/I tails and keeps the central + DB refactor last before whole-wave acceptance. Step 19 owns sub-stream **A, E + and G sign-offs**, the reconciled **EXIT:4 reading** (D22), and **M2.5.5-3** + closure. Full I acceptance remains in Wave 5a. + + +#### [Rows of current.md edited in place on 2026-11-04 — their wording before] + + +- **Sub-stream B, C and H acceptance sign-offs**, then **cut and publish v0.2.0**. **Closes M2.5.5-4.** + + +Roughly 60 open decisions sit across both phases: the ~14 that 2.5.5 flags **blocked** inline +([Dependencies](#dependencies)), the nine unwritten 2.6 ADRs + +| D7 | 0 | Publish v0.1.1 as-is, or supersede with v0.2.0? | v0.2.0 — ADR-0067's Node `>=22` bump is breaking for 0.x | + +| D50 | 6 | ADR row 11: adopt `ink-syntax-highlight`, or build in-house? | **Decline the dependency** per CLAUDE.md rule 2 — the markdown renderer is already in-house and untrimmed `highlight.js` is the phase's own named bundle risk | + + +This ordering is compatible with both phase files' stated dependencies. It departs from their prose in six +places, each deliberate: + +### [current.md's status block, as of 2026-10-25] + +This page tracks what is active **right now** and the immediate next concrete actions. +The full phase plan and the global milestone spine are in [README.md](../README.md). +**Phase 2.5 (CLI Consolidation) is complete** (milestone **M2.5-4**, PR #69, 2026-07-08) — its +breakdown, now historical, is in +[phases/phase-2.5-cli-consolidation.md](phase-2.5-cli-consolidation.md). +A 2026-07-19 full-project code review then opened **Phase 2.5.5 (Hardening and Remediation)**, an +interstitial phase holding the review's verified findings against the pre-2.6 codebase; it runs +**concurrently with Phase 2.6, not before it** — see +[phases/phase-2.5.5-hardening-and-remediation.md]() +and [Active now](../current.md#what-is-active-now). +**Phase 2.6 — Conversational Authoring and the First-Class CLI** is **in progress** (re-scoped +2026-07-08); its plan is in +[phases/phase-2.6-conversational-authoring.md](phase-2.6-conversational-authoring.md). +Workstream **2.6.F (platform floor + the full-screen TUI renderer)** is **merged to `main`** +(PR #74, 2026-07-11), and so is **2.6.C** (the reseat transcript-carry + the `/cost` per-model breakdown, +PR #75, 2026-07-13) — see [Active now](../current.md#what-is-active-now). **2.6.Q** (dynamic model-catalog +enrichment) is **no longer blocked**: [ADR-0071](../../decisions/0071-models-dev-as-the-model-metadata-source.md) +and [ADR-0072](../../decisions/0072-model-metadata-in-the-db-behind-a-generated-offline-floor.md) are both +**Accepted**, resolving the six open maintainer questions the phase file had cited, and the P1–P5 +implementation steps are **merged to `main`** (PR #76, 2026-07-26). P6 and the `/settings` → `/models` +visibility toggle remain open; the bundled offline snapshot was **regenerated** in PR #79 after the D3 +price ruling. + +**The unified build order across both phases is in +[Execution order — Phase 2.5.5 + Phase 2.6](../current.md#execution-order--phase-255--phase-26-temporary) below** — a +temporary section, deleted when both phases close. Its baseline step is discharged (PRs #76 and #77 merged) +and Wave 0 is merged (PR #82), as is Wave 1 (PR #83). The interlude's `W2` (core reliability, six items) is +merged too (PR #85, 2026-08-28). + +### Wave 2 merged — Shut the doors + +**Wave 2 merged to `main` on 2026-10-15 as [PR #94](https://github.com/HodeTech/Relavium/pull/94), +merge commit `ab08209e`. All eight steps and the maintainer corrections are complete.** +The [merge checkpoint](#wave-2-merge-checkpoint--2026-10-15) +joins the forty item dispositions, four new security sittings plus W4 revalidation, +original whole-wave acceptance and all subsequent correction scopes. Existing incoming +navigation anchors are retained; dated implementation checkpoints remain historical. + +Completed work covers invocation-authorized filesystem boundaries and scratch-policy +composition; bounded secret stdin, early admission and private bridge diagnostics; +faithful-snapshot warnings, structural session privacy and export completion; verified +bounded media/CAS/native lifecycles; and the interim provider-header gate. Maintainer +corrections include runner-level error rejection, valid Opus/Vorbis publication, +FLAC structural consistency, generated-result streaming and finite pin-error provenance. + +Final reviewed correction source `4cdfe011` has two NEW independent complete clean +cumulative rounds covering all 67 correction paths / 188 hunks / 24 complete new files. +These scoped rounds do not claim a new review of every original PR path. Full local +CI, ordinary/enforced coverage and all nine jobs in each PR/push CI workflow pass; +SonarCloud and selected native Windows/Node-floor controls pass. Local coverage records +588 files / 12,983 passing tests / 20 explicit skips; CI records 12,982 / 21 with its +established extra CI-only skip. Complete physical controls are independently inspected. +The merge tree matches the reviewed source; no new merge-commit execution is inferred. + +**Satisfied on `main`:** Phase 2.5.5 exit criteria 1–3 and sub-stream D acceptance. +**Still open:** M2.5.5-2's I half, whole Phase 2.5.5/2.6, M/N production lifecycle, +future live headers and existing approved residuals. Wave 2 has no remaining implementation, +review, publication, ADR approval, provider capture, credential, billing or user-decision gate. +The [deferred register](../deferred-tasks.md) retains its existing limits; closure adds no deferral. + + +### Wave 3 Steps 1–8 merged — PR #95 + +**Wave 3 Steps 1–8 merged to `main` on 2026-10-18 as [PR #95](https://github.com/HodeTech/Relavium/pull/95), +merge commit `6d2a7372` (22:44 UTC). Wave 3 itself is not complete: Steps 9–19 had not started at +that merge (Steps 9a.1, 9a.2a and 9a.2b are now accepted on `development`, the native-identity increment NI and Steps 9b.1 and 9b.2 are accepted too, and Steps 9c–19 have not started) and whole-wave acceptance +remains Step 19.** The [merge checkpoint](#wave-3-steps-18-merge-checkpoint--2026-10-18) +joins the completed scope, every review attribution, the independent review's dispositions and the +publication state observed at merge. It was the maintainer-authorized, scoped publication checkpoint. + +On `main` now: close handling, safe source labels and exhaustiveness guards (Step 1); shared admission and +diagnostics (2); secret provenance at public sinks, ADR-0113 (3); the typed error/envelope substrate (4); tool and +engine tails (5); the pure process/discovery substrate, ADR-0110 (6); the lazy runtime, finite compiled graph and +exact-tarball consumers, ADR-0111 (7); and creation-owned native process integration (8). + +Three points were recorded rather than papered over; all three are now resolved. The merged head `a3e164b5` carried +corrections from a further independent review (a secret-scrubbing production rule, two behaviour-neutral import moves, +tests and causal-control pins) with **author verification only** at merge; a post-merge independent review — four +rounds and a security sitting, with fixes in follow-up commits — accepted them, as the +[review record](#independent-review-of-a3e164b5-and-its-follow-up-corrections--2026-10-19) shows. CI had not finished when the merge +happened (the required `lint · typecheck · test` job was still running); the three runs later completed `success`, +15 of 15 jobs each, as the [merge checkpoint](#publication-state-observed-at-merge) +records. And SonarCloud's quality gate, which had failed on `main` since at least PR #90 because its dispositions lived +only on PR branches, was cleared on 2026-10-19 (zero open vulnerabilities, gate OK). Open findings remain, including two +security residuals that predate the follow-up — an `api-key:Bearer ` token leak and unmasked userinfo with a raw +`?`/`#` — in [deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18). + + +### Next work — Wave 3 — Clear the ground + +Wave 3 implementation continues on `development`: Steps 1–8 are merged. A pre-implementation document/source review +of Steps 9–19 and the maintainer's rulings are recorded in the +[2026-10-19 rulings](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) +(no new ADR; 29 recorded increments). **The PR95 increment is closed on `development`** — the PR #95 +scrubber residuals, including the two security residuals — under the maintainer's 2026-10-21 masker acceptance +bar and round cap, as its +[implementation checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20) records: rounds 1–10 were NONCLEAN; the high and +medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the +maintainer counted as high on 2026-10-22, is corrected (in the summary, by the author alone). Its other findings +are recorded in +[deferred-tasks.md](../deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24) after two rounds, the second with no blocker, high or medium finding and no regression ([acceptance](#wave-3-step-9a2b-acceptance--2026-10-24)), and the native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25) after one round with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](#wave-3-ni-acceptance--2026-10-25)), its Windows clause open until three green advisory Windows runs; Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](#wave-3-step-9b-preflight-decisions--2026-10-25)), and Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` ([acceptance](#wave-3-step-9b1-acceptance--2026-10-25)), and Step 9b.2 (the release binding receipt v1, verified ciphertext reuse, the absent production recipient loader and scope separation) is independently accepted on `development` at `563402e6` ([acceptance](#wave-3-step-9b2-acceptance--2026-10-26)), with Step 9c next; the substrate's +9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the +[increment table](#proposed-implementation-increments) and end with +whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the +review scope of new commits. See the [Wave 3 queue](../current.md#wave-3--clear-the-ground). + + +### Where we are + +**Phase 1 — Engine and LLM is COMPLETE** (milestone **M2** reached, PR #27, 2026-06-16; +all workstreams 1.A–1.AH merged through PR #38, 2026-06-21). The engine runs end-to-end: +YAML parse → DAG → run loop → node execution (agent + the six non-agent handlers) → +checkpoint/resume → node retry → provider failover, with per-attempt cost tracking and a +gap-free `sequenceNumber`. The `@relavium/llm` seam is frozen and proven behind a +provider-agnostic `LLMProvider` interface with all three adapters (Anthropic, +OpenAI/DeepSeek, Gemini) and the `FallbackChain` runner. Two additive sub-spines also +completed: the agent-first entry point (`AgentSession` multi-turn + session persistence + +export-to-workflow, Lane C / 1.m5, 1.V–1.AA) and multimodal I/O (media input + engine +plumbing + inline & async output generation + generative adapters, 1.m6, 1.AD–1.AH). See +[Phase 1 detail](phase-1-engine-and-llm.md), the [decision index](../../decisions), +and the [reference specs](../../reference). + +> **Repository checkpoint (recorded 2026-07-26; publication status corrected 2026-10-13):** +> (1) The CI `ci` job became a required check under `main` branch protection, alongside a +> `v*` tag ruleset limiting tag creation to maintain/admin. The optional `TURBO_TOKEN`/ +> `TURBO_TEAM` secrets for cross-runner remote cache were recorded as unset at that checkpoint; +> this does not assert their current configuration. (2) The former v0.1.1 publish-pending note +> was stale: the [npm registry](https://registry.npmjs.org/relavium) records `relavium@0.1.1` +> published on 2026-06-28. This resolves the package-publication status, without asserting the +> current `NPM_TOKEN`, npm 2FA or GitHub release-workflow state. The release procedure remains +> [ADR-0051](../../decisions/0051-cli-distribution-thin-bundle-private-engine.md) / +> [release-a-surface.md](../../runbooks/release-a-surface.md). + + +### What is active now + +#### 2026-07-19 full-project code review — triaged, Phase 2.5.5 opened + +A multi-agent code review of the whole tree at HEAD `f88b0e8` ran while the toolchain was green +throughout (lint, typecheck, and all 226 test files passing) — every finding is a defect a green +build cannot catch. 377 findings went through two-lens adversarial verification (one agent trying +to refute each finding against the code, one auditing its materiality); 373 survived and were +triaged into 205 work items. The review's central conclusion: the gap to first-class is a gap in +**propagation**, not competence — the correct pattern almost always already exists in the repo and +fails to reach the next site (a redaction helper not called twenty lines away, terminal +sanitization built for chat and never carried to `relavium run`, `withBusyRetry` applied to every +writer but the three oldest, cancel-wins implemented in four node handlers and omitted in the +fifth). That makes the remediation cheap and low-risk: the reference implementation is already +in-tree. + +Disposition: 176 work items opened the new **[Phase 2.5.5 — Hardening and Remediation]()** +(engine/LLM/persistence hardening, MCP/secrets, the CLI operator-safety net, TUI terminal safety, +docs accuracy, CI/test-infrastructure, and hygiene), running **concurrently with Phase 2.6, not +before it** (327 findings); 27 findings (16 work items) were folded directly into the charter of an +already-open 2.6 workstream; 16 findings (10 work items) were already tracked in +[deferred-tasks.md](../deferred-tasks.md) and were re-confirmed rather than duplicated; 3 needed no +action per the finding's own argument. Severity of the 373 triaged findings: 3 critical, 48 high, 134 +medium, 155 low, 33 polish. + +The three critical findings, visible here so they aren't buried in the phase file: + +- **A tool-approval preview that is not secret-redacted** (`packages/core/src/tools/registry.ts`) + — the persisted, `--json`-visible `ToolActionPreview` echoes a command/path verbatim instead of + going through the same redaction its sibling input-sanitizer already applies. +- **A transient `history.db` write failure silently drops a chat turn, then crashes the session** + (`apps/cli/src/chat/persister.ts`) — an async write failure surfaces after the reply was already + shown to the user, as an unhandled rejection. +- **Missing terminal-output sanitization on the `relavium run` TUI** (`apps/cli/src/render/tui/RunApp.tsx`) + — `stripTerminalControls`/`sanitizeInline` is applied at nearly every other dynamic-text display + boundary but never reaches this one, leaving an ANSI/OSC/Trojan-Source injection hole. + + +### [CLAUDE.md's status block, as of 2026-10-26] + +**Status: Phase 1 is complete** (2026-06-21) — the engine runs end-to-end behind the +`LLMProvider` seam with all three adapters, `AgentSession`, multimodal I/O, and the full +run-loop. **Phase 2 (CLI) is feature-complete** (published as v0.1.1): +`relavium chat` / `chat-resume` / `agent run`, `relavium run` with `--json`, the +inbound MCP client, and the full YAML lifecycle (`create`/`import`/`export`). +**Phase 2.5 (CLI Consolidation) is complete** (M2.5-4, PR #69, 2026-07-08): the bare-invocation +Home, the two-registry command system (`/` palette + shell commands), per-tool +approval/modes (ask/plan/accept-edits/auto), `@`-mention file injection, context +compaction, the onboarding wizard + live model catalog, reasoning rendering, and regression +hardening. **Phase 2.6 (Conversational Authoring and the First-Class CLI) is in progress** — two +workstreams are merged to `main`: **2.6.F** (the platform floor + the full-screen TUI renderer, +PR #74, 2026-07-11) and **2.6.C** (the reseat transcript-carry + the `/cost` per-model breakdown, +PR #75, 2026-07-13). The phase is a full-screen Home-managed CLI with conversational workflow authoring, management +browsers, competitor-breadth tools, settings/theming/`en`+`tr` localization, and the +run-ops resume follow-up. + +**The interlude between remediation Waves 1 and 2 is complete: Phase 2.6.5 (Core reliability), +49 of 51 items closed on `main`, with approved CR-70/93 and scoped deferrals retained. +W8 merged on 2026-10-11 as PR #92; W7 had restored the non-deferrable criterion** — `W0` (PR #82, 2026-08-11), `W1`, the eight P0 blockers plus `CR-92`, merged +2026-08-24 (PR #83) behind [ADR-0078](../../decisions/0078-ordered-durable-append-and-the-terminal-outbox.md)–[ADR-0084](../../decisions/0084-consent-before-a-local-mcp-spawn.md): +ordered durable append, cross-process run ownership, the durable effect journal, untrusted compaction summaries, +the stream-grammar seam obligation, engine-side input admission and resume identity, and consent before a local +MCP spawn. **`W2` — liveness and deadlines — merged 2026-08-28 (PR #85)** behind +[ADR-0085](../../decisions/0085-the-node-executor-owes-liveness-and-the-engine-enforces-it.md): the authored +agent `timeout_ms` now bounds the node, every media call is bounded, a resume re-arms the run cap and a +pending gate at their REMAINING time rather than renewing them (the node bound is the deliberate exception — +ADR-0085 §2 records why), and an executor that ignores its signal can no longer leave a run without a +terminal. **`W3` — resource governance and bounds — merged 2026-08-30 (PR #86) behind +[ADR-0086](../../decisions/0086-absolute-admission-ceilings-on-authored-values.md)**: authored values carry +absolute admission ceilings, node output / workflow state / durable events are size-bounded, and a long-lived +engine no longer retains every finished run. **It merged with a live blocker** — an un-pulled event stream +drops the terminal event, breaching ADR-0036's gap-free contract — plus nine verified findings, all recorded +in the `W3` residuals of [deferred-tasks.md](../deferred-tasks.md). +[ADR-0087](../../decisions/0087-consumed-streams-size-bounds-and-run-retention.md) records the fix and, after a +review that refused acceptance until nine dated corrections existed, was **Accepted (2026-09-04) with its §1 +unimplemented**. That blocker is now closed by W8/PR #92. Its §3 turned out to be implemented already, in `W5`. **`W4` — the hostile MCP boundary — merged 2026-09-01 (PR #87) +behind [ADR-0088](../../decisions/0088-the-mcp-boundary-is-hostile.md)**: every MCP call is bounded and +cancellable, `http`/`sse` connect by validated pinned IP, a redirect is refused, a remote `websocket` is +refused at admission, a server's ingress is bounded at two levels, and its tool DEFINITIONS are treated as +untrusted. No new product surface in either wave — only the invariants an existing surface already claims. +A systematic review of PR #87 found **five merge blockers** in `W4` — all reproduced and fixed before it +merged; the phase doc's `W4` section records what they were and what the pattern says. **`W5` — media correctness — MERGED 2026-09-02 (PR #88)** behind +[ADR-0089](../../decisions/0089-media-correctness-four-boundaries.md) and +[ADR-0090](../../decisions/0090-a-continuation-token-rides-the-part-it-belongs-to.md): `read_media` delivers +over the media-input rail on a marked, fenced synthesized message; tool and attachment capability is gated on +the MODEL; Gemini's function-call continuation token rides the part it belongs to; a media body streams from +the network into the store under a size ceiling, an idle deadline and the run signal; a `url` output is pinned +to a content-addressed handle at first resolution; and a missing media rate is unpriced rather than a price of +zero. A systematic maintainer review of the branch returned **six merge blockers**, all reproduced and fixed +before it merged. Two items closed one half of a two-part obligation and say so; every residual is written out +rather than left inside a checked box. **`W6` — authoring correctness — MERGED 2026-09-04 (PR #89)** behind [ADR-0091](../../decisions/0091-first-means-first-declared-not-first-to-finish.md)–[ADR-0094](../../decisions/0094-a-tool-grant-is-checked-when-the-plan-is-built.md): +`merge_strategy: first` means first DECLARED and the plan field no longer claims otherwise; a widened tool grant +is refused when the plan is BUILT rather than mid-run; an expression sees only its transitive dependency closure +and a literal out-of-closure `run.outputs` read is refused at parse (as is `edges[].condition`, which nothing +ever read); and an authored `output_schema` is compiled at parse in an allowlist-strict mode and enforced at run +time — with **no new dependency**, because the JSON-Schema→Zod compiler the deferral claimed we needed already +existed in `packages/mcp` and simply moved to `@relavium/shared`. Seven internal review rounds found that **most defects +were in the FIXES rather than the code they repaired** — the branch-order search was wrong three times, the +expression scan's no-false-refusal claim was false in eight ways, and a stand-down gate silently removed the +check it was added to protect; each is a dated correction inside its own ADR. A systematic maintainer review +then returned **two merge blockers**, both breaking a headline claim of the wave — `required` was not enforced +when a property's schema constrained nothing, and the expression scanner's regex was quadratic and synchronous +on a parser that accepts 2 MiB — plus eight further High findings, all reproduced and fixed before the merge. + + +**Between `W6` and `W7`, the two remaining non-deferrable items closed (2026-09-06)**: `CR-73` — `invoke_agent` +was advertised to every model granted it while `ctx.invokeAgent` is wired nowhere in the tree, so the +advertise-filter now checks a tool's declared dispatch DELEGATE and not only its `ToolHost` arm (`read_media` +rides along, for the same reason and by `CR-50`'s own argument) — and `CR-80` — a rejected custom provider +`base_url` was caught and the DEFAULT adapter left standing, which is the official API, so a drifted config +silently sent prompts and keys to `api.openai.com`; it now installs a refusing adapter naming the URL's shape, +never its credentials. The test that was supposed to pin `CR-80` asserted only `toBeDefined()`, which a +refusing adapter satisfies too, so it passed either way — the finding under the defect was the test. +[ADR-0087](../../decisions/0087-consumed-streams-size-bounds-and-run-retention.md) was also **accepted +(2026-09-04) with nine dated corrections** after a review refused it as written; §1 was still unimplemented +at that checkpoint, while §3 turned out to have shipped in `W5` under a different item number. +**W8 merged to `main` on 2026-10-11 as PR #92 (`ef0ffa54`); Phase 2.6.5 is complete with approved deferrals.** +All seven steps, CR-81/82 and ADR-0087 §1 are complete on `main`; the register is **49/51**. +The [merge checkpoint](phase-2.6.5-core-reliability-remediation.md#w8-merge-checkpoint--2026-10-11) +joins the original whole-wave acceptance, all maintainer-correction dispositions and final reviewed-head +`3d4491df` PR/push CI plus SonarCloud success. The ordinary/enforced-coverage inventory is +511 files / 11,026 passing tests / 12 established skips; four wire and 23 usage mechanisms pass. +Every correction increment has two NEW independent complete clean cumulative rounds after its final fix; +scoped correction reviews are not a new whole-PR certification. Fresh full local CI and bare/enforced +coverage at `6139563e` retain their separate source attribution. No merge-commit execution is inferred. +Two nonblocking Sonar complexity suggestions remain visible. Historical counts and pending states +throughout the registers describe their original checkpoints. + +CR-70 (cross-turn tool-history carrying), CR-93 (tenant-scoped mutable state), CR-95's long continuation +and existing exact-capacity/scoped residuals retain their approved limits in +[deferred-tasks.md](../deferred-tasks.md). No new deferral, provider capture, billing, +credential or W8 user decision is pending. The joint [Wave 2 — Shut the doors](../current.md#wave-2-merged--shut-the-doors) merged to `main` on 2026-10-15 as PR #94; +next is [Wave 3 — Clear the ground](../current.md#wave-3--clear-the-ground). +Phase 2.5.5 and Phase 2.6 remain in progress. + +**W7 merged to `main` on 2026-10-10 as PR #90, including final whole-wave Step 12.** +Steps 1–11, all five genuine overflow records, owned measured requests and consolidated provider/MCP/host +lifetimes have independent acceptance. ADR-0095–ADR-0103 are Accepted. The six W7 register items are closed +(47/51 at the W7 merge); PR #90 is merged (`46a6a701`); W7 has no open implementation or review gate. The +[closing register](phase-2.6.5-core-reliability-remediation.md#w7-closing-register--2026-10-10) +joins the accepted scopes, causal controls and systematic-review dispositions. Existing approved deferrals +stay in [deferred-tasks.md](../deferred-tasks.md); W8 subsequently merged in PR #92 and closed the phase with approved deferrals. No user decision, +billing setup, credential or provider capture is pending. + +**Post-closure corrections are independently accepted, 2026-10-10:** core, CLI/DB and tooling/docs +each have two NEW clean complete cumulative review rounds. The supplemental publication-tool +correction also has two NEW complete clean rounds. The [post-closure register](phase-2.6.5-core-reliability-remediation.md#post-closure-systematic-review--2026-10-10) +joins every report disposition, new permanent/causal evidence and current gates. Published-head checks +passed at reviewed head `69ac80b0` before PR #90 merged. No W7 implementation, provider, +billing, key or user decision is pending. The [merge checkpoint](phase-2.6.5-core-reliability-remediation.md#w7-merge-checkpoint--2026-10-10) +records W7 closure. W8 subsequently merged in PR #92; the register is 49/51 on `main`. +Existing approved limits are unchanged; W8 has no open implementation or landing gate. + +**Joint Wave 2 merged to `main` — 2026-10-15, PR #94 (`ab08209e`).** +All eight implementation steps and the maintainer corrections are complete on `main` +under Accepted ADR-0105–0109. The [merge checkpoint](#wave-2-merge-checkpoint--2026-10-15) +joins forty item dispositions, security sittings, the final reviewed source `4cdfe011`, +two NEW clean complete cumulative correction rounds and successful PR/push CI, +coverage, native-platform and independently inspected physical controls. Earlier +whole-wave and correction reviews retain their exact source scopes; no new whole-PR +review or merge-commit execution is inferred. Detailed review records remain local +and Git-ignored. + +Exit criteria 1–3 and the D half of M2.5.5-2 are satisfied on `main`. The I half, +whole Phase 2.5.5/2.6, full M/N production lifecycle, future live provider headers +and existing approved deferrals remain open in their canonical homes. No Wave 2 +implementation, review, publication, ADR approval, provider capture, credential, +billing or user-decision gate remains. Next is [Wave 3 — Clear the ground](../current.md#wave-3--clear-the-ground), +whose ADR-0110–0113/contracts/nineteen-increment plan were approved on 2026-10-16. +Its serialized multi-file Step 1 is independently accepted on `development` +(2026-10-16); Step 2 is also accepted after two NEW clean complete cumulative rounds +following its historical-reader corrections. Step 3 (ADR-0113 secret provenance) +is independently accepted after two NEW complete cumulative clean rounds and a +separate clean security sitting. Step 4 (error/envelope substrate) is independently +accepted after two NEW complete cumulative clean rounds. Step 5 (tools and engine +tails) is independently accepted after two NEW complete cumulative clean rounds +following its registry-documentation correction and a separate clean security +sitting. Step 6 (pure process/discovery substrate) is independently accepted after +two NEW complete cumulative clean rounds and a separate clean security sitting. +Step 7a (lazy runtime and finite compiled graph) is independently accepted after +two NEW complete cumulative clean rounds and a separate clean security sitting. +Step 7b and whole Step 7 are independently accepted at `c836ffe6` after two NEW +complete cumulative clean rounds, a clean security sitting and successful exact-tarball +Linux/macOS/Windows plus Node 22.0.0 consumers. Broader CI retains its separately +attributed status. Step 8a implements origin adapters and an inert native foundation; +it is independently accepted at `e0a5ae19` after two NEW complete cumulative +clean rounds and a separate clean security sitting. +Step 8b (creation-owned native processes) is independently accepted at +`e16a7659` after two NEW complete cumulative clean rounds and a separate clean +security sitting following its canonical-reference correction. Its unchanged +runtime/control evidence and successful full CI plus all four same-tarball +platform consumers remain attributed to `ab50be11`, with independently verified +source/dependency equivalence. Step 8c workflow/preparation/terminal integration +is independently accepted at `dbab25d8` after two NEW complete cumulative clean +rounds, a separate clean security sitting and successful full CI. All four +same-tarball platform consumers pass at that reviewed source. +**Step 8d and whole Step 8 are independently accepted on `development` at +`c83c8575` — 2026-10-17.** Two NEW complete cumulative CLEAN rounds (R25/R26) +each cover all 342 paths / 1,842 ordered hunks after the final fixture correction. +A NEW separate clean security sitting and all fifteen full-CI jobs pass at that +reviewed source, as do all four actual same-tarball platform consumers. Original +failed reviewer-preparation records and source/native/platform limits remain +explicit; source CI does not certify later documentation commits. The +[acceptance register](#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) +joins exact evidence and historical corrections. Steps 1–8 are accepted; +**Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 +itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now +accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Steps 9b.1 and 9b.2 are accepted too, and Steps 9c–19 have not started) and whole-wave acceptance remains Step 19. +The [merge checkpoint](#wave-3-steps-18-merge-checkpoint--2026-10-18) +records what merged and its review attributions. The final head `a3e164b5` (corrections from a +further independent review, including a secret-scrubbing rule) merged with author verification +only and was then independently reviewed (four rounds and a security sitting, with follow-up +fixes on `development`, not yet on `main`); CI for it and the merge commit finished green, and SonarCloud's `main` +gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation review of Steps 9–19 +and the maintainer's [2026-10-19 rulings](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) +give every open PR #95 finding an owner and record 29 increments (no new ADR). The PR95 increment +(the scrubber residuals, including both security residuals) is closed on `development` under the +maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were +NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last +round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` (2026-10-25): rounds 1–3 were not accepted (one, one and two mediums, each corrected), and round 4 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9b1-acceptance--2026-10-25)). Step 9b.2 (the release binding receipt v1, verified ciphertext reuse, the production recipient loader, absent in Wave 3, and scope separation; synthetic recipients only) is independently accepted on `development` at `563402e6` (2026-10-26): an internal rigor pass and rounds 1–3 were not accepted (mediums each time, all corrected), and round 4 reported no blocker, high or medium finding and no regression, its security sitting lows only; its low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9b2-acceptance--2026-10-26)). Step 9c is next. `development` is pushed after each accepted increment. No provider, +credential, billing or new ADR approval is pending for the approved implementation plan. + +### [AGENTS.md's status block, as of 2026-10-26] + +**Status: Phase 1 is complete; Phase 2 (CLI) is feature-complete (published as v0.1.1); +Phase 2.5 (CLI Consolidation) is complete (M2.5-4, 2026-07-08); Phase 2.6 +(Conversational Authoring and the First-Class CLI) is in progress, with an interlude — +Phase 2.6.5 (Core reliability remediation) — completed between remediation Waves 1 and 2: +49 of 51 items closed on `main`, with approved deferrals retained; W8 merged 2026-10-11 in PR #92 (`CR-96` and `CR-97` were opened 2026-09-14, `CR-98` on 2026-09-18; `W7` merged to `main` on 2026-10-10 (PR #90), with ADR-0095–ADR-0103 +accepted) — all eight P0 blockers (ADR-0078–ADR-0084, merged 2026-08-24), the `W2` +liveness-and-deadlines wave (ADR-0085, merged 2026-08-28 as PR #85), `W3` resource governance and bounds (ADR-0086), merged 2026-08-30 as PR #86 **with a live blocker and nine open findings at that merge** — the blocker is now closed by W8/PR #92; scoped residuals remain in docs/roadmap/deferred-tasks.md — and `W4`, the hostile MCP boundary (ADR-0088), merged 2026-09-01 as PR #87 — a systematic review of the branch found five merge blockers, all reproduced and fixed before it merged. `W5`, media correctness (ADR-0089 + ADR-0090), **merged 2026-09-02 as PR #88** — a systematic review of that branch returned six merge blockers, all reproduced and fixed before the merge. `W6`, authoring correctness (ADR-0091–ADR-0094), **MERGED 2026-09-04 as PR #89** — `merge_strategy: first` means first DECLARED, a widened tool grant is refused when the plan is BUILT, an expression sees only its transitive closure with a literal out-of-closure read refused at parse, and an `output_schema` is compiled at parse and enforced at run time with **no new dependency**. Seven internal review rounds found that most defects were in the FIXES rather than the code they repaired; a systematic maintainer review then returned two merge blockers, both breaking a headline claim of the wave, plus eight further High findings — all reproduced and fixed before the merge. Each is a dated correction inside its own ADR. +**W7 merged to `main` on 2026-10-10 as PR #90, including final whole-wave Step 12.** +Steps 1–11, all five genuine overflow records, ADR-0102 request ownership and consolidated ADR-0103 +host integration have independent acceptance. ADR-0095–ADR-0103 are Accepted. All six W7 register +items are closed (47/51 at the W7 merge); PR #90 is merged (`46a6a701`); W7 has no open implementation or review gate. The +[closing register](phase-2.6.5-core-reliability-remediation.md#w7-closing-register--2026-10-10) +joins evidence and every submitted review disposition. W8 subsequently merged in PR #92; approved deferrals remain. +No user decision, billing setup, credential or provider capture is pending. +**W8 merged to `main` on 2026-10-11 as PR #92 (`ef0ffa54`); Phase 2.6.5 is complete with approved deferrals.** +All seven steps, CR-81/82 and ADR-0087 §1 are complete on `main`; the register is **49/51**. +The [merge checkpoint](phase-2.6.5-core-reliability-remediation.md#w8-merge-checkpoint--2026-10-11) +joins original whole-wave acceptance, maintainer corrections and final reviewed-head `3d4491df` +PR/push CI and SonarCloud success. Published ordinary/enforced coverage records 511 files / +11,026 passing tests / 12 established skips; all four wire and 23 usage mechanisms pass. +CR-70/93, CR-95's long continuation and exact-capacity/scoped residuals remain approved deferrals. +The joint [Wave 2 — Shut the doors](../current.md#wave-2-merged--shut-the-doors) merged to `main` on 2026-10-15 as PR #94; +next is [Wave 3 — Clear the ground](../current.md#wave-3--clear-the-ground). +Phase 2.5.5 and Phase 2.6 remain in progress. No W8 implementation or landing gate remains. +**Post-closure corrections are independently accepted, 2026-10-10:** core, CLI/DB and tooling/docs +each have two NEW clean complete cumulative review rounds. The supplemental publication-tool +correction also has two NEW complete clean rounds. The [post-closure register](phase-2.6.5-core-reliability-remediation.md#post-closure-systematic-review--2026-10-10) +joins every report disposition, new permanent/causal evidence and current gates. Published-head checks +passed at reviewed head `69ac80b0` before PR #90 merged. No W7 implementation, provider, +billing, key or user decision is pending. The [merge checkpoint](phase-2.6.5-core-reliability-remediation.md#w7-merge-checkpoint--2026-10-10) +records W7 closure. W8 subsequently merged in PR #92; the register is 49/51 on `main`. +Existing approved limits are unchanged. + +**PR #92 maintainer corrections:** the [correction register](phase-2.6.5-core-reliability-remediation.md#w8-pr-92-systematic-review-corrections--2026-10-11) +records all four increments and the subsequent publication correction accepted after two NEW +clean complete cumulative rounds each. Fresh full CI and bare/enforced coverage pass at `6139563e`, +including four wire and 23 usage controls. +Final reviewed head `3d4491df` passed PR/push CI and SonarCloud before PR #92 merged. +No merge-commit execution or new whole-PR review is inferred. Two nonblocking Sonar complexity +suggestions remain visible; no new deferral or decision is added. + +**Joint Wave 2 merged to `main` — 2026-10-15, PR #94 (`ab08209e`).** +All eight implementation steps and the maintainer corrections are complete on `main` +under Accepted ADR-0105–0109. The [merge checkpoint](#wave-2-merge-checkpoint--2026-10-15) +joins forty item dispositions, security sittings, the final reviewed source `4cdfe011`, +two NEW clean complete cumulative correction rounds and successful PR/push CI, +coverage, native-platform and independently inspected physical controls. Earlier +whole-wave and correction reviews retain their exact source scopes; no new whole-PR +review or merge-commit execution is inferred. Detailed review records remain local +and Git-ignored. + +Exit criteria 1–3 and the D half of M2.5.5-2 are satisfied on `main`. The I half, +whole Phase 2.5.5/2.6, full M/N production lifecycle, future live provider headers +and existing approved deferrals remain open in their canonical homes. No Wave 2 +implementation, review, publication, ADR approval, provider capture, credential, +billing or user-decision gate remains. Next is [Wave 3 — Clear the ground](../current.md#wave-3--clear-the-ground), +whose ADR-0110–0113/contracts/nineteen-increment plan were approved on 2026-10-16. +Its serialized multi-file Step 1 is independently accepted on `development` +(2026-10-16); Step 2 is also accepted after two NEW clean complete cumulative rounds +following its historical-reader corrections. Step 3 (ADR-0113 secret provenance) +is independently accepted after two NEW complete cumulative clean rounds and a +separate clean security sitting. Step 4 (error/envelope substrate) is independently +accepted after two NEW complete cumulative clean rounds. Step 5 (tools and engine +tails) is independently accepted after two NEW complete cumulative clean rounds +following its registry-documentation correction and a separate clean security +sitting. Step 6 (pure process/discovery substrate) is independently accepted after +two NEW complete cumulative clean rounds and a separate clean security sitting. +Step 7a (lazy runtime and finite compiled graph) is independently accepted after +two NEW complete cumulative clean rounds and a separate clean security sitting. +Step 7b and whole Step 7 are independently accepted at `c836ffe6` after two NEW +complete cumulative clean rounds, a clean security sitting and successful exact-tarball +Linux/macOS/Windows plus Node 22.0.0 consumers. Broader CI retains its separately +attributed status. Step 8a implements origin adapters and an inert native foundation; +it is independently accepted at `e0a5ae19` after two NEW complete cumulative +clean rounds and a separate clean security sitting. +Step 8b (creation-owned native processes) is independently accepted at +`e16a7659` after two NEW complete cumulative clean rounds and a separate clean +security sitting following its canonical-reference correction. Its unchanged +runtime/control evidence and successful full CI plus all four same-tarball +platform consumers remain attributed to `ab50be11`, with independently verified +source/dependency equivalence. Step 8c workflow/preparation/terminal integration +is independently accepted at `dbab25d8` after two NEW complete cumulative clean +rounds, a separate clean security sitting and successful full CI. All four +same-tarball platform consumers pass at that reviewed source. +**Step 8d and whole Step 8 are independently accepted on `development` at +`c83c8575` — 2026-10-17.** Two NEW complete cumulative CLEAN rounds (R25/R26) +each cover all 342 paths / 1,842 ordered hunks after the final fixture correction. +A NEW separate clean security sitting and all fifteen full-CI jobs pass at that +reviewed source, as do all four actual same-tarball platform consumers. Original +failed reviewer-preparation records and source/native/platform limits remain +explicit; source CI does not certify later documentation commits. The +[acceptance register](#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) +joins exact evidence and historical corrections. Steps 1–8 are accepted; +**Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 +itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now +accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Steps 9b.1 and 9b.2 are accepted too, and Steps 9c–19 have not started) and whole-wave acceptance remains Step 19. +The [merge checkpoint](#wave-3-steps-18-merge-checkpoint--2026-10-18) +records what merged and its review attributions. The final head `a3e164b5` (corrections from a +further independent review, including a secret-scrubbing rule) merged with author verification +only and was then independently reviewed (four rounds and a security sitting, with follow-up +fixes on `development`, not yet on `main`); CI for it and the merge commit finished green, and SonarCloud's `main` +gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation review of Steps 9–19 +and the maintainer's [2026-10-19 rulings](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) +give every open PR #95 finding an owner and record 29 increments (no new ADR). The PR95 increment +(the scrubber residuals, including both security residuals) is closed on `development` under the +maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were +NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last +round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` (2026-10-25): rounds 1–3 were not accepted (one, one and two mediums, each corrected), and round 4 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9b1-acceptance--2026-10-25)). Step 9b.2 (the release binding receipt v1, verified ciphertext reuse, the production recipient loader, absent in Wave 3, and scope separation; synthetic recipients only) is independently accepted on `development` at `563402e6` (2026-10-26): an internal rigor pass and rounds 1–3 were not accepted (mediums each time, all corrected), and round 4 reported no blocker, high or medium finding and no regression, its security sitting lows only; its low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9b2-acceptance--2026-10-26)). Step 9c is next. `development` is pushed after each accepted increment. No provider, +credential, billing or new ADR approval is pending for the approved implementation plan. diff --git a/docs/roadmap/phases/phase-2.6-conversational-authoring.md b/docs/roadmap/phases/phase-2.6-conversational-authoring.md index 9ec2254d..67882d26 100644 --- a/docs/roadmap/phases/phase-2.6-conversational-authoring.md +++ b/docs/roadmap/phases/phase-2.6-conversational-authoring.md @@ -3,6 +3,9 @@ > Status: **In progress** — **2.6.F** (platform floor + full-screen TUI, PR #74) and **2.6.C** (reseat > transcript-carry + `/cost`, PR #75) are ✅ **Done**, and **2.6.Q P1–P5** landed with PR #76 (2026-07-26). > Joint Wave 2 prerequisites and corrections merged in PR #94 (2026-10-15); the [merge checkpoint](phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records their exact scope. Full M/N lifecycle and remaining workstreams stay open. +> **CLI MVP first (2026-11-04):** the MVP order in [../current.md](../current.md#next-work--the-cli-mvp) takes slices of +> 2.6.A/B/D/E/G/H/I/J/K/M/Q ahead of the rest of the phase; new MVP items are defined under +> [CLI MVP additions](#cli-mvp-additions--2026-11-04). The phase scope, exit criteria and milestones are unchanged. > Live progress is canonical in [../current.md](../current.md); the rest of the phase follows. Depends on the Phase 2.5 > spine (the wired tool-environment and the per-tool approval / mode system), which is **complete** (M2.5-4, > PR #69, 2026-07-08), so this phase is unblocked. @@ -314,6 +317,46 @@ viewport with an inline switch marker (regression-tested on both the Home and st shows per-model spend for a reseated session; the notice is asserted on resume surfaces. **Required ADR:** none (ADR-0059 is Accepted). +#### Implementation record (moved from current.md on 2026-11-04) + +Driven by two maintainer manual-test findings. **F1:** a mid-session `/models` reseat **blanked the alt-screen +viewport** — the reseat builds a fresh view store, and on the full-screen renderer (the TTY default since 2.6.F, with +no native scrollback behind it) that store *is* the scrollback, so the whole conversation vanished the moment a user +switched models. The durable transcript and the new model's context were never at risk; the loss was the view. The +reseat now carries the rendered conversation, gated to full-screen (a seeded **inline** store would re-print the +conversation over the copy ink's `` already put in the terminal's scrollback), and the switch notice lands +**beneath** it as an inline `⇄ model changed → ` marker, keeping ADR-0059's bound disclosure clause. + +**`/cost`** now shows where the money went, behind a new +[ADR-0070](../../decisions/0070-durable-per-model-session-cost-attribution.md): a durable `session_costs` table with the +invariant **`SUM(rows) == agent_sessions.total_cost_microcents`**, true by construction. ADR-0059 had planned to +derive the breakdown from the per-message `model_id`, which **cannot work** — one row holds one model, but a turn +whose tool loop fails over bills two. Along the way it closed a **live bug**: `total_cost_microcents` was being SET +blindly by five writers, so two `chat-resume` processes could permanently corrupt a session's total. It now has a +single owner. `session_messages`' never-written token/cost columns were dropped rather than left as a second, wrong, +zero-valued source. + +Four adversarial review rounds ran over the work and each found something real — including that a tripwire I shipped +was **armed in production** and **swallowed by ink** (so it could never fail), that a perf harness measured a +**pointer assignment** while its ADR note claimed it verified an O(n) bound, and that the `/cost` DB read — the +step's central change — had **zero test coverage** at any layer. All folded, each with a break-verified regression. + +**Open obligations carried out of 2.6.C:** + +- **`chat-resume` opens on an empty viewport** — it has never repainted prior turns, in any mode or version (nothing + projects `session_messages` into rendered entries). Not a 2.6.F regression; tracked in + [deferred-tasks.md](../deferred-tasks.md). +- **Reported spend is a systematic under-estimate of the provider's invoice** — an egress that streamed content but + ended without a usage chunk, and a mid-stream failure, are recorded as 0 on *both* sides of the invariant. That is + a usage-capture gap in the seam, filed against **2.6.Q** (ADR-0070 §3) as its + [usage-capture item](#26q--dynamic-model-catalog-enrichment-modelsdev-per-model-capability-matrix--pricing-long-tail---p1p5-landed-pr-76-2026-07-26-p6-and-the-settings--models-toggle-open). +- **Cross-turn tool-call memory** — a "the reseat forgets tool calls" report turned out to be an *engine* property, + not a reseat one: tool pairs never cross **any** turn boundary (ADR-0062 §6). Deferred behind a default-off toggle, + but **three already-live bugs gate it** (no pre-egress context-window check; overflow is a fatal, non-retryable + `bad_request`; the budget gate prices only output tokens) — see [deferred-tasks.md](../deferred-tasks.md). +- **The CLI e2e suite migrates the developer's real `~/.relavium/history.db`** — verified, and it is what let a + broken migration reach real data while staying invisible to CI. Tracked in [deferred-tasks.md](../deferred-tasks.md). + ### 2.6.D — Session `{{ctx.*}}` prompt interpolation Unchanged in intent; one absorbed deferred item makes the security mandate explicit. @@ -392,7 +435,8 @@ tests. **Required ADR:** `ink-syntax-highlight` dependency + markdown rendering > [ADR-0069](../../decisions/0069-string-width-for-the-cli-renderer.md) (`string-width`, Accepted on the merge). > The shipped inventory (incl. the `✓ Copied` copy-on-select toast), the two adversarial review rounds, and the > open obligations carried out (short-terminal Home overflow → 2.6.G; `relavium run` stays inline; themes → 2.6.L) -> live in the canonical status home [current.md](../current.md); this entry is not restated there. +> are in the implementation record at the end of this section, moved from current.md on 2026-11-04. Its release +> clause (v0.2.0) is still open: it lands with the CLI MVP's MVP-10 ([current.md](../current.md#next-work--the-cli-mvp)). The substrate workstream — it runs **first** because the browsers (2.6.G), settings screens (2.6.L), and render-v2 (2.6.M) all build on it. @@ -442,6 +486,45 @@ and released as a breaking 0.x MINOR bump (e.g. `0.2.0`, per ADR-0067) with migr frozen-clock regression pinned. **Required ADRs:** the ADR-0021-superseding floor ADR + a full-screen renderer & TUI-harness ADR. +#### Implementation record (moved from current.md on 2026-11-04) + +The first 2.6 workstream. Behind [ADR-0067](../../decisions/0067-node-supported-floor-22-reaffirm-better-sqlite3.md) +(Node `>=22` published floor, `>=22.13.0` dev-install floor; `better-sqlite3` re-affirmed), +[ADR-0068](../../decisions/0068-full-screen-tui-renderer-ink7-harness.md) (`ink` 7 + a hand-built alternate-screen +renderer + the `ink-testing-library` harness), and +[ADR-0069](../../decisions/0069-string-width-for-the-cli-renderer.md) (`string-width` for display width — **Accepted** +on the PR #74 merge, the CLAUDE.md rule-2 approval it gated). + +Shipped: `ink` 6→7 and the component-render harness; the alt-screen lifecycle with restore nets on every +termination path; a hand-built transcript **viewport** (grapheme-aware wrapping, windowing, scroll + auto-follow, a +per-entry wrap cache) and the inter-session alt-buffer **hoist**; the **`/scrollback`**, **`/edit`** and **`/copy`** +copy-and-search hatches over a `suspendFullScreen` primitive; **in-app mouse text selection with copy-on-select over +OSC 52** on both surfaces, with edge auto-scroll, a frozen auto-follow and a transient **`✓ Copied` toast** so the +otherwise-silent copy is confirmed; `--no-mouse` / +`[preferences].{alt_screen,mouse,copy_on_select,show_banner}`; a branded Home banner; and DEC-2026 synchronized +output — which `ink` 7 turned out to already emit, so the step **pins** it rather than building the planned +`stdout` Proxy. + +The full-screen renderer is now the **default on a TTY**; `--no-alt-screen` and a machine / non-TTY / `--json` / CI +path stay on the byte-identical inline renderer. + +Two adversarial review rounds (one per step, one over the whole phase) drove the last third of the work. The +whole-phase round found that **the caps-lift ADR-0068 exists to deliver was never implemented** — a name collision +with the Step-4b-3 wrap cache — so a >4000-character answer was still clipped in the very viewport built to scroll +it; that `displayWidth` under-counted 8 539 code points against `ink`'s own width function; and that a keyboard +Ctrl-C during a Home hatch stranded mouse reporting on the user's shell. All folded, each with a break-verified +regression test. + +**Open obligations carried out of 2.6.F:** + +- **The Home landing can overflow a short terminal** — a populated "Attention required" section pushes the top off, + and the alt buffer has no scrollback to recover it. Predates 2.6.F (the strip is 2.5.B); **2.6.G**'s management + browsers replace the strip wholesale, which is the right place to fix it. +- **`relavium run`'s TUI stays inline** (a deliberate 2.6.F scope cut); its retained-scrollable-history is tracked. +- **Themes** (beyond colour / `NO_COLOR`) are deferred to **2.6.L**, per the maintainer's Step-5 decision. +- Real-TTY signal paths (double-Ctrl-C, `kill -TERM`/`-HUP`/`-QUIT`, a hatch under `tmux`) remain a **manual PR-time + check** — the unit tests pin the orchestration around injected seams, not the kernel. + ### 2.6.G — Home management browsers: workflows, runs, agents The interactive browsers that make the Home the management center. The interaction model follows the @@ -899,6 +982,11 @@ sensitive-read floors). approval prompt even renders, defeating the ADR-0057 approval-floor investment. Wire an `allowed_domains` config path alongside the credential-resolver activation above — the same fix pattern. *(S · `apps/cli/src/chat/session-host.ts` (`chatToolPolicy`); G30)* + *(pulled forward, 2026-10-19)*: G30 is not 2.6.M work any more. The approved Wave 3 plan lands it in + [Phase 2.5.5 execution row 13](phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) under + Accepted [ADR-0112](../../decisions/0112-chat-network-grants-and-onboarding-state-are-operator-owned.md): a + global-only `[chat_http].allowed_domains` with live narrowing authority, explicitly **not** coupled to the + credential-resolver or `web_search` activation above. The prescription in this bullet is historical. - **`extra_roots`** *(pulled forward, 2026-10-12)*: invocation grants and committed narrowing selectors are accepted in [Wave 2 Step 2](phase-2.5.5-hardening-and-remediation.md#step-2-acceptance--2026-10-12). Config never grants authority. - **Tool-call rendering v2** (with 2.6.F's renderer): keep the collapsed one-line annotation as the default, add a **details view** (a `/details` toggle or transcript overlay) revealing *sanitized, @@ -923,6 +1011,9 @@ sensitive-read floors). wired for chat in 2.6.N — agent spawning is separate from workflow composition.) - **Default chat agent grant review**: widen the built-in agent's grant to the new idempotent read tools (search/find/todo); write/exec/egress stay opt-in via mode + approval. + *Direction set 2026-11-04 (D62, see [MVP-A14](#cli-mvp-additions--2026-11-04)):* the built-in agent also gets + `write_file`/`edit_file` and `run_command` under the approval card; this lands only with its ADR-0057 amendment and + security review. - **Curl/wget as web-search substrate** *(research item)*: evaluate whether `curl` / `wget` / `httpie` / `xh` should be **default-allowed** in the built-in chat agent's `allowed_commands` as an alternative web-search path — the model can already use `web_search` (once credential-resolved), but a @@ -1213,6 +1304,13 @@ models the four shipped adapters can **already** call and to ADR-0065 custom end opt-in *local-first*) and **clamp vs reject**. It closes the `error: validation` gap in [ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md). A **mandatory security review** of untrusted-community-pricing-into-a-safety-control is a co-gate. +- **Usage capture for streams that end without usage** *(added 2026-11-04, after the MVP; 2.6.C's open + obligation, [ADR-0070](../../decisions/0070-durable-per-model-session-cost-attribution.md) §3 exceptions + 1–2).* An egress that streamed content but ended without a usage chunk, and a mid-stream failure, are + recorded as 0 on both sides of the reconciliation invariant, so reported spend under-estimates the + provider's invoice. Capture or estimate their usage at the seam and mark it as such, or show the + under-report on the money surfaces; ADR-0070 §3's exceptions narrow with it. A PR #96 review found 2.6.C's + "filed against 2.6.Q" with no item here; the maintainer ruled this phase owns it. - **Tests:** an over-ceiling `max_tokens` is clamped per model; an OpenAI reasoning id uses `max_completion_tokens`; an out-of-range effort never reaches the wire and the picker offers only accepted tiers; a reachable-but-unpriced id gets a models-dev price so the cap applies (or is refused under @@ -1232,6 +1330,40 @@ extends ADR-0064; rides ADR-0065 (SSRF / user-pricing precedent) and ADR-0066 (t [ADR-0011](../../decisions/0011-internal-llm-abstraction.md)-respecting (host-side data source, not a `listModels` call). Mandatory security review is a co-gate. +## CLI MVP additions — 2026-11-04 + +The CLI MVP (decision D58, milestone M3.5, v0.2.0) is ordered in [current.md](../current.md#next-work--the-cli-mvp). +Most of its items already have a home in a workstream above. This section is the one home for the items that had +none, found by the 2026-11-04 review of the code at `5a328bca` and ruled by the maintainer +([priority reset](phase-2.5.5-hardening-and-remediation.md#mvp-priority-reset--2026-11-04)). Each belongs to the +workstream named in its row and follows that workstream's ADR and security gates. + +| ID | Workstream | Item and acceptance | +|---|---|---| +| MVP-A1 | 2.6.G | **Human run output.** `relavium run` and `logs` print the workflow's outputs on the human surfaces, bounded and through the shared sanitize floor; `--json` is unchanged. | +| MVP-A2 | 2.6.G | **The run id is visible.** The TTY run view and its final summary show the run id that `logs`, `status` and `gate` accept. | +| MVP-A3 | 2.6.B | **`/export` writes where workflows live.** It writes into `.relavium/workflows/` with a slug from the session title (no UUID file in the working directory) and offers to run the result. | +| MVP-A4 | 2.6.J | **The default model follows the stored key.** With no configured default, the chat model comes from a provider that has a key, and a missing key yields the actionable "no API key … `provider set-key`" error instead of an opaque credential failure. | +| MVP-A5 | 2.6.J | **`relavium chat` offers onboarding** when no key exists, without re-running the wizard on every launch when the keychain is unavailable (#55's loop stays fixed in its own row). | +| MVP-A6 | 2.6.J | **Standard provider environment variables** (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, `DEEPSEEK_API_KEY`) are read after the `RELAVIUM_*` variables, never logged or echoed; [config-spec.md](../../reference/contracts/config-spec.md) is updated with the code. Key-custody review. | +| MVP-A7 | 2.6.Q | **Interactive output budget.** No implicit 4,096-token cap on interactive Anthropic chat: the default is the catalog ceiling, clamped, with a `[chat].max_tokens` key; the budget estimate's interplay (ADR-0101) is checked. | +| MVP-A8 | 2.6.M | **Truthful tool rows.** A failed tool call shows a failure mark, not ✓, and the assistant's text between tool calls stays visible. | +| MVP-A9 | 2.6.M | **Ranged reads.** `read_file` takes line or byte ranges and bounded output spills to a wired `ToolHost.outputStore`; its model-visible description matches the CLI's binary refusal. | +| MVP-A10 | 2.6.M | **Denials reach the model.** A user or guardrail denial returns to the model as a `tool_result` instead of ending the turn (ADR-0057 amendment, D67). | +| MVP-A11 | 2.6.M | **Configurable, finite ceilings.** Tool rounds, correction attempts and the session turn limit are configurable with conservative defaults and stay finite (ADR-0086). | +| MVP-A12 | 2.6.E | **One keymap.** Ctrl+C interrupts the running turn, a second Ctrl+C ends the session, and Ctrl+C at a pending approval rejects it before anything else (keymap ADR, D68; supersedes the 2026-10-19 `/cancel` wording; approval-path sitting). | +| MVP-A13 | 2.6.M | **`[chat].default_mode`.** The starting mode is configurable (default `ask`), with a one-time hint that Shift+Tab changes it (ADR-0057 amendment, D62). | +| MVP-A14 | 2.6.M | **Default grant (D62).** The built-in agent gets the read, search and find tools, `write_file` and `edit_file` (hidden in ask/plan, per-call diff approval in accept-edits) and `run_command` under MVP-A15; egress stays opt-in. Lands only after the approval diff; ADR-0057 amendment and security review. | +| MVP-A15 | 2.6.G | **`run_command` approval card (D63).** A non-allowlisted command prompts once / for the session / to remember; remembered grants live outside the repository, in operator-owned configuration keyed by project root; accept-edits keeps prompting for allowlisted commands; headless paths stay fail-closed; `!` uses the same card. New ADR amending ADR-0029(a)/0061/0063, and a security sitting. | +| MVP-A16 | 2.6.D | **Project instructions.** A project instructions file (`AGENTS.md`) and an environment block enter the system prompt under explicit provenance rules (new ADR; independent of ADR-0060's `{{ctx.*}}`). | +| MVP-A17 | 2.6.E | **Session entry flags.** `relavium chat --model/--provider` and a one-shot prompt mode; headless stays fail-closed; both follow ADR-0110's compatibility rules. | +| MVP-A18 | 2.6.B | **A minimal `relavium init`** scaffolds `.relavium/` instead of being a dead command. | +| MVP-A19 | 2.6.B | **Starter content.** Two or three starter workflows and agents a new user can run unchanged. | +| MVP-A20 | 2.6.I | **Masked `provider set-key` on a TTY** instead of refusing an interactive terminal. Secret-input sitting. | +| MVP-A21 | 2.6.I | **Registered MCP servers reach the default agent**, under ADR-0084 spawn consent and ADR-0088's untrusted tool definitions. | +| MVP-A22 | 2.6.F | **Upgrade from `0.1.1`.** A populated `0.1.1` home (history.db with migrations 0000–0004, config.toml, keychain entries) upgrades through v0.2.0, with a seeded-then-migrated test. | +| MVP-A23 | 2.6.F | **Release prerequisites for v0.2.0.** The D59 release ADR and guard change; version and CHANGELOG including the Node ≥22 break; npm token, 2FA, provenance and the `v*` ruleset verified; the publish credential scoped to the release job, either npm trusted publishing (OIDC, no long-lived token) or the `release` deployment environment with a `v*`-tag deployment policy and a required reviewer holding `NPM_TOKEN`, with the repository-level secret deleted (PR #96 review, 2026-11-04; chosen in the D59 release ADR); the interactive Home and chat checked on a real Windows terminal; a manual three-provider end-to-end turn; first-read docs (README, npm README, tutorials, runbooks) describing what ships. | + ## Deferred-tasks pulled into this phase The [deferred-tasks.md](../deferred-tasks.md) triage (2026-07-08) mapped these confirmed-open items into @@ -1298,7 +1430,7 @@ separate backlog. | In-phase | Completed by | Outcome | |----------|--------------|---------| -| M2.6-1 Foundation | 2.6.F | Node floor decided/shipped; full-screen renderer + TUI harness — the substrate for every arm | +| M2.6-1 Foundation | 2.6.F | Node floor decided/shipped; full-screen renderer + TUI harness — the substrate for every arm *(met except the v0.2.0 release clause, which closes with the CLI MVP's MVP-10)* | | M2.6-2 Authoring core shared | 2.6.A | `@relavium/authoring` + catalog-aware pre-flight | | M2.6-3 Conversational authoring | 2.6.B + 2.6.D | "define a workflow…" → valid YAML in the Home; `{{ctx.*}}` lands | | M2.6-4 The Home-managed CLI | 2.6.G + 2.6.H + 2.6.I + 2.6.J + 2.6.K | Every management task doable from Home; drillable, attributed run history; onboarding v2 | @@ -1506,9 +1638,9 @@ flowchart LR | 6 | *(new)* | Management browsers + durable run detail (amends ADR-0036) | Drafted when 2.6.G starts | 2.6.G / 2.6.H | | 7 | *(new)* | MCP management surface + config-write extension (extends ADR-0063) | Drafted when 2.6.I starts | 2.6.I | | 8 | *(new)* | Onboarding auth paths + Relavium-account forward design (rides ADR-0012–0015) | Drafted when 2.6.J starts | 2.6.J | -| 9 | *(new)* | Toolbelt additions + tool-render/approval-preview contract + the `ask_user` mid-turn pause/resume **engine** interaction (rides the EA4 pause machine; a turn-level pause distinct from the approval pause) (extends ADR-0029/ADR-0057) | Drafted when 2.6.M starts | 2.6.M | +| 9 | *(new)* | Toolbelt additions + tool-render/approval-preview contract + the `ask_user` mid-turn pause/resume **engine** interaction (rides the EA4 pause machine; a turn-level pause distinct from the approval pause) (extends ADR-0029/ADR-0057) | Drafted when 2.6.M starts *Split 2026-11-04 for the MVP: the toolbelt and the tool-render/approval-preview contract first (MVP-3), the `ask_user` pause later.* | 2.6.M | | 10 | *(new)* | i18n + theming architecture | Drafted when 2.6.L starts | 2.6.L | -| 11 | *(new)* | Syntax highlighting dependency (`highlight.js` + `cli-highlight` via `ink-syntax-highlight`) + markdown rendering architecture | Drafted when 2.6.E starts | 2.6.E | +| 11 | *(new)* | Syntax highlighting dependency (`highlight.js` + `cli-highlight` via `ink-syntax-highlight`) + markdown rendering architecture | Drafted when 2.6.E starts *Split 2026-11-04 for the MVP: part 1 is an in-house markdown/code-block renderer with no dependency (MVP-2); the highlighter decision follows the MVP.* | 2.6.E | | 12 | *(new)* | Child-session **foundation** (extends ADR-0024/ADR-0036; parent-child lineage schema, standardized I/O contract, the **host artifact-store port** + central ephemeral root + fs-floor home-anchoring, catalog spawn, cost roll-up, abort propagation, depth/concurrency guardrails) | Drafted when 2.6.N starts | 2.6.N | | 13 | *(new)* | On-the-fly **generation + parallel orchestration security** (the generation-as-codegen threat model, schema-valid≠safe, tool-grant clamping, the parallel error-policy contract; sits on ADR-0060's taint precedent) — **mandatory security review is the ship-gate** | Drafted when 2.6.O starts | 2.6.O | | 14 | *(new)* | `subworkflow` node + **`schema_version` 1.1** additive migration (the first versioned-schema bump — additive/opt-in, subworkflow-only not `loop`; the nested-run event namespace; the recorded deferral of dynamic `invoke_workflow`) | Drafted when 2.6.P starts | 2.6.P | @@ -1548,3 +1680,26 @@ flowchart LR | Competitor-copy erodes Relavium's posture | Patterns are stolen, postures are not: keychain-only secrets, fail-closed approval, YAML-first artifacts, and the ADR-0049 machine contract are non-negotiable filters on every borrowed idea | Part of [roadmap/](../README.md). Carry-over hardening lives in [../deferred-tasks.md](../deferred-tasks.md). +## Status record moved from current.md — 2026-11-04 + +> Moved verbatim from `docs/roadmap/current.md` as it stood at `5a328bca` by the +> [PR #96 review correction](phase-2.5.5-hardening-and-remediation.md#moved-by-the-pr-96-review-correction--2026-11-04). +> Only relative links were rewritten for this directory and heading levels lowered by one. + +### Not started yet + +The rest of **Phase 2.6 — Conversational Authoring and the First-Class CLI** +([phase-2.6-conversational-authoring.md](), in progress — +**2.6.F is done**, +unblocked by the 2.5 close and **re-scoped 2026-07-08** from maintainer UX findings + a competitor +research pass + the deferred-tasks triage): a full-screen, Home-managed CLI (browsers for +workflows/runs/agents, provider + MCP + settings management, onboarding v2 with the Relavium-account +stub), competitor-breadth tools under the governance floor, settings/theming/`en`+`tr` localization, +the run-ops resume follow-up — plus the original spine: the shared `@relavium/authoring` package and +a chat that **authors** standards-valid `.relavium.yaml`. Then the surfaces and the cloud — +everything after the engine critical path: the desktop app (Phase 3) and the VS Code extension (Phase 4), then **Product Phase 2** — first **managed +inference** ([phase-5-managed-inference.md](phase-5-managed-inference.md), the opt-in +`managed` gateway, engine still local), then the **cloud execution layer and web portal** +([phase-6-cloud-execution-portal.md](phase-6-cloud-execution-portal.md)), the two +decoupled per Option B. See the [phase index](../README.md#phase-index) and the +[milestone spine](../README.md#global-milestone-spine) (M3 onward). diff --git a/docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md b/docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md index 6169cff2..b3c1bbd3 100644 --- a/docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md +++ b/docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md @@ -1553,6 +1553,10 @@ Every connect failure had collapsed into one sentence, and every identifier was carried — the package's own tests had to regex human prose to find a server id. Fields and a `reason` now, derived from the caught error's SHAPE, never from its text. +### W4 start gate — maintainer ruling (moved from current.md on 2026-11-04) + +**The `W4`-waits-for-the-`W3`-blocker gate was lifted deliberately on 2026-09-01, by the maintainer, and it is recorded here rather than left to be inferred from the commit order.** The reasoning: the blocker is a `BoundedEventStream` drop on a stream nobody iterates, its fix is ADR-0087 §1 — a decision that was still **Proposed** and unapproved — and `W4` touches a disjoint surface (the MCP boundary). Blocking a whole wave on an unapproved ADR would have stalled the phase rather than protected it. **The blocker stays open and stays first in the `W3` residuals**; nothing here closes it or reduces its severity. **Updated 2026-09-04:** ADR-0087 is now **Accepted**, with nine dated corrections written into it after a review refused acceptance. That removes the "unapproved ADR" half of this gate — §1 is now a settled decision — but §1 remains **unimplemented** and the blocker is unchanged. Its Correction 1 records what §1 must settle before it can be built: the two factories §1 names are not symmetric — a host calls `createSessionHandle` directly, but a `RunHandle` is only ever produced inside `WorkflowEngine.start()` / `resumeFromCheckpoint()`, so the run half needs the mode plumbed onto an engine entry point before it can be passed at all. A `subscribe-only` handle must also not present a silently-empty iterable. + ### W4 closing register **The register, per exit criterion 7 — the code that closes each item, so a reader can check the claim.** @@ -7086,3 +7090,603 @@ blocker, followed by canonical request conformance and unknown usage. Its prefli ADR-0087's accepted corrections with the current W7 lifetime contracts. After W8, re-evaluate every phase exit criterion before the next remediation wave opens; [current.md](../current.md#next-work--wave-2--shut-the-doors) remains canonical for execution order. W8 implementation is not started by this documentation update. +## Execution record moved from current.md — 2026-11-04 + +> Moved verbatim from `docs/roadmap/current.md` as it stood at `5a328bca` by the +> [PR #96 review correction](phase-2.5.5-hardening-and-remediation.md#moved-by-the-pr-96-review-correction--2026-11-04): +> the D66 reset had kept only current.md's live status and order and recorded its 2026-07-26 plan in the Phase 2.5.5 +> register, and these Phase 2.6.5 records had left the docs. Only relative links were rewritten for this directory +> and heading levels lowered by one; a heading in brackets is added. The `W4` start-gate paragraph is kept in +> [its earlier section](#w4-start-gate--maintainer-ruling-moved-from-currentmd-on-2026-11-04). + +### [current.md's header notes, as of 2026-10-25] + + + +> **W7 merged to `main`, 2026-10-10 (PR #90, `46a6a701`):** all twelve steps, five genuine overflow records, +> production request ownership and consolidated host lifetimes have independent acceptance. +> All six W7 items are closed; Phase 2.6.5 stood at **47/51 at that merge**. The +> [closing register](#w7-closing-register--2026-10-10) +> joins per-item causal controls, both final review rounds and every submitted systematic finding. +> PR #90 merged at 13:57:47 UTC. W8 was still open at that checkpoint; it subsequently merged in PR #92. Approved deferrals remain. +> No implementation, user decision, credential, billing or provider capture is pending for W7. + +> **Post-closure corrections accepted, 2026-10-10:** the [new correction register](#post-closure-systematic-review--2026-10-10) +> joins every new report disposition. Core, CLI/DB and tooling/docs each have two NEW complete clean +> cumulative review rounds. Fresh unchanged-production coverage passes 447 files / 9,512 cases / +> eleven existing capability/key-gated LLM skips. Published-head checks are reported in PR #90. +> W7 has no implementation, provider, credential, billing or user-decision prerequisite left. +> The corrections merged with PR #90; W8 subsequently merged in PR #92. Existing approved limits remain. + +> **Publication tool corrections accepted, 2026-10-10:** [two NEW complete clean cumulative rounds](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-10T13-35-14-w7-publication-sonar-round-2-review.md) +> independently verify all four Sonar-label dispositions, preserved UTF-16 ordering and OS-owned Git +> dispatch with PATH-trap causal controls. Shipping production/tests remain unchanged. Final local CI +> and exact published-head GitHub/Sonar checks are reported in PR #90; the prior failed analysis stays recorded. +> W7 has no implementation, capture, credential, billing or user-decision prerequisite left. + + +### W8 merged — Phase 2.6.5 complete + +**W8 merged to `main` on 2026-10-11 as [PR #92](https://github.com/HodeTech/Relavium/pull/92), +merge commit `ef0ffa54`. Phase 2.6.5 is complete with approved deferrals: 49/51 items closed.** +The [merge checkpoint](#w8-merge-checkpoint--2026-10-11) +joins final publication evidence and all seven phase exit criteria. All seven W8 steps, +CR-81, CR-82 and ADR-0087 §1 are complete on `main`; no W8 implementation, review, +publication, provider, billing, credential or user-decision gate remains. + +1. **Consumption:** explicit run/session modes preserve iterated delivery from construction; + subscribe-only handles retain no primary buffer and reject iteration with a typed error. +2. **Conformance:** strict expected exchanges and the 24-case actual installed-SDK HTTP + matrix detect production-lowering regressions. +3. **Usage uncertainty:** complete/partial/missing observations, pending ACK before invocation, + once-only settlement and late receipts, sticky old-writer uncertainty and qualified + human/JSON totals preserve the distinction between unknown usage and actual zero. + +The [original closing register](#w8-closing-register--2026-10-11) +preserves its source acceptance and two NEW complete whole-wave reviews. The +[maintainer correction register](#w8-pr-92-systematic-review-corrections--2026-10-11) +joins all report dispositions and two NEW independent complete clean cumulative rounds per +correction increment. Final reviewed head `3d4491df` passed both seven-job PR/push CI +workflows and the SonarCloud quality gate. Published ordinary/enforced-coverage inventories +are 511 files / 11,026 passing tests / 12 established skips; four wire and 23 usage +mechanisms retain 427 permanent cases and 69 joined native children. Fresh full local +CI and bare/enforced coverage at `6139563e` remain separately attributed evidence. +No merge-commit execution, live-provider invoice or unsupported-platform result is inferred. +Two nonblocking Sonar complexity suggestions remain visible. Detailed review records stay +local and ignored; historical citations remain commit-pinned. + +**What remains:** CR-70 (cross-turn tool-history carrying) and CR-93 (tenant-scoped mutable +state) are approved deferrals. The 49/51 count includes CR-95's short-term fail-closed fix; +its long continuation and existing exact-capacity/scoped residuals remain in +[deferred-tasks.md](../deferred-tasks.md), with their existing severity, trigger and narrowed +claim. ADR-0087 §1 is outside the denominator. Phase closure does not claim 51 fully +implemented items or an empty debt backlog. Phase 2.5.5 and Phase 2.6 remain in progress. +Earlier dated pending checkpoints in this page and the phase register are historical. + + + + + + +### Historical W7 implementation checkpoints + +Dated W7 checkpoints throughout this page retain their original pending states. The closing checkpoint above +and its later post-closure and merge checkpoints control current W7 status; historical pending/draft +wording describes the checkpoint when it was written. + +> **Prior complete composition review:** [host-closure round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-10T00-38-24-w7-host-closure-round-5-review.md) +> finds no new confirmed actionable issue across all 126 paths / 364 hunks. This is the first +> clean complete cumulative round; one further fresh complete clean round remains required. +> Step 8 and final Step 12 remain open, with no user/provider blocker. +> +> **Complete composition review:** [host-closure round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-10T00-22-00-w7-host-closure-round-4-review.md) +> confirms public stream exit can skip underlying cleanup on retirement failure. Parent adds +> twelve paired permanent cases and guarantees cleanup while preserving the original throw. +> Two fresh complete clean rounds, Step 8 and final Step 12 remain required; no user input is pending. +> +> **Complete composition review:** [host-closure round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T23-56-41-w7-host-closure-round-3-review.md) +> confirms two retirement-boundary Highs: mutable returned output/usage and failed cleanup +> suppressing accounting/acknowledgement. Parent corrections add thirty-two permanent cases. +> Fresh complete review remains required; Step 8/12 remain open, with no user/provider blocker. +> +> **Complete composition review:** [host-closure round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T23-34-00-w7-host-closure-round-2-review.md) +> confirms due-budget departure starvation and provider entry after synchronous cancellation. +> Parent corrections add nineteen permanent cases and fix adjacent canonical-documentation drift. +> Fresh complete review is required; Step 8/12 remain open. No user/provider blocker exists. +> +> **Complete composition review:** [host-closure round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T23-05-00-w7-host-closure-round-1-review.md) +> confirms three lifecycle High findings (post-pause receipt cursor, rejected activation joining, +> post-unmount Ink visibility) and a required shipping-registry acceptance gap. Parent corrections +> add eighteen permanent cases and strengthen native output assertions. Fresh complete review of +> the corrected composition remains required before acceptance; Step 8/12 are still open. + +> **W7 live status, 2026-10-09:** all five immutable provider records and Step 7's four-round +> acceptance remain complete. [Two fresh adapter/chain round-3 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T23-47-31-w7-owned-adapter-chain-round-3-review.md) +> accept all 24 cumulative paths/78 hunks through `0c3c185b`; exact-head required CI, coverage, +> Node 22, Windows, peers and Sonar pass. Core measured-round reuse is implemented at `99e78e55`; +> [both independent round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T00-16-47-w7-owned-core-round-1-review.md) +> clear all 13 paths/47 hunks. [Two fresh round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T01-00-00-w7-owned-core-round-2-review.md) +> accept all 19 cumulative paths/53 hunks through `f1281214`; exact-head CI passes. ADR-0103 +> append/receipt integration is implemented at `a35739a4`; +> [both complete round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T01-27-00-w7-append-receipt-round-1-review.md) clear all 17 paths/57 hunks. +> [Two fresh cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T02-00-00-w7-append-receipt-round-2-review.md) accept all 23 paths/63 hunks +> through `01115050`. Raw provider/iterator/media and native child lifetime integration is +> implemented at `fde3ca0c`; [both complete round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T02-36-10-w7-raw-producer-round-1-review.md) +> verify a public-return High, corrected with two regressions (28 permanent cases total). +> [Two fresh cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T03-25-00-w7-raw-producer-round-2-review.md) accept all 20 paths/41 hunks +> through `1172d247` after complete report/artifact/freeze audits; nested Corepack reviewer +> violations get no proof credit. The HTTP descendant increment now has 23 new permanent +> cases, 16 compiled negative controls and a passing cache-qualified root CI; +> [both complete round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T04-16-24-w7-http-descendants-round-1-review.md) +> clear all 22 paths/38 hunks through `db02b83c` after Parent's complete artifact/freeze audits. +> [Two fresh cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T04-55-00-w7-http-descendants-round-2-review.md) +> accept all 24 paths/40 hunks through `6e620556` after Parent's complete evidence and 645,606-entry freeze audits. +> Invocation-local provider/raw-poll integration adds 57 permanent cases; [complete round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T05-43-00-w7-provider-invocation-round-1-review.md) +> verify a method-receiver defect and separately strengthen a weak Promise test. +> [Both complete round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T06-33-00-w7-provider-invocation-round-2-review.md) verify one getter-acquisition defect. +> Its correction passes root CI (404 files / 9,030 cases / 12 existing skips); +> [both NEW complete cumulative round-3 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T07-06-28-w7-provider-invocation-round-3-review.md) +> and Parent's complete artifact/645,743-entry freeze audits accept all 28 paths/67 hunks through +> `f9a8ff37`, with no new confirmed finding. Exact-head required CI, coverage and Sonar pass. +> MCP transport/handler descendants now have 77 new permanent cases. Rounds 1–3 corrected +> queued-reservation, deadline, disposer, coverage, empty-start stdio and callback failures. +> [Complete round-4 reviews and Parent evidence/freeze audits](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T12-58-01-w7-mcp-lifetime-round-4-review.md) verify +> completed control-lane reference retention. Correction `37220a95` releases only fully +> acknowledged lanes and passes actual root CI; seven new causal regressions and fresh enforced +> private coverage pass (412 files / 9,107 cases / 12 skips; MCP branches 90.68%). +> [Two NEW complete cumulative round-5 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T13-37-00-w7-mcp-lifetime-round-5-review.md) +> and Parent's complete source/artifact audits accept all 38 paths/70 hunks through `99c46bbd`, +> with no new confirmed finding and 268 independently passing cases per reviewer. The full +> shared-tree audit has zero file/link changes and one qualified `.git` directory mtime change. +> Fresh startup registers its complete continuation. [Two complete first-round reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T14-28-09-w7-startup-lifetime-round-1-review.md) +> and Parent audits verify reentrant clock/timer cancellation; three permanent regressions +> cover the correction. [Two NEW complete cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T14-55-12-w7-startup-lifetime-round-2-review.md) +> and Parent's exact-source/physical-artifact/647,041-entry freeze audits accept all ten +> paths/sixteen hunks through `5d44b13b`, with zero new confirmed findings and 2,850 core cases per reviewer. +> [Both complete scheduler readiness round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T15-44-15-w7-scheduler-readiness-round-1-review.md) +> verify a secondary readiness-diagnostic failure. Its correction has twelve causal readiness +> regressions and 2,862 passing private core cases. [Two NEW complete cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T16-14-29-w7-scheduler-readiness-round-2-review.md) +> and Parent's source/artifact/647,245-entry unchanged freeze audit accept all nine paths/ten +> hunks through `95f3923a`, with zero new readiness findings. Both inherited dispatch Highs +> (post-start-append cancellation admission and unsafe detached failure formatting) are +> corrected in `1918690d`, with six new permanent cases and passing exact-input root CI. +> [Both complete dispatch-correction round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T17-28-06-w7-dispatch-corrections-round-1-review.md) +> and Parent's artifact/647,355-entry unchanged freeze audits accept the eight-path/ +> seventeen-hunk scope with zero confirmed findings. [Both complete cumulative second-round reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T17-58-41-w7-dispatch-corrections-round-2-review.md) +> verify an inherited retry-start cancellation High. Its correction and a subsequent +> Parent-discovered synchronous attempt-entry correction add ten permanent cases; +> final-source causal controls pass. NEW complete cumulative reviews are required +> before scoped closure; complete scheduler/dispatch roots stay open. +> [Complete cumulative round-3 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T18-37-30-w7-dispatch-corrections-round-3-review.md) verify a retry/deadline duplicate-terminal High. +> Parent confirms and corrects it, with a new permanent causal regression; fresh cumulative review remains required. +> [Complete cumulative round-4 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T19-10-39-w7-dispatch-corrections-round-4-review.md) verify two paths of one attempt-correlation High. +> Parent corrects explicit failure forwarding and first-start reset with five permanent cases; NEW cumulative review is required. +> [Complete cumulative round-5 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T19-41-03-w7-dispatch-corrections-round-5-review.md) verify a pre-start readiness attempt-correlation High. +> Parent corrects its latest-entered-attempt forwarding with two paired permanent cases; NEW cumulative review is required. +> [Complete cumulative round 6](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T20-08-19-w7-dispatch-corrections-round-6-review.md) finds no new confirmed issue. +> [The NEW complete round 7](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T20-29-02-w7-dispatch-corrections-round-7-review.md) verifies a differing-order grace High: +> an unstarted claim can receive a failure. Parent corrects explicit claim/start tracking, +> with three permanent causal cases; NEW complete cumulative reviews remain required. +> A broader private actor prototype exposed CLI closure/lease-handoff failures in root CI. +> Complete scheduler/dispatch/resume registration must land with shipping host departure; +> original failures and assertions are retained, and that integration stays required in W7. +> Complete MCP startup/all-actor retirement, custom-provider descendants and remaining host departure work +> and Step 8 atomic compaction/recovery remain open. The ownership/lifecycle +> Highs, Step 12, final whole-wave CI/coverage/Sonar and all six W7 register items remain open. +> Draft PR #90 is unmerged; no further paid call, credential or ADR approval is needed. +> Earlier progress statements below describe their publication checkpoints. + +> **W7 implementation, 2026-10-02:** the documentation review is closed. The maintainer approved +> [ADR-0099](../../decisions/0099-compaction-has-an-idle-budget-outcome-and-an-unknown-window-policy.md), +> [ADR-0100](../../decisions/0100-budget-authorization-is-durable-state-with-a-replay-barrier.md) and the +> [twelve-step W7 plan](#w7-pre-implementation-review-and-proposed-execution-plan--2026-10-02). +> Work remains on `development`; each step closes with two fresh independent Codex review rounds and verified +> fixes. Step 1's implementation and first review corrections pass `pnpm run ci` and `pnpm coverage`; +> [round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-04-05-w7-step-1-round-1-review.md) verified seven findings; +> [round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-22-32-w7-step-1-round-2-review.md) verified a cleanup race in its fix. +> [Round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-35-41-w7-step-1-round-3-review.md) verified a passing/starting race; +> cleanup now retains shared parents and deletes only owned probes. All corrections pass the gates; +> [Round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-43-31-w7-step-1-round-4-review.md) cleared probe ownership and found +> a capture pathname-cleanup race. The runner now closes its descriptor without deleting a failed +> destination. [Round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-50-35-w7-step-1-round-5-review.md) cleared the +> complete step and final corrections; step 1 is closed. Step 2's structural store boundary and +> durable effect-turn allocator pass the required checks and root coverage (289 suites, 6,367 passing +> tests, 11 skipped); full CI passed at `1927daa1`. +> [Round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T04-29-12-w7-step-2-round-1-review.md) verified nested-transaction +> key reuse and unsafe session media metadata. Both are corrected; checks and coverage pass +> (289 suites, 6,374 passing tests, 11 skipped), with full CI passing at `c96eb9b5`. +> [Round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T04-41-54-w7-step-2-round-2-review.md) cleared the complete step +> and both fixes, including real lock contention and process-crash checks. Step 2 is closed; +> step 3 implementation now connects structural completed turns, atomic persistence, empty-final +> resume/export and per-call effect ids across the CLI hosts. Required checks and coverage pass; +> [Round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T05-34-44-w7-step-3-round-1-review.md) verified a deep-JSON +> metadata failure and a lost policy-denial classification. Both corrections pass the required +> checks and coverage (291 suites, 6,406 passing tests, 11 skipped); full CI passes at `4045272c`. +> [Round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T05-48-54-w7-step-3-round-2-review.md) verified a cached-command +> durability bypass and a missing tool name on recovered filesystem denial. Both corrections pass +> checks and coverage (291 suites, 6,410 passing tests, 11 skipped); full CI passes at `e95a6010`. +> [Round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T06-10-38-w7-step-3-round-3-review.md) independently verified an +> unjournaled-tool durability bypass and a stale export introduction. Dispatch admission now checks +> the live latch after approval/preparation for every actual tool call and releases a proven +> non-dispatch claim. Both corrections pass checks and coverage (291 suites, 6,415 passing tests, +> 11 skipped); full CI passes at `dd50738c`. +> [Round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T06-34-39-w7-step-3-round-4-review.md) cleared dispatch admission +> and verified loss of nontrailing pre-W7 empty-final user text from resume/reseat and export prompts. +> Legacy context and boundary slots now preserve the old read behaviour without fabricating terminals +> or completed counts. Required checks and coverage pass (291 suites, 6,422 passing tests, +> 11 skipped); full CI passes at `6d5b3f0b`. +> [Round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T06-48-45-w7-step-3-round-5-review.md) cleared the complete step +> and all earlier corrections, including independent mixed-history projection, YAML, real SQLite +> boundary and process-crash probes. **Step 3 is closed; step 4 proceeds automatically** with +> session-result privacy, full-history disclosure snapshots and exact committed-row retention. +> Step 4's [first acceptance round](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T08-05-02-w7-step-4-round-1-review.md) +> independently reproduced pre-mount Ink resume/reseat deletion. The verified correction separates +> opening from committed-mount activation and carries driver activity through disclosure and teardown; +> [Round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T08-48-12-w7-step-4-round-2-review.md) reproduced activation before passive Ink input setup +> and before disclosure was drawn. Activation now awaits usable setup and disclosure render acknowledgement, +> observes renderer exit/error and keeps input gated; actual Ink and Home tests verify the display boundary. +> Required checks and coverage pass (297 suites, 6,487 passing tests, 11 skipped); a third fresh +> corrective acceptance round was required before step 4 closes. +> [Round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T09-24-27-w7-step-4-round-3-review.md) verified incomplete-call attribution and +> flush acknowledgement of unwritable output. Completion now uses the shared structural projector; +> rendered disclosure checks the actual Ink output and its lifetime. Required checks, build and coverage +> pass (297 suites, 6,497 passing tests, 11 skipped); a fourth fresh acceptance round is required. +> [Round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T10-04-04-w7-step-4-round-4-review.md) verified headless stderr enqueueing mistaken for delivery +> and inline disclosure lost during Ctrl-Z suspension. Native stderr callbacks now acknowledge delivery; +> notice publication and render acknowledgement share terminal ownership with suspension. Nine focused suites +> pass 217 tests, including delayed native delivery and independent real STOP/CONT controls. +> Required workspace checks, build and coverage pass (298 suites, 6,514 tests, 11 skipped). Full CI +> passed at `b9d52d76`; round 5 required two more corrections. +> [Round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T10-36-22-w7-step-4-round-5-review.md) independently reproduced blocked native stderr +> holding Ctrl-C teardown and late Home acknowledgement deleting evidence during canceled-session MCP close. +> Headless activation now races an owned interrupt; Home activation requires a running session outside teardown. +> Eight focused suites pass 310 tests; native pipe/FIFO before/after controls and late callbacks after actual +> database close pass. Workspace checks, build and coverage pass (298 suites, 6,520 tests, 11 skipped). +> Full CI passed at `e753beb7`. [Round 6](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T10-57-58-w7-step-4-round-6-review.md) accepted the complete +> step and every correction with no new material findings. Parent native/privacy controls pass; compiled-entry +> process-exit delay under queued stdio also reproduces on the exact pre-step baseline and is explicitly qualified. +> **Step 4 is closed; step 5 proceeds automatically.** All six W7 register items remain open. +> Step 5's memory implementation and canonical docs are checked: +> one pure request projection, frozen policy, completed-turn spans and engine/CLI history refusals; +> actual request, workflow-node and SQLite resume/reseat controls pass. Workspace checks, build and coverage +> pass (299 suites, 6,567 passing tests, 11 skipped). Later measured pre-send/recovery and revised compaction +> outcomes remain staged. Clean-tree full CI passed at `ecdcc395`. +> [Step 5 round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T12-00-17-w7-step-5-round-1-review.md) accepted the implementation: +> both reviewers passed 347 focused tests; parent-rerun controls passed 390 state/history cases and 86 +> actual CLI/SQLite cases. Matched source-head operational controls pass; exact pre-step engine controls +> fail as expected. Adjacent working-context/durable-history wording is corrected. +> [Step 5 round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T12-21-33-w7-step-5-round-2-review.md) accepted the complete step: +> both fresh reviewers passed 347 tests; the parent reproduced 1,536 state/history and 86 CLI/Home/SQLite +> cases, plus exact-source positive/negative controls. **Steps 1–5 are closed; all six W7 items remain open.** +> Step 6's independently reviewed [Accepted ADR-0101](../../decisions/0101-configured-output-estimates-apply-only-when-the-wire-is-uncapped.md) +> resolves a newly identified configured-estimate/native-cap precedence gap. The maintainer approved the +> complete revised decision on 2026-10-02; dated forward notes are landed and step 6 proceeds; +> [ADR revision round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T15-09-58-w7-adr-0101-revision-round-1-review.md) records the maintainer's +> verified transport/pricing findings and the corrected measurement-to-attempt handoff. +> [Fresh draft round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T15-18-01-w7-adr-0101-revision-round-2-review.md) accepted the complete +> corrected proposal with no actionable findings; full CI passed at `2b628e1a`. The acceptance and runtime +> obligations remain distinct: runtime acceptance is now complete. +> [Step 6 round 8](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T22-44-01-w7-step-6-round-8-review.md) accepts the complete step and seven corrections. +> Both fresh reviewers reported no verified new findings; parent replays pass 120 contract and 54 +> financial controls. **Steps 1–6 are closed; all six W7 register items remain open.** +> Step 9's quote/debit foundation is implemented, with [candidate preflight and parent causal controls](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T23-40-16-w7-step-9-foundation-preflight-review.md). +> Full CI/coverage pass after the round 3 correction (315 test files, 7,152 passing tests, 11 existing skips). +> [Round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T00-25-08-w7-step-9-round-1-review.md) verified and corrected two caller settlement findings; [Round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T01-10-45-w7-step-9-round-2-review.md) verified and corrected the async consumer and cancellation teardown; [Round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T01-57-14-w7-step-9-round-3-review.md) verified and corrected park/poll host admission loss and the reentrant durable-total race. +> [Fresh round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T02-40-53-w7-step-9-round-4-review.md) accepts the complete foundation with no new findings; both sealed inventories and 70 parent-rerun fresh cases are verified. Its corrected native-loader evidence and prior harness isolation exception are recorded explicitly. +> **Step 9 is closed; approved Step 10 proceeds automatically** with durable workflow activation and replay. +> [Step 10 round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T08-35-49-w7-step-10-round-1-review.md) reproduced and corrected +> passive-resume lease expiry and unbound predecessor CJS/peer/optional edges. Exact-fence renewal and +> an owned, versioned dependency closure pass causal controls, full CI and coverage +> (320 suites, 7,318 passing tests, 11 existing skips), including seventy actual-runner cases, +> twelve real predecessor refusals, two legacy controls and seven permanent graph/runtime controls. +> [Step 10 round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T11-47-48-w7-step-10-round-2-review.md) verified and corrected +> terminal conservative-money ordering, historical budget input compatibility, companion amount symmetry +> and pinned runtime edges. Full CI/coverage pass (320 suites, 7,340 passing tests, 11 existing skips); +> 78 native runner cases, thirteen graph controls and actual historical-input replay pass causal checks. +> [Step 10 round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T12-39-00-w7-step-10-round-3-review.md) verifies and corrects +> authority-witnessed optional-amount duplicate compatibility, preserving strict money conflicts, +> genuine output and unrelated pending work. Eighteen permanent regressions and 300 native selected +> cases pass. Full CI/coverage pass (320 suites, 7,358 tests, 11 existing skips); fresh full-Step-10 +> round 4 precedes closure. All six register items remain open. +> [Step 10 round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T13-33-00-w7-step-10-round-4-review.md) corrects lease cleanup rejection replacing +> primary typed refusals or an already-terminal no-op. Eighteen native regressions cover all six +> guards and successor takeover; full CI/coverage pass (320 suites, 7,376 tests, 11 existing skips). +> Fresh whole-Step-10 round 5 precedes closure; no residual is deferred. +> [Step 10 round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T14-31-39-w7-step-10-round-5-review.md) corrects inherited parallel gate publication handing off +> and then reclaiming a waiting-only lease. Ten permanent regressions cover held pause ACKs, +> actual SQLite fresh-owner resume and immediate human-decision media preparation. Async final-ACK +> limits are explicit; no new clock/timeout policy is chosen. Full CI/coverage pass (320 suites, +> 7,386 tests, 11 existing skips). Fresh whole-Step-10 round 6 precedes closure. +> [Step 10 round 6](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T15-42-00-w7-step-10-round-6-review.md) corrects inherited ordinary +> decision media completion after cancellation, deadline or fenced closure. Twelve permanent +> Core/actual SQLite-CAS regressions prevent late subscriber success and renewed run references; +> guard removal/restoration confirms the failure. Full CI/coverage pass (321 suites, 7,398 tests, +> 11 existing skips). Fresh full-Step-10 round 7 precedes closure. +> [Step 10 round 7](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T16-54-28-w7-step-10-round-7-review.md) corrects inherited media references written before a stale +> event fence is refused. Produced handles now follow append acknowledgement; owned uncertain-terminal +> media remains retained for the outbox. Seven permanent successor/terminal controls and independently +> verified sealed evidence pass, with full CI and sequential coverage (323 files, 7,405 tests, +> 11 existing skips). Fresh full-Step-10 round 8 precedes closure. +> [Step 10 round 8](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T17-50-23-w7-step-10-round-8-review.md) accepts the full 74-file step with no new verified findings. +> Both fresh reviewers pass 857 permanent tests and separate five/six fresh probes; complete reports and +> every sealed inventory entry are parent-verified. **Step 10 is closed; approved Step 11 proceeds automatically** +> with the safe CLI surface, permanent compiled-process smoke coverage and fresh independent reviews. +> Step 11 implementation now passes full CI and sequential coverage (327 files, 7,506 tests, 11 existing skips), +> including permanent compiled CLI refusal/status/abort/MCP-interruption smokes and raw-triple display regressions. +> [Step 11 round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T19-39-26-w7-step-11-round-1-review.md) independently verifies native approval-write cancellation, +> the no-MCP signal guard and stale identified-companion display cleanup. The parent fixes all three, +> adds passive Core cancellation/fence controls and corrects the ownership wording. +> Full CI and sequential coverage pass after the corrections (327 files, 7,519 passing tests, +> 11 existing skips; global line/branch/function 95.88%/92.63%/96.73%). +> [Step 11 round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T21-01-16-w7-step-11-round-2-review.md) verifies two further High findings: +> a successful native busy wait can conceal OS SIGINT until after egress, and terminal write uncertainty +> was misreported as takeover. The CLI host yields across a poll turn before decision acknowledgement; +> terminal uncertainty retains exit 5, while actual successor fencing still refuses without a false +> claim that an earlier approval is absent. Seven new permanent controls pass; both complete sealed +> reports and every inventory entry are parent-verified. Full CI and sequential coverage pass +> (327 files, 7,526 tests, 11 existing skips; 95.88%/92.63%/96.73%). Fresh full-Step-11 round 3 follows +> the correction commits; the step remains open. +> [Fresh Step 11 round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T01-17-23-w7-step-11-round-3-review.md) is clean in both +> complete independent reviews: 452 permanent tests each plus separately reported fresh native, +> passive and compiled-process controls. Both reports and every sealed inventory entry are verified. +> Interrupted earlier round-3 runs contribute no acceptance. **Step 11 is closed.** +> [Independent Step 12 document preflight](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T01-04-15-w7-step-12-document-preflight-review.md) +> verifies and corrects usage/pricing and fixture-coverage prose; connected at-rest/event claims, +> guides and moved follow-up status are reconciled. This independent maintenance proceeds through +> repository checks and fresh post-commit review; Step 12's whole-wave acceptance remains open. +> [Committed maintenance round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T07-00-17-w7-step-12-maintenance-round-1-review.md) +> independently verifies one Medium reservation-existence qualification and one Low inherited +> event-table guarantee. Both are parent-verified and corrected in four text/comment locations. +> Complete reports and all 2,687 sealed entries are verified; 81 fresh reviewer DB tests and +> a real-writer reference-corruption/restoration control pass. Fresh complete maintenance +> round 2 follows the correction commits; whole Step 12 remains open. +> [Committed maintenance round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T07-25-20-w7-step-12-maintenance-round-2-review.md) is clean +> across all 26 paths/37 hunks in both fresh reviews. Both complete reports and all 2,574 sealed +> entries/source pins are verified; fresh 81-test DB/strict checks and real-writer causal controls pass. +> **The independent maintenance subset is closed; whole Step 12 and all six W7 items remain open.** +> [Draft PR #90](https://github.com/HodeTech/Relavium/pull/90) is open on `development`. +> Required Linux CI revealed an actual-Ink test-context mismatch. Further controlled checks found +> undisplayed disclosure under the existing CI opt-outs and Home's inline reseat Static cursor; +> corrections pass two fresh independent review rounds. Prior scoped acceptance does not waive other findings. +> All five required actual overflow records (Anthropic input rejection and native context stop, +> OpenAI, DeepSeek and Gemini) are now preserved in the +> [fixture index](../../../packages/llm/src/conformance/fixtures/overflow/README.md). +> The separate Anthropic native context-stop response was captured on 2026-10-08; +> the evidence prerequisite is satisfied. Step 7 classification, Step 8 and whole-wave acceptance remain open. +> Step 1 repaired collection without deleting private artifacts and provides the +> [maintainer-run live overflow capture procedure](../../runbooks/capture-provider-overflow.md). +> Classification must replay the preserved live fixtures. The six W7 items stay open. + +> **PR 90 systematic follow-up — 2026-10-04.** Two maintainer reports against `95137c34` register 35 +> deduplicated findings, with [qualified independent triage](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T10-35-00-w7-systematic-review-intake-review.md). +> Step 6's request-ownership guarantee is reopened; adjacent session/CLI, replay-tool and documentation +> obligations require correction and fresh acceptance. The [first committed rendering review](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T10-32-37-w7-rendering-round-1-review.md) +> reproduces notice loss after a supported chat resizes and CI route/ownership drift. Its corrections +> are accepted by [fresh round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T12-00-00-w7-rendering-round-2-review.md). +> The rendering corrective group is closed. Previous step closure records remain historical evidence, +> not current whole-wave acceptance. All ten GitHub workflow checks pass at `d4e4fa9c`; SonarCloud and remaining findings stay +> separately under examination. No finding is deferred at intake. Steps 7–8, whole Step 12 and all six +> W7 register items remain open. + +> **Request-ownership decision preflight — 2026-10-04.** [Proposed ADR-0102](../../decisions/0102-a-measured-request-owns-its-inert-data-through-egress.md) +> would close the measured-to-sent alias gap through one owned inert construction and existing candidate +> cap authority. It deliberately narrows non-cap opaque inputs and requires maintainer approval before +> dependent implementation. [Draft round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T12-21-41-w7-adr-0102-round-1-review.md) +> corrects SDK shared-reference parity and incoming measured-plan ordering. [Fresh complete round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T13-02-54-w7-adr-0102-round-2-review.md) +> receives no new material proposal findings; maintainer approval remains required before dependent +> implementation. Independent corrective work continues. The ownership HIGH remains open. + +> **Systematic group 2a, round 1 — 2026-10-04.** [Two complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T13-47-56-w7-systematic-group-2a-round-1-review.md) +> verify the cap/structural repairs but reproduce a new successful-tool observer regression: +> real session usage is lost and the hard turn cap can be bypassed. Corrections retain classified +> engagement/usage and fix the normalizer comment. Forced root checks and build pass; **fresh +> complete round 2 precedes group closure**. Ownership, remaining groups and W7 acceptance stay open. + +> **Systematic group 2a, round 2 — 2026-10-04.** [Both complete reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T14-41-03-w7-systematic-group-2a-round-2-review.md) +> verify the successful observer correction but reproduce an inherited failed-tool observer High. +> `41a3ef9a` retains real usage/engagement for invalid-argument and read-failure notifications, +> including EA7 abort, without broadening registry recovery. Eight new regressions fail before +> repair; forced root checks/build pass afterward. **Fresh complete round 3 precedes closure.** +> Ownership, other groups and whole W7 acceptance remain open. + + +### [The Phase 2.6.5 interlude in current.md's execution order] + +#### Phase 2.6.5 — Core reliability remediation (between Wave 1 and Wave 2) + +Three independent core reviews of the post-Wave-1 tree converged — independently — on the **same seven blocking +gaps**: effect journal, stdio MCP consent-before-spawn, run lease, compaction trust elevation, stream grammar, +input admission, event-log ordering. Three separate reviews landing on the same seven points is not opinion. + +The full, self-contained work list is +[phase-2.6.5-core-reliability-remediation.md]() — **51 items** (`CR-01`…`CR-98`; `CR-96` and `CR-97` were opened 2026-09-14 from the `W7` ADR reviews, `CR-98` on 2026-09-18) with evidence, fix, acceptance criteria and a decision/ADR/gate register, written so the work +can be done from that document alone. An adversarial plan review on 2026-08-10 corrected the phase boundary, +the exit rule and the execution order, and added two items (`CR-17` resume identity, `CR-63` `input_schema` +docs-only); `CR-64` came from the Batch 1 triage, `CR-21b` from ADR-0082 §10 and `CR-21c` from the `W2` +document review on 2026-08-25, which is why the total has moved since the list was first written. + +> **Historical phase checkpoint — 41 of 51 closed (updated 2026-09-18; at W6's merge it was 39 of 48 — see the register for why not 43: W5 was counted twice). `W0`–`W2` merged clean; `W3` merged 2026-08-30 (PR #86) with one reproduced BLOCKER and nine verified findings open — see the `W3` residuals in [deferred-tasks.md](../deferred-tasks.md). `W4` MERGED 2026-09-01 (PR #87) behind [ADR-0088](../../decisions/0088-the-mcp-boundary-is-hostile.md): a systematic review of the branch found five merge blockers — an orphaned MCP child on a signal, an optional validated `fetch`, a bypassable transport byte bound, discovery paging past its budget, and two cloud-metadata endpoints reachable through the local opt-in — all reproduced and fixed before it merged. `W5` MERGED 2026-09-02 (PR #88) behind +> [ADR-0089](../../decisions/0089-media-correctness-four-boundaries.md) + +> [ADR-0090](../../decisions/0090-a-continuation-token-rides-the-part-it-belongs-to.md): all six of +> `CR-50`–`CR-55` closed, two of them for one half of a two-part obligation and saying so in their own +> headings. A systematic maintainer review of the branch returned **six merge blockers** — the async +> media-job result bypassing the first-resolution pin, a run-scoped re-host ceiling that reset on resume, a +> capability gate keyed on the model id alone (breaking custom `base_url`), a negative DB media rate buying +> strict-cap headroom, a load-bearing migration backfill with no upgrade test, and a drifted canonical +> AgentRunner spec — all reproduced and fixed before it merged. +> +> **`W6` — authoring correctness — MERGED 2026-09-04 (PR #89)** — `CR-60`–`CR-64` behind +> [ADR-0091](../../decisions/0091-first-means-first-declared-not-first-to-finish.md)–[ADR-0094](../../decisions/0094-a-tool-grant-is-checked-when-the-plan-is-built.md), +> 32 commits — 27 of work over seven internal review rounds, then five folding four maintainer rounds on +> PR #89: `first` means first DECLARED and the plan stops claiming otherwise; a +> tool grant is refused when the plan is built rather than mid-run; an expression sees only its transitive +> closure and a literal out-of-closure read is refused at parse; an `output_schema` is compiled at parse in an +> allowlist-strict mode and enforced at run time — with **no new dependency**, because the compiler the +> deferral said we needed already existed one package away. +> +> **What the wave is actually worth recording for is that most of what the review rounds found were defects in +> the FIXES, not in the code they repaired.** The branch-order search was wrong three times in three rounds, +> each time in the mirror of the previous error. The expression scan's central soundness claim — that it never +> produces a false refusal, which is the entire justification for a regex over a parser — was asserted in three +> documents and false in EIGHT distinct ways — the eighth, a Unicode identifier escape, found after the +> wave was declared complete. One stand-down gate, added to prevent a false refusal, silently +> removed the whole check it was protecting. Every one is a dated correction inside its own ADR rather than a +> quiet rewrite, because the shape of the error is the reusable part. +> +> **The maintainer review then found two merge blockers, and both broke a headline claim of the wave.** Deep +> validation did not enforce `required` when the property's schema constrained nothing — `{ properties: { +> status: {} }, required: ['status'] }` accepted `{}`, because an unconstrained property compiles to +> `z.unknown()`, which is optional inside a `z.object`, and presence was checked only for names ABSENT from +> `properties`. And the expression scanner's regex was quadratic and synchronous: 514 ms at 32 K of +> whitespace, on a parser that accepts 2 MiB, with plan build synchronous so no deadline or cancel could +> interrupt it. Both reproduced and fixed before the merge, with eight further High findings — among them a +> crash on `{ type: 'string', format: 'email', maxLength: 254 }`, which is the exact shape the new canonical +> subset document tells authors to write, caused by an `as unknown as` cast that CLAUDE.md rule 1 forbids +> precisely because it turns a type error into a runtime one. +> +> **Eleven residuals** are recorded in [deferred-tasks.md](../deferred-tasks.md) — ten under the four `W6` +> residual headings plus ADR-0091's zero-branch-merge question, filed with the fan-in items. Two await a +> maintainer ruling. +> +> *(The `W4` start-gate paragraph is kept in [its earlier section](#w4-start-gate--maintainer-ruling-moved-from-currentmd-on-2026-11-04).)* +> +> - **Batch 1, merged 2026-08-11 (PR #82)** — the prerequisite (`#W15-1`), the oracle (`CR-90`, `CR-91`) and +> all of `W0` (`CR-01`–`CR-03`). `CR-64` was added in the same batch and shipped 2026-09-02 with `W6`. +> - **Batch 2, merged 2026-08-24 (PR #83)** — all of `W1`: the eight P0 blockers plus `CR-92`, behind +> ADR-0078…ADR-0084. A comprehensive review of the assembled PR found seven further defects, six of them in +> the W1 code itself; all seven were fixed and mutation-verified before merge. +> +> - **Batch 3, merged 2026-08-28 (PR #85)** — `W2`, liveness and deadlines, behind +> [ADR-0085](../../decisions/0085-the-node-executor-owes-liveness-and-the-engine-enforces-it.md). All six +> items closed; the wave's own closing register is in the phase document. The assembled PR drew a +> **Request changes** verdict with fourteen findings, five of them blockers, plus a later round of eight; +> all were verified against the code before being acted on, and the register records which were fixed and +> which were declined with the reason. The five blockers had one root cause — an ownership decision taken +> at a point in time, guarding work that spans an await. +> Its document review moved the count twice, and the two moves **cancel**: `CR-21` turns out to have +> shipped **with `CR-14`** in PR #83 (ADR-0082 decides both, and the execution graph above already +> scheduled them together) and only its heading was stale, while `CR-21c` — an unbounded individual +> `pollMediaJob` call — was found during the same review and added. `W2` was therefore still **five** open +> items out of six, not four; the phase total moved 47 → 48, not the wave. All five then closed on +> 2026-08-27 — `CR-22`, `CR-21b`, `CR-21c`, and finally `CR-20` and `CR-23` together, which +> [ADR-0085](../../decisions/0085-the-node-executor-owes-liveness-and-the-engine-enforces-it.md) decides as +> one because they are one mechanism at one place. +> +> **Not to be confused with the 2.5.5 remediation's own `W2`** ("Shut the doors", in the graph above). Same +> label, different phase: this one is 2.6.5's liveness wave. +> +> Per-item history, the seven post-review findings and the carried-forward gaps live in the phase document. + +> **Pre-`W7` close-out, 2026-09-06.** *(Superseded in part on 2026-09-14: `CR-96` and `CR-97` reopened exit criterion 1, +> the count is 41 of 51, and `CR-94` is decided and scheduled into `W7` — see the next block.)* The last two non-deferrable items closed, so **exit criterion 1 is met** +> and the phase count is **41 of 48** (40 ✅ headings plus `CR-95`'s short-term half; the arithmetic is shown +> because it has been wrong twice here). `CR-73`: `invoke_agent` was advertised to every model granted it while +> its delegate is wired nowhere in the tree, so the advertise-filter now reads a tool's declared +> `requiresDelegate` and not only its `ToolHost` arm — `read_media` is dropped for the same reason, which is +> `CR-50`'s own argument for leaving it unwired. `CR-80`: a rejected custom `base_url` was caught and the +> DEFAULT adapter left standing, which is the official API; it now installs a refusing adapter that names the +> URL's shape and never its credentials. **Criteria 5 and 7 also closed**: the three missing security sittings +> (prompt/trust provenance, hostile MCP, provider/config trust) and the `W5` and `W6` closing registers are +> written, each stating that it was written after its wave merged rather than with it. **Criterion 6** passes +> on both halves, read from the exit code — `pnpm run ci | tail` reports `tail`'s status and produced one false +> green in this session. Still open: criterion 2 (`CR-94` has no deferral record, and that is a maintainer's +> call) and criterion 4 (the `W3` live blocker keeps the gap-free stream promise wider than the code). +> *(Criteria 2 and 4 have both moved on: `CR-94` was decided 2026-09-13 and its remainder now has a deferral +> record, and ADR-0087 §1 — criterion 4's blocker — is scheduled into `W8`. Criterion 5, called closed above, +> REOPENED on 2026-09-18 with a fifth sitting, `history.db` at rest.)* + +> **`W7` decisions made and accepted, 2026-09-14.** +> +> **What was decided.** The maintainer settled `CR-70`–`CR-72` and `CR-94` on 2026-09-12 and 2026-09-13. Four ADRs +> were **accepted** on 2026-09-14, with their implementation staged in `W7`: +> +> - [ADR-0095](../../decisions/0095-what-an-agent-session-remembers-across-turns.md) decides that tool history is +> persisted as structure, never content; that carrying it into the model's context is deferred; and that `memory` +> is implemented. `memory` stays inert until `W7` lands. +> - [ADR-0096](../../decisions/0096-a-request-is-measured-before-it-is-sent.md) decides that a request is measured +> before it is sent, that a context overflow is classified and recovered only before any tool runs, and that input +> is priced. Until `W7` lands, the estimate still prices output only. +> - [ADR-0097](../../decisions/0097-a-budget-approval-is-an-allowance-not-an-exemption.md) decides that a budget +> approval is a dispatch-owned allowance, shown before it is granted. Until `W7` lands, an approved step still runs +> uncapped. +> - [ADR-0098](../../decisions/0098-a-session-effect-row-holds-no-result-and-never-replays.md) decides that a session's +> effect row holds no result and never replays. +> +> **What the reviews found.** The first six review rounds shaped the ADRs; a seventh, on 2026-09-18, reviewed them +> against the tree before any code (see the block below). The six surfaced two shipping defects, now +> register items `CR-96` and `CR-97`. Both meet the phase's non-deferrable definition, so exit criterion 1 is open +> again, and the count is **41 of 51**. +> +> **What `W7` now covers:** `CR-71`, `CR-72`, `CR-98` (ADR-0096's prerequisites, opened as an item 2026-09-18), +> `CR-94`, `CR-96` and `CR-97`, plus `relavium budget resume`, which moves here from 2.6.K because ADR-0097's +> shown-amount guarantee otherwise has no non-TTY surface. Five canonical claims that were already false are +> narrowed ahead of the code: +> +> - the session spec's tool history and export `tools` union; +> - `memory`'s default; +> - the uncapped approved budget step; +> - the SSE schema's "user/assistant/tool messages are persisted" sentence (2026-09-18); +> - the chat reference's "full transcript" in the export (2026-09-18). +> +> *(This `W7` is Phase 2.6.5's seventh wave. It is not the remediation's "Wave 7 — Orchestration & the gate" in +> the graph at the top of this file, exactly as `W2` is not Wave 2.)* + +> **`W7` document review, 2026-09-18 — before any code.** Eight dimensions reviewed the four ADRs, the register, +> the deferrals and every canonical document they touch against the tree; each finding was verified by an +> independent adversarial pass. 113 findings held, and the maintainer settled seventeen decisions the same day. +> Every ADR carries a dated note; the register carries a per-document landing checklist. Three document defects +> mattered most, and all three would have been discovered mid-implementation: +> +> - **ADR-0098's join does not exist.** It says the effect journal already records the tool-call id; every session +> surface writes a wiring constant instead, so `W7` must thread an engine-assigned id through the dispatch port. +> - **ADR-0098's monotonic turn key could not be derived** from the journal, because the sweep deletes the rows the +> high-water mark would be read from. It becomes a durable column on the session row. +> - **ADR-0096 §1's per-part floor** would have charged an inline image megabytes of tokens, contradicting the +> per-modality media ceiling in the same section. +> +> Two of the seventeen decisions change what ships beyond those ADRs: `history.db` opens with +> `PRAGMA secure_delete = ON`, so "no tool result at rest" is true of the file's bytes and not only of the row; and +> a pre-content HTTP 4xx now releases its admission instead of committing an input-sized estimate, which keeps a +> rate-limit burst from exhausting a session's cap once input is priced. ADR-0087 §1 — the `W3` live blocker that +> belonged to no wave — is scheduled into `W8`. + +**This is the corrected execution order, and it is what the graph above shows:** + +1. **`#W15-1` first.** Its five staged steps touch `run-event.ts`, `engine.ts`, `checkpoint.ts` and + `run-history-store.ts` — the same four files `CR-10` and `CR-12` restructure. Landing it after the durability + spine means writing its barrier against a persistence path that is about to change. +2. **Then Phase 2.6.5, closing before Wave 2 opens.** Which is only true if the work Wave 2 would block on + lives there — so **the hostile-MCP threat class moves into 2.6.5**: `CR-16` (consent before a stdio spawn) + plus `CR-40`–`CR-42` (transport cancellation, DNS/redirect SSRF, ingress bounds). Wave 2's queue item 2 is + **executed as 2.6.5's `W4`**; the 2.5.5 finding ids keep their home in the 2.5.5 phase doc and certify from + there. Wave 2 keeps the MCP name-collision/tool-poisoning trust items, the fs jail, the secrets layer, + persistence-security and the certification. +3. **Within 2.6.5, the oracle comes before the spine.** `CR-90`/`CR-91` (crash + durable-truth harness) land + first because they are the instrument `CR-10 → CR-11 → CR-92 → CR-12` is proven with; `CR-92` moved out of + the harness group into the spine, since an API returning `completed` while history says `failed` is a + runtime defect, not a test concern. + +Two rules that changed with the plan review: + +- **Nineteen items are non-deferrable** — every W0/W1 item plus `CR-50`, `CR-55`, `CR-73`, `CR-80`, `CR-92`, `CR-95`'s short-term fix, and (added 2026-09-14) `CR-96` and `CR-97`. Each has a cheap fail-closed option (refuse, remove, narrow the claim), so "too big + to fix now" argues for the cheap option, never for deferral. The previous exit criterion permitted deferring + all 43 items and declaring the phase complete. +- **The gate is `pnpm run ci` AND `pnpm coverage`.** `coverage` is a separate required CI check and is not + inside the `ci` script; this phase edits `packages/core` heavily, and the CI coverage job does not block on + `core`. + +`CR-01`–`CR-03` were half-closed by Wave 1 and finished right after the oracle — **all three are closed as of +2026-08-11** (PR #82). `CR-03` was a propagation gap from `#W15-10`'s own fix: the finding named three `--json` +paths that never got the safe serializer, and closing it found **five**, which is why the call-site half is now +an ESLint selector rather than a list. + + +### [Systematic group 2a, round 3 — from current.md's “Not started yet”] + +**Systematic group 2a, round 3 — 2026-10-04.** [Two complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T15-44-35-w7-systematic-group-2a-round-3-review.md) +verify all four tool notifications, then reproduce an inherited raw-callback usage/hard-cap gap and +skipped realized recording. `e7973031` preserves original error identity while carrying canonical +accounting and starts mandatory recording after authoritative cost aggregation despite delivery failure. +Original baseline: 40 failing/13 passing; corrected root: 7,625 passing/12 skipped across 331 files, +23 forced tasks and six forced builds. Every sealed report/entry is verified before Original resumes. +**Fresh complete round 4 is required; group 2a, ownership, remaining groups, Steps 7–8/12, Sonar and PR remain open.** diff --git a/docs/runbooks/local-dev-setup.md b/docs/runbooks/local-dev-setup.md index 14998fe3..ed51bbe2 100644 --- a/docs/runbooks/local-dev-setup.md +++ b/docs/runbooks/local-dev-setup.md @@ -1,6 +1,6 @@ # Local Dev Setup -> Last updated: 2026-06-03 +> Last updated: 2026-11-04 This runbook gets the Relavium monorepo building on your machine. Relavium is a **Turborepo + pnpm** TypeScript monorepo with a thin Rust layer for the Tauri desktop @@ -21,6 +21,7 @@ For the full package layout this runbook references, see | Rust toolchain (`rustup`, `cargo`) | Builds the Tauri v2 desktop backend | Only needed to build `apps/desktop`. The engine, CLI, and tests do not need Rust. | | Tauri OS prerequisites | Native WebView + build deps | macOS: Xcode Command Line Tools. Windows: WebView2 + MSVC build tools. Linux: WebKitGTK + libsecret dev packages. See the [Tauri prerequisites guide](https://v2.tauri.app/start/prerequisites/). | | Git | Workflows and agents are git-native YAML | — | +| GnuPG ≥ 2.2.8 (`gpg`) | The release custody tests (`pnpm run ci`) encrypt synthetic map containers | Required, never skipped: the tests fail without it, and with a system-wide `gpg.conf` or `common.conf` in the directory `gpgconf --list-dirs sysconfdir` names (a global configuration refuses as `GPG_CONFIG`; Homebrew ships only `scdaemon.conf` there, and Debian and Ubuntu ship no such directory). macOS: `brew install gnupg`. Debian/Ubuntu: `apt install gnupg` (the CI Ubuntu runner has it). Windows: run these tests under WSL, since they use POSIX process and permission tools; the CI runs them on Ubuntu only. Release tooling, never a runtime dependency. | > You can do meaningful work on the engine and CLI **without** the Rust/Tauri > toolchain. Install Rust only when you start on `apps/desktop`. @@ -62,28 +63,21 @@ pnpm --filter @relavium/core test Once the engine is green, bring up a surface. Build order is CLI → desktop → extension (see [architectural-principles.md](../standards/architectural-principles.md) §1). -**CLI (fastest feedback loop):** +**CLI (fastest feedback loop):** the CLI has no watch-mode script; build it, then run the bundle. ```bash -pnpm --filter relavium dev +pnpm turbo run build --filter=relavium # then, against a committed workflow file: -relavium run ./workflows/.relavium.yaml --input topic= +node apps/cli/dist/index.js run ./workflows/.relavium.yaml --input topic= ``` -**Desktop app (Tauri v2, needs the Rust toolchain):** +Rebuild after every source change: the bundle is what runs, so a stale `dist/` runs old code. To call it +as `relavium`, put an alias or a link to `apps/cli/dist/index.js` on your `PATH`. -```bash -pnpm --filter @relavium/desktop tauri dev -``` - -This starts the Vite + React 19 frontend and the Rust backend together with hot reload. - -**VS Code extension:** - -```bash -pnpm --filter @relavium/vscode-extension build -# then press F5 in VS Code on apps/vscode-extension to launch an Extension Host -``` +**Desktop app and VS Code extension:** not in the repository yet. `apps/desktop` and +`apps/vscode-extension` hold only a README; the desktop app is built in +[Phase 3](../roadmap/phases/phase-3-desktop.md) and the extension in +[Phase 4](../roadmap/phases/phase-4-vscode.md), and this section gains their commands then. ## 4. Add a provider key for local runs @@ -99,8 +93,8 @@ in the repo. | 2 | `packages/llm` | `shared` | | 3 | `packages/core` | `shared`, `llm` | | 4 | `apps/cli` | `core` | -| 5 | `apps/desktop` + `packages/ui` | `core` | -| 6 | `apps/vscode-extension` | `core`, `ui` | +| 5 | `apps/desktop` + `packages/ui` | `core` — **Phase 3; not in the repository yet** | +| 6 | `apps/vscode-extension` | `core`, `ui` — **Phase 4; not in the repository yet** | | 7 | `apps/api` + `apps/portal` | `core`, `ui` — **Phase 2 (cloud) only** | ## Troubleshooting diff --git a/docs/runbooks/release-a-surface.md b/docs/runbooks/release-a-surface.md index 0a402263..8b8da656 100644 --- a/docs/runbooks/release-a-surface.md +++ b/docs/runbooks/release-a-surface.md @@ -47,6 +47,14 @@ and the bundle boundary are [ADR-0051](../decisions/0051-cli-distribution-thin-b **This is the first release flow; desktop and VS Code inherit its shape (pack → cross-OS smoke → publish).** +> **Publication is refused — interim, 2026-10-22 (Wave 3 Step 9a.1).** The `publish` job first runs +> a fail-closed guard that refuses every artifact: only a verified release candidate (exact tarball, +> original ciphertext and release binding receipt) may publish, and none can exist until Wave 3 Step +> 9d's custody graph and Wave 4b's production recipient do, so a tag push stops after the cross-OS smoke and nothing — not +> even a v0.1.x patch — reaches npm (the [Interim release guard ruling](../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) of 2026-10-19). +> The steps below describe the pre-custody flow; Step 9d replaces them with the +> [custody graph](../reference/cli/packaging.md#archive-retry-and-publication). + ### What the build produces - `apps/cli/dist/index.js` — the single ESM bin (shebang, minified, **no** source map: a map would ship the diff --git a/docs/standards/code-review.md b/docs/standards/code-review.md index 7c27e9c3..ee1a9650 100644 --- a/docs/standards/code-review.md +++ b/docs/standards/code-review.md @@ -11,8 +11,16 @@ are actually enforced — automation catches format and types; a human catches t ## Ground rules -- Nothing merges to the default branch without a review approval and green required CI - (typecheck, lint, unit tests, fixture-mode conformance — see [testing.md](testing.md)). +- Nothing merges to the default branch without a recorded review and green required CI + (typecheck, lint, unit tests, fixture-mode conformance — see [testing.md](testing.md)). While the + project has one maintainer, GitHub cannot enforce an approval (it refuses a self-approval), so branch + protection requires the CI checks and the review is the maintainer's own plus an independent review + round recorded for the change under the roadmap's acceptance bar. A GitHub approval becomes a required + check when a second maintainer joins. + + > Amended 2026-11-04: "a review approval" read as a GitHub approval that `main`'s protection has never + > required (0 approvals, no CODEOWNERS); a PR #96 review found the mismatch and the maintainer ruled + > the rule should state the practice. - The author runs lint, format, and tests **before** requesting review; review is not a substitute for the toolchain. - Keep PRs small and scoped to one concern; a reviewable diff gets a real review. Large diff --git a/docs/standards/error-handling.md b/docs/standards/error-handling.md index abaf37f5..9c18b83f 100644 --- a/docs/standards/error-handling.md +++ b/docs/standards/error-handling.md @@ -120,6 +120,17 @@ owns the fixture-pinned classification and custom-endpoint downgrade. Actual usa with a failed native response is still priced and recorded; a reserved estimate is never reported as actual usage. Surfaces supply their own policy-aware commands. +### Diagnostic secret backstop + +`makeLlmError` passes every `message` and `code` through `scrubSecrets` +(`packages/llm/src/scrub-secrets.ts`), and the CLI's display sanitizer composes the same function. +It is a defense-in-depth **backstop**: the adapter still owns passing an already-redacted +message, and the backstop masks a finite set of secret **shapes**, not arbitrary sensitive +content. + +The finite shape list, with its documented limits, is specified in +[diagnostic-secret-backstop.md](../reference/shared-core/diagnostic-secret-backstop.md). + ## No silent catches - A `catch` block must do one of: handle the error meaningfully, enrich and rethrow it, or diff --git a/package.json b/package.json index efa79ef4..6f22957e 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "coverage": "vitest run --coverage", "coverage:enforced": "node tools/coverage-gate/run.mjs", "ci:workspaces": "turbo run lint typecheck && turbo run test --concurrency=1", - "ci": "pnpm lint:test-isolation && pnpm ci:workspaces && pnpm db:sync-check && pnpm typecheck:tools && pnpm lint:tools && turbo run build format:check && pnpm lint:fence-check && pnpm lint:engine-deps && pnpm lint:bundle-closure && pnpm smoke:cli-package && pnpm smoke:cli-bootstrap && pnpm smoke:cli && pnpm smoke:media-packaging && pnpm smoke:overflow-capture && pnpm smoke:budget-replay && pnpm test:acceptance-tools && pnpm check:wire-mutations && pnpm check:usage-mutations && pnpm check:fs-mutations && pnpm check:secrets-mutations", + "ci": "pnpm lint:test-isolation && pnpm ci:workspaces && pnpm db:sync-check && pnpm typecheck:tools && pnpm lint:tools && turbo run build format:check && pnpm lint:fence-check && pnpm lint:engine-deps && pnpm lint:bundle-closure && pnpm smoke:cli-package && pnpm check:map-controls && pnpm smoke:cli-bootstrap && pnpm smoke:cli && pnpm smoke:media-packaging && pnpm smoke:overflow-capture && pnpm smoke:budget-replay && pnpm test:acceptance-tools && pnpm check:wire-mutations && pnpm check:usage-mutations && pnpm check:fs-mutations && pnpm check:secrets-mutations", "lint:test-isolation": "node tools/test-isolation/check.mjs && node tools/test-isolation/check-ownership.mjs", "lint:fence-check": "node tools/lint-fixtures/assert-fence.mjs", "lint:engine-deps": "node tools/engine-deps/check.mjs", @@ -36,12 +36,13 @@ "smoke:budget-replay": "node tools/budget-replay-compat/check.mjs", "check:wire-mutations": "node tools/wire-conformance/check.mjs", "check:usage-mutations": "node tools/usage-coverage-controls/check.mjs", - "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/consumer-lifetime.test.mjs", + "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/ustar.test.mjs tools/cli-package/container.test.mjs tools/cli-package/gnupg.test.mjs tools/cli-package/custody.test.mjs tools/cli-package/release-recipient.test.mjs tools/cli-package/release-binding.test.mjs tools/cli-package/release-custody.test.mjs tools/cli-package/release-scope.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/map-controls.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", "check:fs-mutations": "node tools/fs-policy-controls/check.mjs", "check:secrets-mutations": "node tools/secrets-controls/check.mjs", "smoke:cli-bootstrap": "node tools/cli-bootstrap/check.mjs", "package:cli:pack": "node tools/cli-package/pack.mjs", "package:cli:consume": "node tools/cli-package/consume.mjs", + "check:map-controls": "node tools/cli-package/map-controls.mjs", "smoke:cli-package": "node tools/cli-package/check.mjs" }, "devDependencies": { diff --git a/packages/core/src/tools/bounding.test.ts b/packages/core/src/tools/bounding.test.ts index 05c28d8e..edcfd70c 100644 --- a/packages/core/src/tools/bounding.test.ts +++ b/packages/core/src/tools/bounding.test.ts @@ -1,15 +1,42 @@ +import { scrubSecrets } from '@relavium/llm/scrub-secrets'; import type { AbortSignalLike } from '@relavium/shared'; -import { describe, expect, it, vi } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { boundForModel, redactInlineMedia, redactSecretShapedText, + projectSecretShapedValue, redactSecretShapedValue, utf8ByteLength, } from './bounding.js'; +import { fixedPointOf, toFixedPoint } from './fixed-point.js'; import type { ToolHost, ToolResultLimits } from './types.js'; +// Observe the projection's bounded fixed point, retaining its actual implementation, so one case can +// make it fail and see which value the projection fails closed with. +vi.mock('./fixed-point.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + fixedPointOf: vi.fn(actual.fixedPointOf), + toFixedPoint: vi.fn(actual.toFixedPoint), + }; +}); + +/** + * Eight-MiB cases take well under a second. Their bound is this case deadline, far below the minutes a + * quadratic scan would need, rather than a wall-clock assertion that a loaded runner can trip. It is + * also the largest case budget the secrets causal-control phases admit (`CAUSAL_TIMEOUT_BUDGET`). + */ +const ADMITTED_SIZE_CASE_TIMEOUT_MS = 15_000; + +// The observation must not keep every admitted-size input and projection alive for the whole file. +afterEach(() => { + vi.mocked(fixedPointOf).mockClear(); + vi.mocked(toFixedPoint).mockClear(); +}); + const BIG: ToolResultLimits = { maxBytes: 50_000, maxLines: 2000 }; const TINY: ToolResultLimits = { maxBytes: 20, maxLines: 3 }; @@ -139,6 +166,20 @@ describe('boundForModel', () => { const bounded = await boundForModel(undefined, TINY, host()); expect(bounded.truncated).toBe(false); expect(bounded.value).toBeUndefined(); + expect(bounded.summary).toBe(''); + }); + + it('summarizes a result that has no JSON form as unserializable, without rejecting', async () => { + const throwing = Object.defineProperty({}, 'body', { + enumerable: true, + get() { + throw new Error('getter fails'); + }, + }); + for (const result of [{ n: 10n }, throwing]) { + const bounded = await boundForModel(result, BIG, host()); + expect(bounded.summary).toBe('[unserializable]'); + } }); it('renders a circular result without throwing (the cycle-safe redaction walk breaks the cycle)', async () => { @@ -247,6 +288,40 @@ describe('boundForModel', () => { expect(bounded.summary).toContain('omitted'); }); + it('counts a 30 MiB data URI the texts omit against the model-facing ceiling', async () => { + // A PR #96 review: the ceiling was measured on the 25-character marker, so the whole URI went to the + // model and was settled into the effect journal. The spill and the preview stay byte-free (I3). + const uri = `data:image/png;base64,${'A'.repeat(30 * 1024 * 1024)}`; + let spilledText = 'unset'; + const spill = vi.fn((text: string) => { + spilledText = text; + return Promise.resolve({ ref: 'spill://uri', byteLength: text.length }); + }); + const bounded = await boundForModel(uri, BIG, host({ outputStore: { spill } })); + expect(bounded.truncated).toBe(true); + expect(String(bounded.value)).not.toContain('AAAAAAAA'); + expect(String(bounded.value)).toBe( + `[base64 data URI omitted]\n\n[… truncated: ${utf8ByteLength(uri) + 25} bytes / 1 lines; full output at spill://uri …]`, + ); + expect(spilledText).toBe('[base64 data URI omitted]'); + expect(bounded.summary).not.toContain('AAAAAAAA'); + }); + + it('counts base64 sources and binary buffers inside a result, and keeps one that fits', async () => { + const limits: ToolResultLimits = { maxBytes: 1024, maxLines: 100 }; + const fits = { source: { kind: 'base64', data: 'Q'.repeat(512) } }; + expect((await boundForModel(fits, limits, host())).value).toBe(fits); + for (const over of [ + { source: { kind: 'base64', data: 'Q'.repeat(2048) } }, + { thumb: new Uint8Array(2048) }, + [{ uri: `data:audio/mpeg;base64,${'Q'.repeat(2048)}` }], + ]) { + const bounded = await boundForModel(over, limits, host()); + expect(bounded.truncated, JSON.stringify(Object.keys(over))).toBe(true); + expect(String(bounded.value)).not.toContain('QQQQ'); + } + }); + it('redacts base64 from EVERY element of an array-of-media result (the redaction walk recurses arrays)', async () => { const a = 'aGVsbG8gZmlyc3QgbWVkaWEgcGFydCBwYXlsb2Fk'; const b = 'd29ybGQgc2Vjb25kIG1lZGlhIHBhcnQgcGF5bG9hZA=='; @@ -629,9 +704,13 @@ describe('redactSecretShapedText — URL userinfo (ADR-0080 §11)', () => { for (const text of [ 'https://vanity@api.invalid/x', 'https://api.invalid?public=alice@example.invalid', - 'https://api.invalid#alice@example.invalid', + 'https://api.invalid#contact=alice@example.invalid', ]) expect(redactSecretShapedText(text)).toBe(text); + // A host fragment with no assignment may be a malformed password: masked, failing closed. + expect(redactSecretShapedText('https://api.invalid#alice@example.invalid')).toBe( + 'https://[redacted]@example.invalid', + ); expect(redactSecretShapedText(`https://${'u'.repeat(600)}@api.invalid/x`)).toBe( 'https://[redacted]@api.invalid/x', ); @@ -806,6 +885,66 @@ describe('redactSecretShapedValue — the KEY decides, whatever the value looks }); }); + it.each([ + '--auth-token', + '--access-token', + '--pwd', + '--db-password', + '--x-api-key', + '--apiKey', + '--credentials', + '--refresh_token', + ])( + 'STRUCTURAL_ARGV_WORDS masks the value after %s, whose last word names a credential', + (flag) => { + // A PR #96 review: the object-key rule masked these names as keys while the anchored flag list passed + // them verbatim into `agent:tool_call` and the journal projection. + expect(redactSecretShapedValue({ args: [flag, 'Qz7flagValue', '--count=4'] })).toEqual({ + args: [flag, '[redacted]', '--count=4'], + }); + expect(redactSecretShapedValue({ args: [`${flag}=Qz7flagValue`] })).toEqual({ + args: [`${flag}=[redacted]`], + }); + expect(projectSecretShapedValue({ args: [flag, 'Qz7flagValue'] })).toEqual({ + value: { args: [flag, '[redacted]'] }, + settled: true, + }); + }, + ); + + it('STRUCTURAL_ARGV_WORDS leaves a flag that only mentions a credential, and its next argument', () => { + // Masking these would give calls that differ only in the next argument one journal identity. + const benign = { + args: [ + ...['--max-tokens', '4096', '--password-stdin', 'next', '--token-file', '/tmp/t'], + ...['--no-password', '-h', 'db', '--key', 'k3', '--author', 'me', '--token-type', 'bearer'], + ], + }; + expect(redactSecretShapedValue(benign)).toEqual(benign); + }); + + it('STRUCTURAL_ARGV_USERINFO masks a curl user:password in every spelling, and only that', () => { + expect( + redactSecretShapedValue({ + args: ['-u', 'alice:Qz7pw', '-ualice:Qz7pw', '--user=bob:Qz7pw', '--user', 'carol:Qz7pw'], + }), + ).toEqual({ + args: ['-u', '[redacted]', '-u[redacted]', '--user=[redacted]', '--user', '[redacted]'], + }); + const benign = { + args: [ + ...['-u', 'alice', '--user', 'carol', '-u', '1000:1000', '--user=dave', '-uroot'], + ...['--user', 'git+https://example.invalid/r.git'], + ], + }; + expect(redactSecretShapedValue(benign)).toEqual(benign); + // A value that is not a user:password is still read by every other rule. + const key = `sk-${'Q'.repeat(20)}`; + expect( + JSON.stringify(redactSecretShapedValue({ args: ['--user', key, '-u', key] })), + ).not.toContain(key); + }); + it('leaves ordinary structured args alone — the negative control', () => { // Without this the rule above is satisfied by redacting everything, which would strip the stored row of // the diagnostic value that is its whole remaining purpose. @@ -1001,15 +1140,17 @@ describe('PR95 B2 projection boundaries and admitted-scale controls', () => { }, ); - it('bounds admitted-size failed finite header preparation', () => { - const fragment = 'wordx-\tapi-\nkey '; - const input = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); - const started = performance.now(); - const output = redactSecretShapedText(input); - expect(output.length, 'SECRETS_B2_HEADER_PREPARATION').toBe(input.length); - expect(output === input, 'SECRETS_B2_HEADER_PREPARATION').toBe(true); - expect(performance.now() - started, 'SECRETS_B2_HEADER_PREPARATION').toBeLessThan(5_000); - }); + it( + 'bounds admitted-size failed finite header preparation', + () => { + const fragment = 'wordx-\tapi-\nkey '; + const input = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); + const output = redactSecretShapedText(input); + expect(output.length, 'SECRETS_B2_HEADER_PREPARATION').toBe(input.length); + expect(output === input, 'SECRETS_B2_HEADER_PREPARATION').toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); it.each([ ['sk-proj-', '\n'], @@ -1074,41 +1215,47 @@ describe('PR95 B2 projection boundaries and admitted-scale controls', () => { } }); - it('bounds admitted-size repeated invalid JWT starts without losing later valid spans', () => { - const fragment = `eyJ${'A'.repeat(61)}\n`; - const publicPrefix = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); - const started = performance.now(); - // No dot, or a huge second run with no second dot: both full JWT attempts would scan to EOF. - for (const suffix of ['', `.${'B'.repeat(128)}`]) { - const input = `${publicPrefix}${suffix}; sk-proj-${'C'.repeat(32)}\t${'D'.repeat(32)}; public=A`; - const expected = `${publicPrefix}${suffix}; [redacted]; public=A`; + it( + 'bounds admitted-size repeated invalid JWT starts without losing later valid spans', + () => { + const fragment = `eyJ${'A'.repeat(61)}\n`; + const publicPrefix = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); + // No dot, or a huge second run with no second dot: both full JWT attempts would scan to EOF. + for (const suffix of ['', `.${'B'.repeat(128)}`]) { + const input = `${publicPrefix}${suffix}; sk-proj-${'C'.repeat(32)}\t${'D'.repeat(32)}; public=A`; + const expected = `${publicPrefix}${suffix}; [redacted]; public=A`; + const output = redactSecretShapedText(input); + expect(output.length, 'SECRETS_B2_BOUNDED_PROBE').toBe(expected.length); + expect(output === expected, 'SECRETS_B2_BOUNDED_PROBE').toBe(true); + } + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it( + 'bounds admitted-size raw hyphen-separated invalid JWT starts and masks a later whole key', + () => { + const fragment = 'eyJAAAAAAAAAA-'; + const publicPrefix = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); + const input = `${publicPrefix}sk-proj-${'C'.repeat(32)}-${'D'.repeat(32)}; public=A`; + const expected = `${publicPrefix}[redacted]; public=A`; const output = redactSecretShapedText(input); - expect(output.length, 'SECRETS_B2_BOUNDED_PROBE').toBe(expected.length); - expect(output === expected, 'SECRETS_B2_BOUNDED_PROBE').toBe(true); - } - expect(performance.now() - started, 'SECRETS_B2_BOUNDED_PROBE').toBeLessThan(5_000); - }); - - it('bounds admitted-size raw hyphen-separated invalid JWT starts and masks a later whole key', () => { - const fragment = 'eyJAAAAAAAAAA-'; - const publicPrefix = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); - const input = `${publicPrefix}sk-proj-${'C'.repeat(32)}-${'D'.repeat(32)}; public=A`; - const expected = `${publicPrefix}[redacted]; public=A`; - const started = performance.now(); - const output = redactSecretShapedText(input); - expect(output.length, 'SECRETS_B2_RAW_BOUNDED_PROBE').toBe(expected.length); - expect(output === expected, 'SECRETS_B2_RAW_BOUNDED_PROBE').toBe(true); - expect(performance.now() - started, 'SECRETS_B2_RAW_BOUNDED_PROBE').toBeLessThan(5_000); - }); - - it('preserves dense admitted-size public boundaries with bounded preparation', () => { - const started = performance.now(); - for (const fragment of ['a\n', 'a ']) { - const input = `${fragment.repeat(4 * 1024 * 1024 - 1)}\n`; - expect(redactSecretShapedText(input) === input, 'SECRETS_B2_BOUNDED_PROBE').toBe(true); - } - expect(performance.now() - started, 'SECRETS_B2_BOUNDED_PROBE').toBeLessThan(5_000); - }); + expect(output.length, 'SECRETS_B2_RAW_BOUNDED_PROBE').toBe(expected.length); + expect(output === expected, 'SECRETS_B2_RAW_BOUNDED_PROBE').toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it( + 'preserves dense admitted-size public boundaries with bounded preparation', + () => { + for (const fragment of ['a\n', 'a ']) { + const input = `${fragment.repeat(4 * 1024 * 1024 - 1)}\n`; + expect(redactSecretShapedText(input) === input, 'SECRETS_B2_BOUNDED_PROBE').toBe(true); + } + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); it('retains prefix evidence inside a header and preserves the same public tail after multiple masks', () => { const credential = `sk-proj-${'A'.repeat(24)}\n${'B'.repeat(24)}`; @@ -1154,3 +1301,1898 @@ describe('PR95 B2 projection boundaries and admitted-scale controls', () => { expectCredentialProjection(redactSecretShapedText(publicText)).toBe(publicText); }); }); + +describe('PR95 raw ?/# userinfo in the journal projection', () => { + const userinfoContact = (value: unknown) => expect(value, 'SECRETS_B2_CORE_USERINFO_CONTACT'); + + it.each([ + ['https://us?er:TOKEN@h.example/', 'https://[redacted]@h.example/'], + ['https://us?er=1:TOKEN@h.example/', 'https://[redacted]@h.example/'], + ])('masks userinfo whose raw ?/# follows no host-shaped prefix: %s', (input, expected) => { + userinfoContact(redactSecretShapedText(input)).toBe(expected); + }); + + it.each([ + ['https://my.user:1234?TOKEN@h.example/', 'https://[redacted]@h.example/'], + ['https://provider.example#section@x.example', 'https://[redacted]@x.example'], + ])('masks a host-shaped prefix that no assignment follows: %s', (input, expected) => { + userinfoContact(redactSecretShapedText(input)).toBe(expected); + }); + + it.each(['?', '#'])('retains public %s contact text after a host-shaped prefix', (separator) => { + for (const host of [ + 'provider.invalid', + 'provider.invalid:8443', + 'provider.invalid.', + '10.0.0.1', + 'localhost:3000', + ]) { + const input = `connect https://${host}${separator}contact=alice@example.invalid failed`; + expect(redactSecretShapedText(input)).toBe(input); + } + }); + + it('keeps its documented differences from the LLM backstop', () => { + // Core retains a colonless userinfo under 16 characters; the LLM diagnostic backstop masks it. + expect(redactSecretShapedText('http://#TOKEN@h.example')).toBe('http://#TOKEN@h.example'); + expect(scrubSecrets('http://#TOKEN@h.example')).toBe('http://[REDACTED]@h.example'); + // Core's authority also ends at `"`, `'`, `}` and `]`, the delimiters of the JSON and quoted texts it + // reads, so it leaves userinfo containing one of them as it is: an already-masked password, a + // bracketed-host prefix, a raw delimiter. + for (const [input, masked] of [ + ['https://app:[REDACTED]@h.example/', 'https://[REDACTED]@h.example/'], + ['https://[::1]:8443?u:TOKENTOKEN@h.example/', 'https://[REDACTED]@h.example/'], + ['https://user:pa"ssword@host.example/', 'https://[REDACTED]@host.example/'], + ["https://user:pa'ssword@host.example/", 'https://[REDACTED]@host.example/'], + ['https://user:pa}ssword@host.example/', 'https://[REDACTED]@host.example/'], + ] as const) { + expect(redactSecretShapedText(input), input).toBe(input); + expect(scrubSecrets(input), input).toBe(masked); + } + }); + + it('reads a `,` or `/` in a password through its `@`, never past a port, query or other delimiter', () => { + // A PR #96 review found the authority scan leaving `ssword` and `ss` public here. + for (const [input, expected] of [ + ['https://user:pa,ssword@host.example/', 'https://[redacted]@host.example/'], + ['https://user:pa/ss@host.example/x', 'https://[redacted]@host.example/x'], + ['{"u":"https://user:p,w/x@h.example/v1"}', '{"u":"https://[redacted]@h.example/v1"}'], + ['https://user:pa/ss@host.example/@scope/pkg', 'https://[redacted]@host.example/@scope/pkg'], + // A raw `?` that is not a host's public query stays userinfo, as the authority scan reads it. + ['https://u:p?x/y@h.example/', 'https://[redacted]@h.example/'], + // A key in the username names the user, so its value stops at the `@` and the host stays. + ['https://x-access-token:pa,ss@github.com/o/r', 'https://[redacted]@github.com/o/r'], + // With no `@` to read on to, the scan resumes inside the authority and finds the later scheme. + ['https://xhttps://user:pw@h.example/', 'https://xhttps://[redacted]@h.example/'], + ] as const) { + expect(redactSecretShapedText(input), input).toBe(expected); + } + // The backstop already reads a comma; both now mask it. + expect(scrubSecrets('https://user:pa,ssword@host.example/')).toBe( + 'https://[REDACTED]@host.example/', + ); + for (const input of [ + 'https://host.example:8080/a@b.example', + 'https://host.example:8080,bob@x.example', + 'https://host.example:8080?next=/a@b.example', + 'https://example.com/@user', + 'https://user:pa,ss/x', + '{"u":"https://h.example:p","e":"x@y.example"}', + "'https://u:pw/x', 'a@b.example'", + 'https://a:b/https://c@d.example', + `https://user:pa/${'s'.repeat(600)}@host.example/`, + ]) { + expect(redactSecretShapedText(input), input).toBe(input); + } + }); + + it('agrees with the LLM backstop on every raw ?/# decision outside those differences', () => { + for (const input of [ + 'https://alice:pa?ss@provider.invalid/v1', + 'https://alice:pa#ss@provider.invalid/v1', + 'https://alice:pa?ss=value@provider.invalid/v1', + 'https://us?er:TOKEN@h.example/', + 'https://us?er=1:TOKEN@h.example/', + 'https://my.user:1234?TOKEN@h.example/', + 'https://provider.example#section@x.example', + 'https://provider.invalid?contact=alice@example.invalid', + 'https://provider.invalid:8443#contact=alice@example.invalid', + 'https://localhost:3000?contact=alice@example.invalid', + 'https://10.0.0.1#contact=alice@example.invalid', + 'https://john.doe:s3cret?x=1@h.example/', + 'https://localhostuser:pw?a=b@h.example/', + 'https://user:pw.evil.com?x=1@h.example/', + 'https://h.example:123456?x=1@h.example/', + 'https://h.example:65535?contact=alice@example.invalid', + ]) { + expect(redactSecretShapedText(input) === input, input).toBe(scrubSecrets(input) === input); + } + }); + + it.each([ + 'https://john.doe:s3cret?x=1@h.example/', + 'https://localhostuser:pw?a=b@h.example/', + 'https://user:pw.evil.com?x=1@h.example/', + 'https://h.example:123456?x=1@h.example/', + ])('masks userinfo whose prefix only resembles a host: %s', (input) => { + expect(redactSecretShapedText(input), 'SECRETS_B2_CORE_USERINFO_CONTACT').toBe( + 'https://[redacted]@h.example/', + ); + }); + + it('bounds the host-shaped prefix at exactly 260 characters, case-insensitively', () => { + const host = (length: number) => `${'a'.repeat(length - 2)}.b`; + const retained = `connect https://${host(260)}?contact=alice@example.invalid failed`; + expect(redactSecretShapedText(retained), 'SECRETS_B2_CORE_USERINFO_HOST_LENGTH').toBe(retained); + expect( + redactSecretShapedText(`https://${host(261)}?contact=alice@example.invalid`), + 'SECRETS_B2_CORE_USERINFO_HOST_LENGTH', + ).toBe('https://[redacted]@example.invalid'); + const upper = 'connect https://PROVIDER.INVALID:8443?contact=alice@example.invalid failed'; + expect(redactSecretShapedText(upper)).toBe(upper); + }); + + it( + 'masks an admitted-size dotted userinfo prefix without exhausting the RegExp stack', + () => { + const labels = 'a.'.repeat(Math.floor((8 * 1024 * 1024 - 256 - 64) / 2)); + const input = `upstream said https://${labels}a?x=1@h.example failed`; + let output = ''; + expect(() => { + output = redactSecretShapedText(input); + }, 'SECRETS_B2_CORE_USERINFO_HOST_LENGTH').not.toThrow(); + expect( + output === 'upstream said https://[redacted]@h.example failed', + 'SECRETS_B2_CORE_USERINFO_HOST_LENGTH', + ).toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); +}); + +describe('PR95 review round 4: ambiguous authorities, URL schemes after labels and split edges', () => { + it('masks a secret query value that holds a raw @ after a single-label host', () => { + for (const [input, expected, tail] of [ + ['mysql://db?user=root&password=Adm1n@2024!secure', 'mysql://[redacted]', '2024!secure'], + ['https://svc?access_token=ya29@abcdefXYZ', 'https://[redacted]', 'abcdefXYZ'], + ] as const) { + const output = redactSecretShapedText(input); + expect(output, 'SECRETS_B4_CORE_AUTHORITY_REST').toBe(expected); + expect(output.includes(tail), 'SECRETS_B4_CORE_AUTHORITY_REST').toBe(false); + } + // A raw ?/# in a password does not hide the host it connects to. + for (const [input, expected] of [ + ['https://alice:pa?ss@provider.invalid/v1', 'https://[redacted]@provider.invalid/v1'], + ['https://u:p?w@[::1]:8443/x', 'https://[redacted]@[::1]:8443/x'], + ['https://u:p?w@localhost:3000/x', 'https://[redacted]@localhost:3000/x'], + ] as const) { + expect(redactSecretShapedText(input), 'SECRETS_B4_CORE_AUTHORITY_REST').toBe(expected); + } + }); + + it( + 'masks an admitted-size query value inside an ambiguous userinfo', + () => { + const labels = 'a.'.repeat(Math.floor((8 * 1024 * 1024 - 256 - 64) / 2)); + const input = `upstream said https://user:pw?x=1@${labels}a failed`; + let output = ''; + expect(() => { + output = redactSecretShapedText(input); + }, 'SECRETS_B5_CORE_QUERY_EXTENT').not.toThrow(); + expect( + output === 'upstream said https://[redacted] failed', + 'SECRETS_B5_CORE_QUERY_EXTENT', + ).toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it('masks the userinfo of a URL behind a scheme label, split or not, and a token glued to a URL', () => { + for (const [input, expected] of [ + [ + 'invalid token postgres://app:hunter2secret@db\n', + 'invalid token [redacted]://[redacted]@db\n', + ], + ['invalid token postgres://app:pa#ss@db\n', 'invalid token [redacted]://[redacted]@db\n'], + ['token postgresql://app:hunter2secret@db', 'token [redacted]://[redacted]@db'], + ['token abcdefghijkl\n', 'token [redacted]\n'], + ['Bearer abcdefgh12345678/https://x.example', 'Bearer [redacted]://x.example'], + ['Bearer abcDEF123456xyzhttps://h.example', 'Bearer [redacted]://h.example'], + ['token\tabcdefgh12345678/https://x', 'token [redacted]://x'], + [ + 'Authorization: Bearer abcDEF123456xyzhttps://u:p@h.example', + '[redacted] [redacted]://[redacted]@h.example', + ], + ] as const) { + expect(redactSecretShapedText(input), 'SECRETS_B5_CORE_SCHEME_ORDER').toBe(expected); + } + }); + + it.each(['\n', '\t'])('masks URL userinfo split at either of its edges (%j)', (separator) => { + for (const input of [ + `https://user:hunter2secret${separator}@host.example/x`, + `https://${separator}user:hunter2secret@host.example/x`, + ]) { + expect(redactSecretShapedText(input), 'SECRETS_B4_CORE_SPLIT_EDGE').toBe( + 'https://[redacted]@host.example/x', + ); + } + }); + + it('reaches a fixed point', () => { + // The opaque value removes the authority's last `@`, so the second pass reads a new userinfo. + for (const [input, expected] of [ + ['g://s.hos#mHPGEAXXQ8@?q==a@b.exam/user:3eYPUzfxo', 'g://[redacted]@?q=[redacted]'], + ['[redacted]', '[redacted]'], + ] as const) { + const output = redactSecretShapedText(input); + expect(output, 'SECRETS_B4_CORE_FIXED_POINT').toBe(expected); + expect(redactSecretShapedText(output), 'SECRETS_B4_CORE_FIXED_POINT').toBe(output); + } + }); +}); + +describe('PR95 review round 5: an ambiguous userinfo masked with exactly the values later rules read', () => { + const pem = [ + 'https://a?b:c@-----BEGIN RSA PRIVATE KEY-----', + 'MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu', + '-----END RSA PRIVATE KEY-----', + ].join('\n'); + const projects = (marker: string, pairs: readonly (readonly [string, string, string])[]) => { + for (const [input, expected, secret] of pairs) { + const output = redactSecretShapedText(input); + expect(output, marker).toBe(expected); + expect(output.includes(secret), marker).toBe(false); + expect(redactSecretShapedText(output), marker).toBe(output); + } + }; + + it('masks an ambiguous userinfo with the query values its later rules would mask whole', () => { + projects('SECRETS_B5_CORE_QUERY_EXTENT', [ + ['mysql://db?user=root&password=Adm1n@2024.prod', 'mysql://[redacted]', '2024.prod'], + ['https://gateway?password=P@ss.word1', 'https://[redacted]', 'ss.word1'], + ['https://svc?client_secret=abcdefgh@def.example', 'https://[redacted]', 'def.example'], + ['https://svc?x.client_secret=abcdefgh@def.example', 'https://[redacted]', 'def.example'], + [ + 'postgres://db?sslmode=require&password=Adm1n@x/Secret2024', + 'postgres://[redacted]', + 'Secret2024', + ], + [ + 'see https://svc?token=abcdefghij@foo.example/x for details', + 'see https://[redacted] for details', + 'foo.example', + ], + ['https://user:pw?token=abcdefgh@host', 'https://[redacted]', 'host'], + [`https://svc?sig=${'A'.repeat(24)}@tail.example`, 'https://[redacted]', 'tail.example'], + [ + 'https://proxy?redirect_target=a@http://user:secretpw@host.example/', + 'https://[redacted]', + 'secretpw', + ], + ['https://svc:pw?x=1@ftp://user:pw2secret@h.example/', 'https://[redacted]', 'pw2secret'], + ]); + // A value no later rule masks leaves the userinfo mask where it was. + for (const [input, expected] of [ + ['https://us?er=1:TOKEN@h.example/', 'https://[redacted]@h.example/'], + ['https://alice:pa?ss@provider.invalid/v1', 'https://[redacted]@provider.invalid/v1'], + ] as const) { + expect(redactSecretShapedText(input), 'SECRETS_B5_CORE_QUERY_EXTENT').toBe(expected); + } + }); + + it('keeps the context of a later credential after an ambiguous authority', () => { + projects('SECRETS_B5_CORE_LATER_CONTEXT', [ + [ + 'url: postgres://app:pa#ss@db.internal\npassword: hunter2hunter2\n', + 'url: postgres://[redacted]@db.internal\n[redacted]\n', + 'hunter2hunter2', + ], + [ + 'url: postgres://app:pa#ss@db.internal\n\ttoken: hunter2hunter2\n', + 'url: postgres://[redacted]@db.internal\n\t[redacted]\n', + 'hunter2hunter2', + ], + [ + 'https://user:pa?ss@minio?sig=Zm9vYmFy/YmF6cXV4cXV1eHh4eHh4eHh4eHh4', + 'https://[redacted]@minio?sig=[redacted]', + 'YmF6cXV4', + ], + [ + 'https://a?b:c@Bearer abcdefgh12345678', + 'https://[redacted]@Bearer [redacted]', + 'abcdefgh12345678', + ], + [pem, 'https://[redacted]@[redacted]', 'MIIBOgIBAAJBAKj34'], + [ + 'DATABASE_URL=mysql://db?user=root&password=Adm1n@2024\nREPLICA_URL=postgres://app:hunter2secret@replica/x', + 'DATABASE_URL=mysql://[redacted]\nREPLICA_URL=postgres://[redacted]@replica/x', + 'hunter2secret', + ], + [ + 'https://gateway?password=P@ssw0rd123\nBearer hunter2secretvalue', + 'https://[redacted]\nBearer [redacted]', + 'hunter2secretvalue', + ], + [ + 'https://gateway?password=P@ssw0rd123\tx-api-key: hunter2secretvalue', + 'https://[redacted]\t[redacted]', + 'hunter2secretvalue', + ], + [ + 'mysql://db?user=root&password=Adm1n@2024;Bearer hunter2secretvalue', + 'mysql://[redacted];Bearer [redacted]', + 'hunter2secretvalue', + ], + ]); + }); + + it.each(['\n', '\t'])( + 'masks URL userinfo whose scheme is split by an LF or TAB (%j)', + (separator) => { + for (const [input, expected] of [ + [ + `https:/${separator}/user:hunter2secret@host.example/x`, + `https:/${separator}/[redacted]@host.example/x`, + ], + [ + `https${separator}://user:hunter2secret@host.example`, + `https${separator}://[redacted]@host.example`, + ], + [ + `ssh:${separator}//deploy:hunter2secret@h.example`, + `ssh:${separator}//[redacted]@h.example`, + ], + ] as const) { + expect(redactSecretShapedText(input), 'SECRETS_B5_CORE_SPLIT_SCHEME').toBe(expected); + } + }, + ); + + it( + 'masks a token glued to a URL at admitted size', + () => { + const input = `Bearer ${'a'.repeat(8 * 1024 * 1024 - 64)}://h.example`; + let output = ''; + expect(() => { + output = redactSecretShapedText(input); + }, 'SECRETS_B5_CORE_SCHEME_ORDER').not.toThrow(); + expect(output === 'Bearer [redacted]://h.example', 'SECRETS_B5_CORE_SCHEME_ORDER').toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); +}); + +describe('PR95 review round 6: one merged pass of every core rule', () => { + const pem = [ + '-----BEGIN RSA PRIVATE KEY-----', + 'MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu', + '-----END RSA PRIVATE KEY-----', + ].join('\n'); + const projects = (marker: string, pairs: readonly (readonly [string, string, string])[]) => { + for (const [input, expected, secret] of pairs) { + const output = redactSecretShapedText(input); + expect(output, marker).toBe(expected); + expect(output.includes(secret), marker).toBe(false); + expect(redactSecretShapedText(output), marker).toBe(output); + } + }; + + it('keeps every credential and its context when core masks overlap', () => { + projects('SECRETS_B6_CORE_MERGE', [ + [ + 'https://a?b:c&password=xy@Bearer abcdefgh12345678', + 'https://[redacted] [redacted]', + 'abcdefgh12345678', + ], + [ + 'https://svc?password=abcd@c/Bearer abcdefgh12345678', + 'https://[redacted] [redacted]', + 'abcdefgh12345678', + ], + [`https://a?b:c&password=xy@${pem}`, 'https://[redacted]', 'MIIBOgIBAAJBAKj34'], + [ + `https://u:pw?x=${'a'.repeat(24)}@h;Bearer abcdefgh12345678`, + 'https://[redacted] [redacted]', + 'abcdefgh12345678', + ], + [ + 'postgres://app:pw?application_name=reporting-service-01@db;password="S3cr3tValue99"', + 'postgres://[redacted]', + 'S3cr3tValue99', + ], + [ + 'mysql://db?user=root&password=Adm1n@2024.prod/Bearer hunter2secretvalue', + 'mysql://[redacted] [redacted]', + 'hunter2secretvalue', + ], + [`redis://db#client_secret=G5oK47GKkhEp@2024.prod/${pem}`, 'redis://[redacted]', 'MIIBOg'], + ]); + }); + + it.each(['\n', '\t', 'word\t', 'DATABASE_URL:\n'])( + 'masks an ambiguous userinfo with its query values behind %j', + (prefix) => { + projects('SECRETS_B6_CORE_MERGE', [ + [`${prefix}https://gateway?password=P@ssw0rd123`, `${prefix}https://[redacted]`, 'ssw0rd'], + [ + `${prefix}mysql://db?user=root&password=Adm1n@2024.prod`, + `${prefix}mysql://[redacted]`, + '2024.prod', + ], + [ + `${prefix}see https://svc?token=abcdefghij@foo.example/x for details`, + `${prefix}see https://[redacted] for details`, + 'foo.example', + ], + [ + `${prefix}https://svc?sig=${'A'.repeat(24)}@tail.example`, + `${prefix}https://[redacted]`, + 'tail.example', + ], + ]); + }, + ); + + it('masks an ambiguous userinfo whose scheme is split by an LF', () => { + projects('SECRETS_B6_CORE_MERGE', [ + ['https:/\n/svc?password=Secret@tail99', 'https:/\n/[redacted]', 'tail99'], + ['https://\nsvc?password=Secret@tail99', 'https://[redacted]', 'tail99'], + ]); + }); + + it("reads a key inside another key's value", () => { + projects('SECRETS_B6_CORE_NESTED_KEY', [ + ['token=abcdef>password: hunter2hunter2', '[redacted]', 'hunter2hunter2'], + [ + 'https://user:pw?token=K9wR2v@db/x>password: hunter2hunter2', + 'https://[redacted]', + 'hunter2hunter2', + ], + [ + 'postgres://u:p?password=K9wR2v@db/path[api_key: SECRETVALUE99', + 'postgres://[redacted]', + 'SECRETVALUE99', + ], + [ + '{"msg":"http://app:pw?a=1&x=OGePIzn69aW2APSxEBy7@def.example\\nx-api-key: R2FIHKWkS0J2kZ0L35"}', + '{"msg":"http://[redacted]"}', + 'R2FIHKWkS0J2kZ0L35', + ], + ['x-api-key: "abc password="hunter2 secret value"', '[redacted]', 'secret value'], + ]); + }); + + it("reads a scheme label inside another label's token", () => { + projects('SECRETS_B6_CORE_NESTED_LABEL', [ + ['Bearer abcdefgh/Bearer hunter2secretvalue', 'Bearer [redacted] [redacted]', 'hunter2'], + ['token abcdefgh12/Basic hunter2secretvalue==', 'token [redacted] [redacted]', 'hunter2'], + ]); + }); + + it('reads a value through the masks before it, line breaks included', () => { + projects('SECRETS_B6_CORE_VALUE_THROUGH_MASK', [ + [ + '{"a":"x-api-key": "QL9uUu2jzxnM\txoxb-QWZDhKp63WKU\naccess_token="Qi3YmmwU37GH Qqu6LcyxbzW8"', + '{"a":"[redacted]', + 'QL9uUu2jzxnM', + ], + [ + `x-api-key: "v@${pem}client_secret="Q9w8EwQDxFP2 QmeNGkQBxtfm"`, + '[redacted]', + 'Q9w8EwQDxFP2', + ], + [ + 'token: "abc sk-proj-abcdefghij\nklmnopqrstuv" password: "secret value 1"', + '[redacted] [redacted]', + 'secret value', + ], + ]); + }); + + it('joins a split header label right after a credential or a PEM block', () => { + projects('SECRETS_B6_CORE_MASK_BOUNDARY', [ + [ + 'url: AKIA\nQRTBLS2XNANSABCDx-api-\tkey": "QzJsBjkrL2GS?access_token="QeKrgCE47mCW QW2Y8jaZkAPA"', + 'url: [redacted]', + 'QzJsBjkrL2GS', + ], + [`${pem}authori\nzation: QFzQnbCdq2TL`, '[redacted]', 'QFzQnbCdq2TL'], + ['AKIAQNNKD8CEOFHOABCDauthori\nzation:QNauurwS3vLp', '[redacted]', 'QNauurwS3vLp'], + ]); + }); + + it('reads a scheme label or a credential glued to the end of another credential', () => { + projects('SECRETS_B6_CORE_MASK_BOUNDARY', [ + [ + 'x\n\tclient_secret=QRYyV2UYKhnz!AKIAQPJ53XE28CC4A\nBCDtoken QNZd4i2J6YY5', + 'x\n\t[redacted] [redacted]', + 'QNZd4i2J6YY5', + ], + ['AKIAQPJ53XE28CC4ABCDtoken QNZd4i2J6YY5', '[redacted]token [redacted]', 'QNZd4i2J6YY5'], + [ + 'client_secret=QRYyV2UYKhnz!AKIAQPJ53XE28CC4ABCDtoken QNZd4i2J6YY5', + '[redacted] [redacted]', + 'QNZd4i2J6YY5', + ], + ['AKIAQPJ53XE28CC4ABCDsk-proj-abcdefghijklmnopqrstuv', '[redacted][redacted]', 'ghijklmnop'], + [ + 'AKIAQPJ53XE28CC4A\nBCDsk-proj-abcdefghijklmnopqrstuv', + '[redacted][redacted]', + 'ghijklmnop', + ], + ]); + }); + + it('keeps the host of a URL whose userinfo names a secret', () => { + const userinfo = (value: unknown) => expect(value, 'SECRETS_B6_CORE_USERINFO_ZONE'); + for (const [input, expected] of [ + [ + 'https://x-access-token:ghp_abcdefghijklmnopqrstuvwxyz0123@github.com/org/repo.git', + 'https://[redacted]@github.com/org/repo.git', + ], + [ + 'https://token:abcdef@h.example/x?password=hunter2hunter2', + 'https://[redacted]@h.example/x?[redacted]', + ], + [`https://a&sig=${'X'.repeat(28)}:pw@h.example/`, 'https://[redacted]@h.example/'], + // Before a raw ?/#, a `user:password` owns its text; the query after it stays readable. + ['https://token:pa?ss@h.example/repo', 'https://[redacted]@h.example/repo'], + ['https://token:abc?x=1@host/path?y=2', 'https://[redacted]@host/path?y=2'], + ] as const) { + userinfo(redactSecretShapedText(input)).toBe(expected); + } + }); + + it('reads every key of an ambiguous userinfo that names no user', () => { + projects('SECRETS_B6_CORE_USERINFO_COLON', [ + ['https://password=Secret1?x@t123456', 'https://[redacted]', 't123456'], + [`https://a&sig=${'X'.repeat(28)}?y@tail99`, 'https://[redacted]', 'tail99'], + ['https://svc?a=1;password=P@ssw0rd123', 'https://[redacted]', 'ssw0rd123'], + ['https://svc?password:P@ssw0rd99', 'https://[redacted]', 'ssw0rd99'], + ['https://svc?x!password=Secr@t123456', 'https://[redacted]', 't123456'], + ['https://svc?a=b+password=Secr@t123456', 'https://[redacted]', 't123456'], + [`https://svc?${'n'.repeat(70)}.password=Secr@t123456`, 'https://[redacted]', 't123456'], + ]); + }); + + it('projects a split URL into the summary and the journal value alike', async () => { + const result = await boundForModel('see\nhttps://gateway?password=P@ssw0rd123 now', BIG, {}); + expect(result.summary, 'SECRETS_B6_CORE_MERGE').toBe('see https://[redacted] now'); + expect( + redactSecretShapedValue({ url: 'x\thttps://gateway?password=P@ssw0rd123' }), + 'SECRETS_B6_CORE_MERGE', + ).toEqual({ url: 'x\thttps://[redacted]' }); + }); + + it( + 'keeps many query assignments in an ambiguous userinfo linear at admitted size', + () => { + const size = 8 * 1024 * 1024 - 256; + const run = (prefix: string, unit: string, suffix: string) => + prefix + + unit.repeat(Math.floor((size - prefix.length - suffix.length) / unit.length)) + + suffix; + for (const [input, expected] of [ + [run('https://x', '?password=', '@h'), 'https://[redacted]'], + [run('https://x', '?a=', '@h'), 'https://[redacted]'], + [run('https://svc?', '#password=x', '@h'), 'https://[redacted]'], + [run('see https://svc', '?token=a@b', ''), 'see https://[redacted]'], + [run('redis://app:pw', '?client_secret=x', '@db'), 'redis://[redacted]'], + ] as const) { + let output = ''; + expect(() => { + output = redactSecretShapedText(input); + }, 'SECRETS_B6_CORE_MERGE').not.toThrow(); + expect(output === expected, 'SECRETS_B6_CORE_MERGE').toBe(true); + } + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it( + 'keeps nested values, labels and open PEM blocks linear at admitted size', + () => { + const size = 8 * 1024 * 1024 - 256; + const count = (unit: string) => Math.floor(size / unit.length); + const keys = 'token=ab>'.repeat(count('token=ab>')); + expect(redactSecretShapedText(keys) === '[redacted]', 'SECRETS_B6_CORE_NESTED_KEY').toBe( + true, + ); + const labels = count('Bearer abcdefgh/'); + expect( + redactSecretShapedText('Bearer abcdefgh/'.repeat(labels)) === + `Bearer${' [redacted]'.repeat(labels)}`, + 'SECRETS_B6_CORE_NESTED_LABEL', + ).toBe(true); + // Only the BEGIN marker whose body reaches the far END within the body bound opens a block; the + // markers before it stay, and each of them reuses one END search instead of scanning to it again. + const marker = '-----BEGIN PRIVATE KEY-----\n'; + const open = marker.repeat(count(marker) - 1); + const first = + Math.ceil((open.length - 20_000 - (marker.length - 1)) / marker.length) * marker.length; + expect( + redactSecretShapedText(`password=hunter2secret ${open}-----END PRIVATE KEY-----`) === + `[redacted] ${open.slice(0, first)}[redacted]`, + 'SECRETS_B6_CORE_MERGE', + ).toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it('masks a PEM block whose body reaches exactly the body bound', () => { + const begin = '-----BEGIN PRIVATE KEY-----'; + const end = '-----END PRIVATE KEY-----'; + const bounded = (value: unknown) => expect(value, 'SECRETS_B6_CORE_MERGE'); + bounded(redactSecretShapedText(`${begin}${'A'.repeat(20_000)}${end}`)).toBe('[redacted]'); + const longer = `${begin}${'A'.repeat(20_001)}${end}`; + bounded(redactSecretShapedText(longer) === longer).toBe(true); + }); + + it('fails closed with its own mask when the projection does not settle', () => { + vi.mocked(toFixedPoint).mockImplementationOnce((_pass, _text, failed) => failed); + expect(redactSecretShapedText('public text'), 'SECRETS_B6_CORE_FAIL_CLOSED').toBe('[redacted]'); + }); +}); + +describe('PR95 review round 7: userinfo ownership, admitted quoted values, glued credentials', () => { + const projects = (marker: string, pairs: readonly (readonly [string, string, string])[]) => { + for (const [input, expected, secret] of pairs) { + const output = redactSecretShapedText(input); + expect(output, marker).toBe(expected); + expect(output.includes(secret), marker).toBe(false); + expect(redactSecretShapedText(output), marker).toBe(output); + } + }; + + it("owns no text past a split userinfo's first LF or TAB", () => { + projects('SECRETS_B7_CORE_SPLIT_ZONE', [ + [ + 'see https://\nDATABASE_PASSWORD=P@ssw0rd123 now', + 'see https://[redacted] now', + 'ssw0rd123', + ], + ['x https://\tSERVICE_TOKEN_VALUE=Adm1n@2024x', 'x https://[redacted]', '2024x'], + [ + 'DATABASE_URL=https://db.internal.example.com\nDB_PASSWORD=P@ssw0rd123\n', + 'DATABASE_URL=https://[redacted]\n', + 'ssw0rd123', + ], + [ + 'REDIS_URL=redis://cache.internal:6379\nREDIS_PASSWORD=r3d1s@Passw0rd\n', + 'REDIS_URL=redis://[redacted]\n', + 'Passw0rd', + ], + // A username-shaped key on the next line is no username of this URL. + ['see https://\nDB_PASSWORD:P@ssw0rd123 now', 'see https://[redacted] now', 'ssw0rd123'], + ['x https://\tSERVICE_TOKEN:Adm1n@2024x', 'x https://[redacted]', '2024x'], + // A mask past the userinfo's @ stops at the next line, which is read on its own. + [ + 'DATABASE_URL=https://db.internal.example.com\nDB_PASSWORD=P@ssw0rd123\nNEXT=1', + 'DATABASE_URL=https://[redacted]\nNEXT=1', + 'ssw0rd123', + ], + // Read on its own line, the key masks its whole value, a host-shaped tail included. + [ + 'see https://\nDB_PASSWORD=P@host.example now', + 'see https://\n[redacted] now', + 'host.example', + ], + ]); + }); + + it("reads a key in a userinfo's password or parameter position", () => { + projects('SECRETS_B7_CORE_USERNAME_ZONE', [ + [ + 'jdbc:sqlserver://db.example:1433;databaseName=app;user=sa;password=P@ssw0rd2024;encrypt=true', + 'jdbc:sqlserver://[redacted];encrypt=true', + 'ssw0rd2024', + ], + ['https://svc;a=1;password=P@ssw0rd123', 'https://[redacted]', 'ssw0rd123'], + ['mysql://h.example;password=Secr3tValue@x', 'mysql://[redacted]', 'Secr3tValue'], + [ + '{"content":"REDIS_URL=redis://cache.internal:6379\\nREDIS_PASSWORD=r3d1s@Passw0rd\\n"}', + '{"content":"REDIS_URL=redis://[redacted]"}', + 'Passw0rd', + ], + // Its value runs to the key rule's own delimiter, through the host, path and query. + [ + 'https://user:password=Secr3tValue1@host.example/v1/a', + 'https://[redacted]', + 'host.example', + ], + [ + 'GET https://svc:secret=Qz7abc@api.example.com/v1/items/123?expand=all&page=2 200', + 'GET https://[redacted]&page=2 200', + 'api.example.com', + ], + ['sqlserver://;password=Adm1n@2024.prod/app', 'sqlserver://;[redacted]', '2024.prod'], + [ + '"custom\t://user:db.password=Qz5SjmRYV4@Qz7uzLd396yJU1\neyJQz4kX2Nk3X2tw9s.Qz6CteaZX56zE1JL09cObAqM06.Qz0wvWBdMRLWj7 x', + '"custom\t://[redacted]\n[redacted] x', + 'Qz7uzLd396yJU1', + ], + ]); + // From a raw ?/# on, a key reads as a query parameter whose value runs to its own end too, and a + // colon after a raw ?/# or before any username marks no user. + projects('SECRETS_B7_CORE_USERNAME_ZONE', [ + ['https://password=abcdefgh?x:y@host.example/path', 'https://[redacted]', 'abcdefgh'], + ['https://:password=abcdefghij?x@host.example/p', 'https://[redacted]', 'abcdefghij'], + ]); + // Only a value some rule reads runs to its own end: a short value under another name is masked + // through the userinfo's last '@', and its text after that '@' reads as the host. + expect( + redactSecretShapedText('https://svc?a=1&code=Adm1n@2024prod'), + 'SECRETS_B7_CORE_USERNAME_ZONE', + ).toBe('https://[redacted]@2024prod'); + // Such a value is a recognized credential: calls that differ only in its host or path share a + // journal projection, while a URL whose userinfo holds no key keeps them apart. + const identity = (url: string) => projectSecretShapedValue({ url, method: 'POST' }).value; + expect( + identity('https://user:password=Secr3tValue1@host.example/v1/a'), + 'SECRETS_B7_CORE_USERNAME_ZONE', + ).toEqual(identity('https://user:password=Secr3tValue1@other.example/v2/delete/7')); + expect( + identity('https://user:Secr3tValue1@host.example/v1/a'), + 'SECRETS_B7_CORE_USERNAME_ZONE', + ).not.toEqual(identity('https://user:Secr3tValue1@host.example/v1/b')); + }); + + it('keeps the host when only the username names a secret, split after its colon or not', () => { + const host = (value: unknown) => expect(value, 'SECRETS_B7_CORE_USERNAME_ZONE'); + host( + redactSecretShapedText( + 'https://x-access-token:\nghp_abcdefghijklmnopqrstuvwxyz0123@github.com/o/r', + ), + ).toBe('https://[redacted]@github.com/o/r'); + // Split inside the username, the key after the break is read and masks the host: fail-closed. + host( + redactSecretShapedText( + 'https://x-access-\ntoken:ghp_abcdefghijklmnopqrstuvwxyz0123@github.com/o/r', + ), + ).toBe('https://[redacted]'); + }); + + it( + 'projects exactly admitted-size quoted values without exhausting the RegExp stack', + () => { + const size = 8 * 1024 * 1024; + const exact = (prefix: string, suffix: string) => + `${prefix}${'a'.repeat(size - prefix.length - suffix.length)}${suffix}`; + for (const [input, expected] of [ + // An unclosed value is read as an unquoted run from after its quote; here that run reaches the + // end of the text. + [exact('token="', ''), '[redacted]'], + [exact('password="', '"'), '[redacted]'], + [exact("password='", "'"), '[redacted]'], + [exact('{"access_token":"', '"}'), '{"[redacted]}'], + // One JSON level down, closed and unclosed. + [exact('{"b":"{\\"token\\":\\"', '\\"}"}'), '{"b":"{\\"[redacted]}"}'], + [exact('{"b":"token=\\"', '"}'), '{"b":"[redacted]"}'], + ] as const) { + expect(input.length, 'SECRETS_B7_CORE_QUOTED_SCAN').toBe(size); + let output = ''; + expect(() => { + output = redactSecretShapedText(input); + }, 'SECRETS_B7_CORE_QUOTED_SCAN').not.toThrow(); + expect(output === expected, 'SECRETS_B7_CORE_QUOTED_SCAN').toBe(true); + } + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it("keeps quoted values on one line and pins the key rule's names and delimiters", () => { + const keys = (value: unknown) => expect(value, 'SECRETS_B7_CORE_KEY_PIECES'); + for (const input of [ + 'passwd=hunter2hunter2', + 'access_key=hunter2hunter2', + 'access-key: hunter2hunter2', + 'private_key=hunter2hunter2', + 'private-key: hunter2hunter2', + 'api_key_id=abcdef123456', + ]) { + keys(redactSecretShapedText(input)).toBe('[redacted]'); + } + keys(redactSecretShapedText('password=hunter2hunter2&next=public')).toBe( + '[redacted]&next=public', + ); + keys(redactSecretShapedText('token=abcdefgh,next')).toBe('[redacted],next'); + keys(redactSecretShapedText("token=abcdefgh'tail")).toBe("[redacted]'tail"); + // A quoted value closes on its line. One that does not, even past an escaped line break, is read + // as an unquoted run from after its quote, to that line's end at most: its quote may instead close + // a string the key sits in. + for (const kept of [ + 'password="abc\ndef" tail', + "password='abc\rdef' tail", + 'password="abc\ndef tail', + 'password="abc\\\ndef tail', + ]) { + keys(redactSecretShapedText(kept)).toBe(kept); + } + keys(redactSecretShapedText('password="abcdef\nghi" tail')).toBe('[redacted]\nghi" tail'); + keys(redactSecretShapedText('password="abcdef\\\nghi tail')).toBe('[redacted]\nghi tail'); + // A key must start within its word's first 33 characters. + const listed = `x${'y'.repeat(40)}token=listed`; + keys(redactSecretShapedText(listed)).toBe(listed); + }); + + it('recognizes a credential glued to another one pass later and redacts a run that never settles', async () => { + const two = 'AKIAAAAAAAAAAAAAAAAAASIABBBBBBBBBBBBBBBB'; + projects('SECRETS_B8_CORE_GLUED', [ + [`${two} public tail`, '[redacted][redacted] public tail', 'BBBBBBBB'], + // Four glued ids need four passes: the text does not settle and fails closed. + [`${two}AKIACCCCCCCCCCCCCCCCASIADDDDDDDDDDDDDDDD public tail`, '[redacted]', 'DDDDDDDD'], + [ + 'AKIAABCDEFGHIJKLMNOPghp_AAAAAAAAAAAA\nBBBBBBBBBBBBBBBB done', + '[redacted][redacted] done', + 'BBBBBBBB', + ], + [ + 'AKIAABCDEFGHIJKLMNOPsk-proj-AAAAAAAAAAAAAAAA\nBBBBBBBBBBBB done', + '[redacted][redacted] done', + 'BBBBBBBB', + ], + // A glued credential is read at the mask's boundary, so it neither cuts a wider credential + // that starts inside it nor swallows a key name or a scheme label after it. + [ + 'AKIAIOSFODNN7EXAMPLExoxb-eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.SflKxwRJSMeKKF2QT4fwpM', + '[redacted]xoxb-[redacted]', + 'SflKxwRJSMeKKF2QT4fwpM', + ], + [ + 'AKIAIOSFODNN7EXAMPLExoxb-sk-proj-abcdefghij_klmnopqrstuv', + '[redacted]xoxb-[redacted]', + 'klmnopqrstuv', + ], + [ + 'AKIAIOSFODNN7EXAMPLEnpm_abcdefghij0123456789token=SuperSecret99', + '[redacted][redacted]', + 'SuperSecret99', + ], + [ + 'AKIAABCDEFGHIJKLMNOPsk-AKIAABCDEFGHIJKLMNOPBearer abcdefghijklmnop', + '[redacted]sk-[redacted]Bearer [redacted]', + 'abcdefghijklmnop', + ], + ]); + const summary = await boundForModel( + 'AKIAABCDEFGHIJKLMNOPsk-proj-AAAAAAAAAAAAAAAA\tBBBBBBBBBBBB done', + BIG, + {}, + ); + expect(summary.summary, 'SECRETS_B8_CORE_GLUED').toBe('[redacted][redacted] done'); + }); + + it("summarizes with the projection's fixed point before one-line normalization", async () => { + const result = await boundForModel( + 'g://s.hos#mHPGE\nAXXQ8@?q==a@b.exam/user:3eYPUzfxo', + BIG, + {}, + ); + expect(result.summary, 'SECRETS_B7_CORE_SUMMARY').toBe('g://[redacted]@?q=[redacted]'); + }); + + it('reads a value through earlier masks of every kind, in text order', () => { + projects('SECRETS_B6_CORE_VALUE_THROUGH_MASK', [ + ['token=xyz?sig=AAAAAAAAAAAAAAAAAAAAAAAA;BBB#TAILSECRET', '[redacted]', 'TAILSECRET'], + [ + 'password=https://svc?sig=AAAAAAAAAAAAAAAAAAAAAAAAAAA&x=1@hostsecretTail', + '[redacted]', + 'secretTail', + ], + [ + 'password=-----BEGIN RSA PRIVATE KEY-----\nabc\n-----END RSA PRIVATE KEY-----TAIL https://u:p@h', + '[redacted] https://[redacted]@h', + 'TAIL', + ], + ]); + }); + + it('keeps a scheme label and its space when its token is also a standalone credential', () => { + projects('SECRETS_B6_CORE_MERGE', [ + ['Bearer sk-abcdefghijklmnopqrstuvwx', 'Bearer [redacted]', 'abcdefghij'], + [ + 'Authorization: Bearer sk-proj-abcdefghijklmnopqrstuvwx', + '[redacted] [redacted]', + 'abcdefghij', + ], + ['token sk-abcdefghijklmnopqrstuvwx', 'token [redacted]', 'abcdefghij'], + ]); + }); + + it('reports whether a journal projection settled', () => { + const journal = (value: unknown) => expect(value, 'SECRETS_B7_CORE_JOURNAL_SETTLED'); + const ids = 'AKIAAAAAAAAAAAAAAAAAASIABBBBBBBBBBBBBBBB'; + journal( + projectSecretShapedValue({ path: 'test/fixtures/keys.txt', content: `${ids}\nscan(x)\n` }), + ).toEqual({ + value: { path: 'test/fixtures/keys.txt', content: '[redacted][redacted]\nscan(x)\n' }, + settled: true, + }); + // Four glued ids need four passes, and a key whose short value names the next key one per key. + for (const content of [`${ids}${ids}\nscan(x)\n`, 'token:token =token ://?-=Q']) { + journal(projectSecretShapedValue({ path: 'p', content })).toEqual({ + value: { path: 'p', content: '[redacted]' }, + settled: false, + }); + } + vi.mocked(fixedPointOf).mockImplementationOnce(() => undefined); + journal(projectSecretShapedValue('drifting text')).toEqual({ + value: '[redacted]', + settled: false, + }); + }); +}); + +describe('PR95 review round 8: structured summaries, escaped and unclosed quotes, continuation labels', () => { + const projects = (marker: string, pairs: readonly (readonly [string, string, string])[]) => { + for (const [input, expected, secret] of pairs) { + const output = redactSecretShapedText(input); + expect(output, marker).toBe(expected); + expect(output.includes(secret), marker).toBe(false); + expect(redactSecretShapedText(output), marker).toBe(output); + } + }; + + it('summarizes a structured result from its JSON once credential members are masked', async () => { + for (const [result, secrets] of [ + [ + { content: 'DB_PASSWORD="hunter2hunter2"\nNODE_ENV=production\n', mimeType: 'text/plain' }, + ['hunter2hunter2'], + ], + [ + { + content: + '{\n "database": {"password": "Qz9correcthorse"}, "client_secret": "Qz5clientsecretval"}', + }, + ['Qz9correcthorse', 'Qz5clientsecretval'], + ], + [ + { + status: 200, + headers: {}, + body: '{"access_token":"2YotnFZFEjr1zCsicMWpAA","refresh_token":"tGzv3JOkF0XG5Qx2TlKWIA"}', + }, + ['2YotnFZFEjr1zCsicMWpAA', 'tGzv3JOkF0XG5Qx2TlKWIA'], + ], + [ + { + exitCode: 0, + stdout: 'export AWS_SECRET_ACCESS_KEY="wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY"\n', + stderr: '', + }, + ['wJalrXUtnFEMIK7MDENG'], + ], + // The walk also masks what no text rule reads: a secret-named key's whole value and the + // element after a credential flag. + [{ token: { value: 'Qz7nestedSecret' } }, ['Qz7nestedSecret']], + [{ argv: ['--password', 'Qz7flagSecret'] }, ['Qz7flagSecret']], + ] as const) { + const bounded = await boundForModel(result, BIG, {}); + expect(bounded.value, 'SECRETS_B8_CORE_SUMMARY_WALK').toBe(result); + for (const secret of secrets) { + expect(bounded.summary.includes(secret), 'SECRETS_B8_CORE_SUMMARY_WALK').toBe(false); + } + } + const summary = await boundForModel( + { content: 'DB_PASSWORD="hunter2hunter2"\nNODE_ENV=production\n' }, + BIG, + {}, + ); + expect(summary.summary, 'SECRETS_B8_CORE_SUMMARY_WALK').toBe( + '{"content":"[redacted]\\nNODE_ENV=production\\n"}', + ); + }); + + it('reads a quoted value one JSON string down', () => { + projects('SECRETS_B8_CORE_ESCAPED_QUOTE', [ + [ + '{"body":"{\\"password\\":\\"hunter2hunter2\\",\\"user\\":\\"bob\\"}"}', + '{"body":"{\\"[redacted],\\"user\\":\\"bob\\"}"}', + 'hunter2hunter2', + ], + [ + '{"stdout":"export TOKEN=\\"Qz7abc12345\\" && run"}', + '{"stdout":"export [redacted] && run"}', + 'Qz7abc12345', + ], + [ + '{"h":"{\\"x-api-key\\":\\"Bearer Qz7abcdefghijkl\\"}"}', + '{"h":"{\\"[redacted]}"}', + 'Qz7abcdefghijkl', + ], + // An escaped tab or line break may stand between the separator and the escaped quote. + ['{"cfg":"password:\\t\\"hunter2Secr3t\\""}', '{"cfg":"[redacted]"}', 'hunter2Secr3t'], + ['{"cfg":"password:\\n \\"hunter2Secr3t\\""}', '{"cfg":"[redacted]"}', 'hunter2Secr3t'], + // A raw tab may stand among them, an escaped CR LF is one line break, and an escaped apostrophe + // opens such a value too; so does a raw one, which JSON leaves unescaped. + ['{"cfg":"password:\\n\t\\"hunter2Secr3t\\""}', '{"cfg":"[redacted]"}', 'hunter2Secr3t'], + ['{"cfg":"password:\\r\\n\\"hunter2Secr3t\\""}', '{"cfg":"[redacted]"}', 'hunter2Secr3t'], + ['{"cfg":"password:\\n \\\'hunter2Secr3t\\\'"}', '{"cfg":"[redacted]"}', 'hunter2Secr3t'], + ['{"yaml":"password:\\t\'Adm1nPassw0rd\'"}', '{"yaml":"[redacted]"}', 'Adm1nPassw0rd'], + ['{"yaml":"password:\\n \'Adm1nPassw0rd\'"}', '{"yaml":"[redacted]"}', 'Adm1nPassw0rd'], + // An opaque value runs to its delimiter, the backslash before a quote included. + [ + '{"m":"{\\"u\\":\\"https://h.example/cb?sig=AbCdEfGhIjKlMnOpQrStUvW\\"}"}', + '{"m":"{\\"u\\":\\"https://h.example/cb?sig=[redacted]"}"}', + 'AbCdEfGhIjKlMnOpQrStUvW', + ], + // A quote escaped one level further down stays inside the value. + [ + '{"cmd":"export TOKEN=\\"Qz\\\\\\"SecretTail\\" end"}', + '{"cmd":"export [redacted] end"}', + 'SecretTail', + ], + // An escaped backslash escapes only the character right after it. + [ + '{"cmd":"export TOKEN=\\"Qz7abcdef\\\\x\\" public"}', + '{"cmd":"export [redacted] public"}', + 'Qz7abcdef', + ], + ]); + }); + + it('reads a quoted value that does not close on its line as an unquoted run', () => { + const runs = [ + ['{"password":"hunter2hunter2', '{"[redacted]', 'hunter2hunter2'], + ['password: "hunter2hunter2\nnext line', '[redacted]\nnext line', 'hunter2hunter2'], + ["DB_PASSWORD='hunter2hunter2\nNEXT=1", '[redacted]\nNEXT=1', 'hunter2hunter2'], + ['api_key="Qz7abcdefghij\\', '[redacted]', 'Qz7abcdefghij'], + // The run ends at the delimiters of an unquoted value and at its line's end, which a CR, a + // backslash before one or a lone backslash also is. + ['password="Qz7abcdef public tail', '[redacted] public tail', 'Qz7abcdef'], + ['password="Qz7abcdef,next', '[redacted],next', 'Qz7abcdef'], + ['password="Qz7abcdef;next', '[redacted];next', 'Qz7abcdef'], + ['password="Qz7abcdef&x=1', '[redacted]&x=1', 'Qz7abcdef'], + ['password="Qz7abcdef\'x', "[redacted]'x", 'Qz7abcdef'], + ['password="Qz7abcdef\tpublic', '[redacted]\tpublic', 'Qz7abcdef'], + ['password="Qz7abc\ndef\nghi tail', '[redacted]\ndef\nghi tail', 'Qz7abc'], + ['password="Qz7abc\ndef\\', '[redacted]\ndef\\', 'Qz7abc'], + ['password="Qz7abc\rdef tail', '[redacted]\rdef tail', 'Qz7abc'], + ['password="Qz7abc\\\rdef tail', '[redacted]\rdef tail', 'Qz7abc'], + ['password="Qz7abcdef\\\nmore" tail', '[redacted]\nmore" tail', 'Qz7abcdef'], + ['password="Qz7abcdef\\\rmore" tail', '[redacted]\rmore" tail', 'Qz7abcdef'], + // A value that closes on a later line is read the same way, and a run of such values settles in + // one pass. + ['password="Qz7abcdef\nmore" tail', '[redacted]\nmore" tail', 'Qz7abcdef'], + [ + 'A_TOKEN="Qz7aaa1\nB_TOKEN=\'Qz7bbb2\nC_TOKEN="Qz7ccc3\nD_TOKEN="Qz7ddd4"', + '[redacted]\n[redacted]\n[redacted]\n[redacted]', + 'Qz7bbb2', + ], + // One JSON level down only an escaped quote closes the value; the raw quote or raw line break + // that ends the string around it, like an escaped line break, ends its run. + [ + '{"a":"password=\\"Qz7abcdefgh","b":"public"}', + '{"a":"[redacted]","b":"public"}', + 'Qz7abcdefgh', + ], + ['{"a":"password=\\"Qz7abcdef ghi","b":"x"}', '{"a":"[redacted] ghi","b":"x"}', 'Qz7abcdef'], + [ + '{"a":"password=\\"Qz7abcdefgh\\nline two","b":"public"}', + '{"a":"[redacted]\\nline two","b":"public"}', + 'Qz7abcdefgh', + ], + [ + '{"a":"password=\\"Qz7abcdefgh\\rline two","b":"public"}', + '{"a":"[redacted]\\rline two","b":"public"}', + 'Qz7abcdefgh', + ], + [ + '{"a":"password=\\"Qz7abc\\nQz7def\\" tail"}', + '{"a":"[redacted]\\nQz7def\\" tail"}', + 'Qz7abc', + ], + [ + '{"a":"password=\\"Qz7abcdefgh\n{"b":"say \\"hi\\""}', + '{"a":"[redacted]\n{"b":"say \\"hi\\""}', + 'Qz7abcdefgh', + ], + [ + 'echo password=\\"Qz7abc\\\nline two \\\nline three\nend', + 'echo [redacted]\nline two \\\nline three\nend', + 'Qz7abc', + ], + [ + '{"a":"A_TOKEN=\\"Qz7aaa1\\n B_TOKEN=\\"Qz7bbb2\\n C_TOKEN=\\"Qz7ccc3\\n D_TOKEN=\\"Qz7ddd4\\""}', + '{"a":"[redacted]\\n [redacted]\\n [redacted]\\n [redacted]"}', + 'Qz7aaa1', + ], + // A raw quote of the other kind does not end it: an apostrophe is text in a JSON string. + [ + '{"a":"token=\\\'Qz7secret","b":"public one"}', + '{"a":"[redacted]","b":"public one"}', + 'Qz7secret', + ], + // A value whose line break another mask of the pass takes (a scheme label's token, a nested + // key's value) closes as the next pass reads it, and is masked whole. + ['DB_PASSWORD="Bearer\nabcdefghij Zq7Wk3Xp9Rm2" tail', '[redacted] tail', 'Zq7Wk3Xp9Rm2'], + [ + 'Authorization: "Bearer\r\nabcdefghij Zq7Wk3Xp9Rm2" tail', + '[redacted] tail', + 'Zq7Wk3Xp9Rm2', + ], + ['password="abcdef Bearer\nabcdefghij Zq7Wk3Xp9Rm2" tail', '[redacted] tail', 'Zq7Wk3Xp9Rm2'], + ['token:"Kx96Wv Qze6S0YzVYHq Bearer\nQVI3ml-K"', '[redacted]', 'Qze6S0YzVYHq'], + [ + 'client_secret="token:\nabcdefghijkl Zq7Wk3Xp9Rm2Vn5Ty8Lb" end', + '[redacted] end', + 'Zq7Wk3Xp9Rm2Vn5Ty8Lb', + ], + // A quote that closes another value is that value's: no value it opens is read through the + // other masks. + ["token='abc password='xyz] public", '[redacted]xyz] public', 'abc'], + // So does one whose escaped closing quote an opaque value's mask unescapes, behind an odd or + // an even run of backslashes. + [ + 'password="say \\"Zq7Wk3Xp9Rm2Vn5Ty8Lb?sig=AAAAAAAAAAAAAAAAAAAAAAAA\\"', + '[redacted]', + 'Zq7Wk3Xp9Rm2Vn5Ty8Lb', + ], + [ + 'password="my Zq9SECRETvalue7Xk?sig=AbCdEfGhIjKlMnOpQrStUvWx\\\\" next', + '[redacted] next', + 'Zq9SECRETvalue7Xk', + ], + ] as const; + projects('SECRETS_B8_CORE_UNCLOSED_QUOTE', runs); + for (const [input, expected] of runs) { + expect(projectSecretShapedValue({ a: input }), 'SECRETS_B8_CORE_UNCLOSED_QUOTE').toEqual({ + value: { a: expected }, + settled: true, + }); + } + // Its quote may instead close a string the key sits in, so the text after the run stays, and calls + // that differ only there keep distinct projections. A run under six characters is no value. + const script = 'read -p "Enter password: " PW\necho "connecting to $HOST"\n'; + for (const kept of [ + script, + '{\n "label": "Password:",\n "hint": "at least 8 chars"\n}', + 'print("token: " + t)\nprint("done")', + 'fix(auth): stop logging "token=" query values in the billing service', + "grep -rn 'api_key=' config/prod.yaml", + `psql "password="$PW -c 'DROP TABLE orders_a'`, + '{"msg":"stop logging \\"token=\\" values in billing"}', + 'A_TOKEN="Qz7a1\nx=\\\\"y', + ]) { + expect(redactSecretShapedText(kept), 'SECRETS_B8_CORE_UNCLOSED_QUOTE').toBe(kept); + } + for (const [one, other] of [ + [script, script.replace('PW\n', 'PW; rm -rf ./build\n')], + [ + 'fix(auth): stop logging "token=" query values in the billing service', + 'fix(auth): stop logging "token=" query values in the payments service', + ], + [ + `psql "password="$PW -c 'DROP TABLE orders_a'`, + `psql "password="$PW -c 'DROP TABLE orders_b'`, + ], + [ + '{"msg":"stop logging \\"token=\\" values in billing"}', + '{"msg":"stop logging \\"token=\\" values in payments"}', + ], + [ + '{"cmd":"export TOKEN=\\"Qz7abcdef; echo done"}', + '{"cmd":"export TOKEN=\\"Qz7abcdef; rm -rf x"}', + ], + ]) { + expect( + projectSecretShapedValue({ content: one }).value, + 'SECRETS_B8_CORE_UNCLOSED_QUOTE', + ).not.toEqual(projectSecretShapedValue({ content: other }).value); + } + }); + + it('reads a label at a line start inside a split continuation', () => { + projects('SECRETS_B8_CORE_CONTINUATION_LABEL', [ + [ + 'sk-proj-AbCdEfGhIjKlMnOpQrSt\nauthori\tzation: Qz7SecretValue99', + '[redacted]', + 'Qz7SecretValue99', + ], + ['glpat-AbCdEfGhIjKlMnOpQrSt\napi-k\tey: Qz7SecretValue99', '[redacted]', 'Qz7SecretValue99'], + ]); + }); + + it('masks two and three private-key blocks in one text', () => { + const block = (kind: string, body: string) => + `-----BEGIN ${kind} PRIVATE KEY-----\n${body}\n-----END ${kind} PRIVATE KEY-----`; + projects('SECRETS_B8_CORE_PEM_BLOCKS', [ + [ + `a ${block('RSA', 'AAA')}\nb ${block('EC', 'BBBSECRET')} c`, + 'a [redacted]\nb [redacted] c', + 'BBBSECRET', + ], + [ + `${block('RSA', 'X1SECRET')} ${block('EC', 'X2SECRET')} ${block('OPENSSH', 'X3SECRET')}`, + '[redacted] [redacted] [redacted]', + 'X3SECRET', + ], + ]); + }); + + it("keeps a label's own whitespace before a split credential", () => { + expect( + redactSecretShapedText('Bearer sk-\nabcdefghijklmnopqrstuvwx done'), + 'SECRETS_B8_CORE_FILL', + ).toBe('Bearer [redacted] done'); + }); + it('summarizes a value that closes on a later line whole', async () => { + for (const [result, summary, secret] of [ + [ + 'token=\'abc password="first-half\nSECOND-HALF-SECRET" done', + '[redacted] done', + 'SECOND-HALF', + ], + ['password="password="password="\nQ7mZ4pW9xK"', '[redacted]', 'Q7mZ4pW9xK'], + ['secret: \'line1\nline2 token="it\'s"', '[redacted]', 'line2'], + [{ content: 'password="hunter2 value\nmore text"' }, '{"content":"[redacted]"}', 'hunter2'], + [ + { stdout: 'password="first-half\r\nSECOND-HALF" done' }, + '{"stdout":"[redacted] done"}', + 'SECOND-HALF', + ], + [['password="hunter2 value\nmore text"'], '["[redacted]"]', 'more text'], + // A masked value after it can take the quote that would close it. + ["token='\ntoken=''Qz7secret'", '[redacted]', 'Qz7secret'], + ['password="\npassword=""Qz7secret"', '[redacted]', 'Qz7secret'], + // When masked values take every later quote, the summary masks through the last of them. + ["token='\ntoken=''Qz7secret token='", '[redacted]', 'Qz7secret'], + // A structured result is read in its JSON, where the value one JSON string down runs on over the + // escaped line break to the end of the text. + [ + { stdout: 'token=\'abc password="first-half\nSECOND-HALF-SECRET" done' }, + '{"stdout":"[redacted]', + 'SECOND-HALF', + ], + ['{"log":"password=\\"Qz7secret\\nmore\\" ok"}', '{"log":"[redacted] ok"}', 'Qz7secret'], + ] as const) { + const bounded = await boundForModel(result, BIG, {}); + expect(bounded.summary, 'SECRETS_B8_CORE_SUMMARY_CLOSER').toBe(summary); + expect(bounded.summary.includes(secret), 'SECRETS_B8_CORE_SUMMARY_CLOSER').toBe(false); + } + // A value that reaches the end of the text unclosed is masked to that end. The nearest quote that + // closes the value ends its mask, so public text after it stays; a raw line break behind a backslash + // ends a value one JSON string down, and a quote behind three backslashes closes no value, one + // JSON string down too, where one behind five closes it. Each value is decided after its own line: + // a quote there that a nested value's mask leaves unescaped closes nothing. + for (const [result, summary] of [ + ['token="Qz7abcdef ghi jkl', '[redacted]'], + ['password="Qz7abc\ndefSECRET\\\\" tail', '[redacted] tail'], + ['password="Qz7abc\ndefSECRET\\\\\\\\" tail', '[redacted] tail'], + ['token="Qz7first\ntoken="Qz7second\nend', '[redacted] end'], + ['{"cmd":"export TOKEN=\\"Qz7abcdef; echo done"}', '{"cmd":"export [redacted]"}'], + ['token:"token=\\\'token=h\\\\\\r\\"\n"', '[redacted]'], + ['password=\\"Qz7abcdefgh\\nx \\\\\\" y', '[redacted]\\nx \\\\\\" y'], + ['password=\\"Qz7abcdefgh\\nx \\\\\\\\\\" y', '[redacted] y'], + ['password="Qz7abc\ndef" public "kept"', '[redacted] public "kept"'], + ['echo password=\\"Qz7abc\\\nline two \\" end', 'echo [redacted] line two \\" end'], + ['password="Qz7abcdefgh\nmore \\\\\\" text', '[redacted] more \\\\\\" text'], + ] as const) { + expect((await boundForModel(result, BIG, {})).summary, 'SECRETS_B8_CORE_SUMMARY_CLOSER').toBe( + summary, + ); + } + // The journal projection reads such a value only to its first line's end: its quote may close a + // string around it. + expect( + redactSecretShapedText('password="first-half\nSECOND-HALF" done'), + 'SECRETS_B8_CORE_SUMMARY_CLOSER', + ).toBe('[redacted]\nSECOND-HALF" done'); + }); + + it("reads a structured result's values across its line breaks and members", async () => { + // The text rules read the JSON as one text: a JSON-escaped line break is no delimiter, and a PEM + // block or a quoted value that runs on into later members keeps its key and markers. + const pem = [ + '-----BEGIN RSA PRIVATE KEY-----', + 'MIIEowIBAAKCAQEAuY5wq3Zx9bQk1s8hR2pV7tX0aLmN4cQe', + 'Kq1sW9eR4tY7uI0oP2aS5dF8gH1jK3lZ', + '-----END RSA PRIVATE KEY-----', + ]; + const signed = + 'curl -sS https://bucket.s3.example/obj?X-Amz-Signature=0123456789abcdef0123456789abcdef'; + for (const [result, summary, secret] of [ + [ + { + lines: [ + 'APP_NAME=demo', + `PRIVATE_KEY="${pem[0]}`, + pem[1], + pem[2], + `${pem[3]}"`, + 'PORT=8080', + ], + }, + '{"lines":["APP_NAME=demo","[redacted]","PORT=8080"]}', + 'MIIEowIBAAKCAQEA', + ], + [ + { lines: [`password=${pem[0]}`, pem[1], pem[3]] }, + '{"lines":["[redacted]"]}', + 'MIIEowIBAAKCAQEA', + ], + [ + { lines: ["export DB_PASSWORD='first", "hunter2secretpw'", 'echo ok'] }, + '{"lines":["export [redacted]","echo ok"]}', + 'hunter2secretpw', + ], + [{ stdout: "?token='\nhunter2secretpw\\'" }, '{"stdout":"?[redacted]"}', 'hunter2secretpw'], + ...['\n', '\r\n', '\r', '\t'].map( + (separator) => + [ + { stdout: `DB_PASSWORD=hunter2hunter2${separator}Q9zXw7TokenValue` }, + '{"stdout":"[redacted]"}', + 'Q9zXw7', + ] as const, + ), + ...['\n', '\r\n', '\t'].map( + (separator) => + [ + { stdout: `${signed}${separator}0123456789abcdefSECRETCONTINUATION -o out` }, + '{"stdout":"curl -sS https://bucket.s3.example/obj?X-Amz-Signature=[redacted] -o out"}', + 'SECRETCONTINUATION', + ] as const, + ), + [ + { + content: [ + { + type: 'text', + text: 'private_key=MIIEvQIBADANBgkqhkiG9w0BAQEFAASC\nSECONDLINEu8XJk2Wq4tRmYv0yZp3\nTHIRDLINEaBcDeFgHiJk', + }, + ], + }, + '{"content":[{"type":"text","text":"[redacted]"}]}', + 'SECONDLINE', + ], + ] as const) { + const bounded = await boundForModel(result, BIG, {}); + expect(bounded.summary, 'SECRETS_B8_CORE_SUMMARY_JSON').toBe(summary); + expect(bounded.summary.includes(secret), 'SECRETS_B8_CORE_SUMMARY_JSON').toBe(false); + } + }); + + it('shows a summary that does not settle as its own mask', async () => { + // Four glued ids need four passes, so neither summary pass settles. + const run = 'AKIAAAAAAAAAAAAAAAAAASIABBBBBBBBBBBBBBBB'.repeat(2); + for (const [result, summary] of [ + [run, '[redacted]'], + [{ stdout: run }, '[redacted]'], + ] as const) { + expect( + (await boundForModel(result, BIG, {})).summary, + 'SECRETS_B8_CORE_SUMMARY_UNSETTLED', + ).toBe(summary); + } + }); + + it( + 'summarizes a many-part result within the case deadline', + async () => { + // An MCP result near the 4 MiB http/sse message bound: every member is walked once, and its JSON + // is read as one text. + const parts = [{ type: 'text', text: 'token=Qz7manyPartSecret' }]; + for (let index = 0; index < 150_000; index += 1) { + parts.push({ type: 'text', text: `part ${index}` }); + } + const { summary } = await boundForModel({ content: parts }, BIG, {}); + expect(summary.includes('Qz7manyPartSecret'), 'SECRETS_B8_CORE_SUMMARY_PARTS').toBe(false); + expect( + summary.startsWith('{"content":[{"type":"text","text":"[redacted]"},'), + 'SECRETS_B8_CORE_SUMMARY_PARTS', + ).toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it('reports a journal projection whose member names collide as unsettled', () => { + const colliding = { + 'sk-AAAAAAAAAAAAAAAAAAAA': 'rm -rf build', + 'sk-BBBBBBBBBBBBBBBBBBBB': 'echo done', + }; + expect(projectSecretShapedValue(colliding), 'SECRETS_B8_CORE_KEY_COLLISION').toEqual({ + value: { '[redacted]': 'echo done' }, + settled: false, + }); + expect( + projectSecretShapedValue({ + nested: { 'https://u:pw1@h/x': 'one', 'https://u:pw2@h/x': 'two' }, + }).settled, + 'SECRETS_B8_CORE_KEY_COLLISION', + ).toBe(false); + // The report reaches objects inside arrays too. + for (const value of [{ list: [colliding] }, { argv: ['run', colliding] }]) { + expect(projectSecretShapedValue(value).settled, 'SECRETS_B8_CORE_KEY_COLLISION').toBe(false); + } + // Names that stay distinct settle, and the display projection still shows one member. + expect( + projectSecretShapedValue({ 'password=a': 1, 'password=b': 2 }).settled, + 'SECRETS_B8_CORE_KEY_COLLISION', + ).toBe(true); + expect(redactSecretShapedValue(colliding), 'SECRETS_B8_CORE_KEY_COLLISION').toEqual({ + '[redacted]': 'echo done', + }); + }); +}); + +describe('PR95 review round 8, sixth pre-review: escapes under nested masks, PEM markers, escaped separators', () => { + it('closes a quoted value at the later of its readings with and without the masks', async () => { + // A mask can take the backslash that escapes a later quote. A quoted value closes at the later of two + // readings: with the masks before and inside it filled, as the next pass reads them, and with each + // mask keeping the backslashes it ends with, as the text has them. A nested key's value that takes + // the backslash before a later quote therefore does not close the value early, in the journal + // projection, whose line break a scheme token takes, and in the summary alike; after an even run of + // backslashes the quote is unescaped in both readings and closes the value. + const a26 = 'a'.repeat(26); + for (const [input, expected] of [ + ['password="Bearer\nabcdefgh token=xyzxyz\\"secretpart" more', '[redacted] more'], + [ + 'password="a Bearer\nabcdefgh9 DB_PASSWORD=x9y8z7\\"Sup3rS3cretTail" next', + '[redacted] next', + ], + ['password="Bearer\nabcdefgh token=xyzxyz\\\\"tail more" end', '[redacted]tail more" end'], + ['password="Bearer\nabcdefgh token=xyzxyz\\\\\\"secret tail" end', '[redacted] end'], + // A value that closes only where a nested key's mask unescapes its last quote is masked whole. + ['"token":"Qz7abcdefghijkl,Qz7mnopqrstuv&\\ttoken:password:\\t\\"', '"[redacted]'], + // An opaque value's mask runs to the escaped quote and takes its backslash; a quote it unescapes + // closes the value only when no later quote does, behind a URL userinfo in it or not. + [`password="?sig=${a26}https://u:p@h\\"Qz7SecretTail" public`, '[redacted] public'], + [ + `password="token\nabcdefgh9 ?sig=${a26}https://u:p@h\\"Qz7SecretTail" public`, + '[redacted] public', + ], + [`password="x ?sig=${a26}\\"Qz7SecretTail" public`, '[redacted] public'], + // The earlier rules' masks stay filled in both readings: a quote inside a private-key block + // closes nothing. + [ + 'password="Bearer\nabcdefgh -----BEGIN RSA PRIVATE KEY----- x"y -----END RSA PRIVATE KEY----- tail" more', + '[redacted] more', + ], + ] as const) { + expect(redactSecretShapedText(input), 'SECRETS_B8_CORE_CLOSER_ESCAPE').toBe(expected); + expect(projectSecretShapedValue({ a: input }), 'SECRETS_B8_CORE_CLOSER_ESCAPE').toEqual({ + value: { a: expected }, + settled: true, + }); + } + // A summary masks a value that stops at a line break through the later of the two readings' closers. + for (const [input, expected] of [ + ['export PASSWORD="line1\nDB_TOKEN=abcdef\\"Secr3tTail" done', 'export [redacted] done'], + [ + 'export PASSWORD="line1\nDB_TOKEN=abcdef\\"Secr3tTail" done "kept"', + 'export [redacted] done "kept"', + ], + ['password="Qz7abc\nDB_TOKEN=abcdef\\"', '[redacted]'], + ['password="Qz7abc\nDB_TOKEN=abcdef\\"SECRETtail" done', '[redacted] done'], + ['password="Qz7abc\nDB_TOKEN=abcdef\\\\"tail" done', '[redacted]tail" done'], + ['password="Qz7abc\nDB_TOKEN=abcdef\\\\\\"SECRET tail" done', '[redacted] done'], + [`password="Qz7abc\nx ?sig=${a26}\\"Qz7SecretTail" public`, '[redacted] public'], + [`password="Qz7abc\nx ?sig=${a26}\\"`, '[redacted]'], + // The second reading falls back to the last quote that could close the value, like the first. + ['password="Qz7\ntoken="abcdef" token=zzzzzz\\" token="ghijkl" tail', '[redacted] tail'], + // Each value is decided at its own end, whatever order the values were found in. + ['passwd="\ntoken:"\n" tail', '[redacted] tail'], + ['passwd="token=\\\'\\rtoken=ef\\\\n\\"\ntoken=bcdef\\"', '[redacted]'], + ] as const) { + expect((await boundForModel(input, BIG, {})).summary, 'SECRETS_B8_CORE_CLOSER_ESCAPE').toBe( + expected, + ); + } + }); + + it("reads a structured summary's members before the walk masks them whole", async () => { + const begin = '-----BEGIN RSA PRIVATE KEY-----'; + const end = '-----END RSA PRIVATE KEY-----'; + // The text rules read the result's JSON once before the walk's whole masks, which that same pass + // applies, so a private-key block that runs into or out of a credential member or flag argument is + // masked across members, whatever the masked value holds. + for (const [result, summary, secret] of [ + [ + { private_key: `${begin}\nMIIEAAAAfirst`, rest: `MIIEBBBBsecondbodyline\n${end}` }, + '{"[redacted]}', + 'MIIEBBBB', + ], + [ + { a: `${begin}\nMIIEpAIBAAKCAQEAbodylineone`, private_key: `bodylinetwo\n${end}` }, + '{"a":"[redacted]}', + 'bodylineone', + ], + [ + ['--private-key', begin, 'MIIEpAIBAAKCAQEAsecretbodyline1', 'abcdefsecretbodyline2', end], + '["--private-key","[redacted]""]', + 'secretbodyline', + ], + [ + [`--private-key=${begin}`, 'MIIEpAIBAAKCAQEAsecretbodyline1', end], + '["--[redacted]"]', + 'MIIE', + ], + [ + { + a: `${begin}\nAAAbody1`, + private_key: `body1tail\n${end}\n-----BEGIN EC PRIVATE KEY-----\nbody2head`, + c: 'BBBbody2\n-----END EC PRIVATE KEY-----', + d: 'public', + }, + '{"a":"[redacted]","d":"public"}', + 'BBBbody2', + ], + [ + { + a: `${begin}\nAAAfirstbody`, + private_key: `bbb\n${end}\n${begin}\nccc\n${end}`, + z: 'public', + }, + '{"a":"[redacted],"z":"public"}', + 'AAAfirstbody', + ], + [ + { private_key: `${begin}\nAAA\n${end}\n${begin}\nBBBhead`, rest: `CCCbody\n${end}` }, + '{"[redacted]}', + 'CCCbody', + ], + [ + { items: [{ private_key: `${begin}\nMIIEnested`, rest: `NESTEDbody\n${end}` }] }, + '{"items":[{"[redacted]}]}', + 'NESTEDbody', + ], + // A credential member that holds a whole block of its own, an object, or a scheme label. + [ + { + stdout: `${begin}\nMIIEpAIBAAKCAQEAQZXSECRETBODYLINE1abc\n[output truncated]`, + private_key: `${begin}\nMIIBOwIBAAJBAKj34\n${end}`, + }, + '{"stdout":"[redacted]}', + 'QZXSECRETBODYLINE1', + ], + [ + { + credentials: { type: 'service_account', private_key: '-----BEGIN PRIVATE KEY-----' }, + body: 'MIIEvQQZXSECRET\n-----END PRIVATE KEY-----\n', + }, + '{"credentials":"[redacted]"}', + 'QZXSECRET', + ], + [ + { token: `Bearer x ${begin}`, rest: `MIIEvQQZXSECRET\n${end}` }, + '{"[redacted]}', + 'QZXSECRET', + ], + ] as const) { + const bounded = await boundForModel(result, BIG, {}); + expect(bounded.summary, 'SECRETS_B8_CORE_SUMMARY_PEM').toBe(summary); + expect(bounded.summary.includes(secret), 'SECRETS_B8_CORE_SUMMARY_PEM').toBe(false); + } + expect( + ( + await boundForModel( + { private_key: `${begin}\nMIIEwhole\n${end}`, b: `see ${end} there` }, + BIG, + {}, + ) + ).summary, + 'SECRETS_B8_CORE_SUMMARY_PEM', + ).toBe(`{"[redacted],"b":"see ${end} there"}`); + // The journal and the display projections keep their whole mask. + expect( + projectSecretShapedValue({ private_key: `${begin}\nMIIEAAAA`, rest: 'x' }).value, + 'SECRETS_B8_CORE_SUMMARY_PEM', + ).toEqual({ private_key: '[redacted]', rest: 'x' }); + expect( + redactSecretShapedValue(['--private-key', begin]), + 'SECRETS_B8_CORE_SUMMARY_PEM', + ).toEqual(['--private-key', '[redacted]']); + }); + + it("reads a structured summary's JSON-escaped line breaks and tabs", async () => { + const jwt = + 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U'; + // JSON writes a line break or a tab in a member as two characters, which the summary reads as that + // separator for a credential or a scheme label after it and for a credential split by it. + for (const [result, summary, secret] of [ + [ + { stdout: 'keys:\nsk-proj-abcdefghijklmnopqrstuvwxyz', stderr: '', exitCode: 0 }, + '{"stdout":"keys:\\n[redacted]","stderr":"","exitCode":0}', + 'sk-proj-abc', + ], + [ + { stdout: 'ACCESSKEY\tAKIAIOSFODNN7EXAMPLE\tActive\n' }, + '{"stdout":"ACCESSKEY\\t[redacted]\\tActive\\n"}', + 'AKIAIOSFODNN7', + ], + [ + { stdout: 'one sk-proj-aaaaaaaaaaaaaaaaaaaaaaaa\r\nsk-proj-bbbbbbbbbbbbbbbbbbbbbbbb' }, + '{"stdout":"one [redacted]\\r\\n[redacted]"}', + 'bbbbbbbb', + ], + [{ stdout: 'AKIAABCDEFGH\nIJKLMNOP' }, '{"stdout":"[redacted]"}', 'IJKLMNOP'], + [ + { stdout: 'progress 50%\rsk-proj-abcdefghijklmnopqrstuvwxyz' }, + '{"stdout":"progress 50%\\r[redacted]"}', + 'sk-proj-abc', + ], + [ + { stdout: 'Bearer\nabcdefghijklmnop' }, + '{"stdout":"Bearer [redacted]"}', + 'abcdefghijklmnop', + ], + [ + { content: [{ type: 'text', text: `Token issued:\n${jwt}` }] }, + '{"content":[{"type":"text","text":"Token issued:\\n[redacted]"}]}', + 'eyJhbGciOiJIUzI1NiJ9', + ], + // The backslash before the letter may end a longer run: a backslash before a line break, or a JSON + // text inside a member. + [ + { stdout: 'x\\\nsk-proj-abcdefghijklmnopqrstuvwxyz' }, + '{"stdout":"x\\\\\\n[redacted]"}', + 'sk-proj-abc', + ], + [ + { stdout: '{"msg":"line1\\nsk-proj-abcdefghijklmnopqrstuvwxyz"}' }, + '{"stdout":"{\\"msg\\":\\"line1\\\\n[redacted]\\"}"}', + 'sk-proj-abc', + ], + ] as const) { + const bounded = await boundForModel(result, BIG, {}); + expect(bounded.summary, 'SECRETS_B8_CORE_SUMMARY_ESCAPES').toBe(summary); + expect(bounded.summary.includes(secret), 'SECRETS_B8_CORE_SUMMARY_ESCAPES').toBe(false); + } + // A path's backslashes before a letter add no mask without a credential after them. + expect( + (await boundForModel({ stdout: 'C:\\new\\token path\\nothing here' }, BIG, {})).summary, + 'SECRETS_B8_CORE_SUMMARY_ESCAPES', + ).toBe('{"stdout":"C:\\\\new\\\\token path\\\\nothing here"}'); + // The journal projection, which is compared rather than shown, reads no escape. + expect( + (await boundForModel({ msg: 'Use\nToken handling' }, BIG, {})).summary, + 'SECRETS_B8_CORE_SUMMARY_ESCAPES', + ).toBe('{"msg":"Use\\nToken [redacted]"}'); + expect( + redactSecretShapedText('{"msg":"Use\\nToken handling"}'), + 'SECRETS_B8_CORE_SUMMARY_ESCAPES', + ).toBe('{"msg":"Use\\nToken handling"}'); + }); +}); + +describe('PR95 review round 9: closing views, unwalked members, wrapped markers and linear views', () => { + it('closes a quoted value in the views the key rule read it in before its rules merged', async () => { + // A value whose line break a scheme label's token takes closes as the next pass reads it. It is also + // read without the key rule's own masks, so another key's value that takes its closing quote, or ends + // at its opening one, does not keep it open; and where it is first read, with the scheme labels' + // tokens filled too, so a token that takes its line break lets it close at its real quote. + for (const [input, expected] of [ + ['password="correct horse battery Bearer\nxyzxyzxy token="q" more', '[redacted] more'], + ['token\\":"x password="Bearer\nabcdefghij Wsecret12 Wsecret34" tail', '[redacted] tail'], + ["password:\\t'x api_key='Bearer\nabcdefghij Wsecret12 Wsecret34' tail", '[redacted] tail'], + [ + 'api_key=password="password="Bearer\nabcdefghij Wsecret12 Wsecret34" tail', + '[redacted] tail', + ], + ['{"secret": "hunter2 QZXsecondsecret Bearer\npassword": "x"}', '{"[redacted]}'], + ['token\\\':"secret="\'QZXJ7WKM2V Bearer\nabcdefgh"', '[redacted]'], + ['accesskey\\"="token:"Token\rabcd1234\\QZXJ7WKM2V"', '[redacted]'], + [ + 'authorization="g://&a=m3z1VvzzvmzZZ@gcom/v1/a\\"\\QZXJ7WKM2V BASIC\nqv8x~V6t"', + '[redacted]', + ], + // A filled scheme token only moves a closing that a reading already reaches: a value that no + // reading closes is decided once the next key's value, which opens at its first quote, is known. + ['Authorization: "Bearer\nAuthorization: ""Zq7Wk3Xp9Rm2"', '[redacted]'], + ['password="Bearer\npassword=""Zq7Wk3Xp9Rm2"', '[redacted]'], + // So does one behind an opaque value whose mask ends with a backslash, where the scheme views are read. + [ + `x ?sig=${'a'.repeat(26)}\\" Authorization: "Bearer\nAuthorization: ""Zq7Wk3Xp9Rm2"`, + 'x ?sig=[redacted]" [redacted]', + ], + ] as const) { + expect(redactSecretShapedText(input), 'SECRETS_B9_CORE_CLOSING_VIEWS').toBe(expected); + expect(projectSecretShapedValue({ a: input }), 'SECRETS_B9_CORE_CLOSING_VIEWS').toEqual({ + value: { a: expected }, + settled: true, + }); + expect((await boundForModel(input, BIG, {})).summary, 'SECRETS_B9_CORE_CLOSING_VIEWS').toBe( + expected, + ); + } + }); + + it('masks a summary value through a later closer when one of its readings stops at a line break', async () => { + // One reading closes the value at a quote an opaque value's mask unescapes, the other stops at the + // line break before its real closer; the summary masks through that closer. + for (const input of [ + 'password="see ?sig=https://u:p@cdn.example/abc\\" Wsecret12\nmore"', + 'password="?sig=https://u:p@cdn.example/abc\\"Wq0zkx\n"', + "passwd='?g=ttps://u:p@h.example/\\n\\'PLANT9\n'", + ]) { + expect((await boundForModel(input, BIG, {})).summary, 'SECRETS_B9_CORE_SUMMARY_STOPPED').toBe( + '[redacted]', + ); + } + }); + + it("reads a structured summary's quoted value across a member masked whole", async () => { + for (const [result, summary, secret] of [ + [ + { argv: ['mysql', "--password='correct", 'horse', "battery'", '-h', 'db'] }, + '{"argv":["mysql","--[redacted]","-h","db"]}', + 'battery', + ], + [ + { argv: ['mysql', "--password='correcthorse", "battery'", '-h', 'db'] }, + '{"argv":["mysql","--[redacted]","-h","db"]}', + 'battery', + ], + [ + { note: 'x', pin: "password='abc", b: "Wdef12' tail" }, + '{"note":"x","pin":"[redacted]" tail"}', + 'Wdef12', + ], + ] as const) { + const bounded = await boundForModel(result, BIG, {}); + expect(bounded.summary, 'SECRETS_B9_CORE_SUMMARY_MEMBERS').toBe(summary); + expect(bounded.summary.includes(secret), 'SECRETS_B9_CORE_SUMMARY_MEMBERS').toBe(false); + } + }); + + it('reads a private-key marker that a line break, a tab or a doubled space splits in a summary', async () => { + for (const [input, summary] of [ + [ + 'PRIVATE_KEY=-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0QZXSECRET\n-----END PRIVATE\nKEY-----', + '[redacted]', + ], + [ + 'token=-----BEGIN RSA PRIVATE KEY-----\nMIIEbodyQZXSECRET\n-----END\nPRIVATE KEY-----', + '[redacted]', + ], + [ + 'secret=-----BEGIN\tPRIVATE KEY-----\nMIIEbodyQZXSECRET\n-----END PRIVATE KEY-----', + '[redacted]', + ], + [ + 'PRIVATE_KEY=-----BEGIN RSA PRIVATE KEY-----\nMIIEbodyQZXSECRET\n-----END RSA PRIVATE KEY-----', + '[redacted]', + ], + [ + '-----BEGIN PRIVATE\nKEY-----\nMIIEbodyQZXSECRET\n-----END PRIVATE KEY-----apikey:"', + '[redacted]', + ], + [ + 'pem: -----BEGIN RSA PRIVATE KEY-----\nMIIEbodyQZXSECRET\n-----END RSA PRIVATE KEY----- done', + 'pem: [redacted] done', + ], + ] as const) { + expect((await boundForModel(input, BIG, {})).summary, 'SECRETS_B9_CORE_WRAPPED_MARKERS').toBe( + summary, + ); + } + // The journal projection, which is compared rather than shown, reads such a marker as before. + const wrapped = + 'pem: -----BEGIN RSA PRIVATE KEY-----\nMIIEbody\n-----END RSA PRIVATE KEY----- done'; + expect(redactSecretShapedText(wrapped), 'SECRETS_B9_CORE_WRAPPED_MARKERS').toBe(wrapped); + }); + + it("keeps the walk's masks in place for a result whose own text holds a placeholder", async () => { + // The summary marks each value the walk masks whole with a private-use placeholder; a result that + // already holds one is summarized from the walk's masks alone, and its own text stays as it is. + const result = { password: 'Qz7secretvalue', note: 'x\uE0000\uE001y' }; + expect( + (await boundForModel(result, BIG, {})).summary, + 'SECRETS_B9_CORE_SUMMARY_PLACEHOLDER', + ).toBe('{"[redacted],"note":"x\uE0000\uE001y"}'); + }); + + it("reads an opaque parameter in a URL username behind that username's zone", () => { + expect( + redactSecretShapedText(`https://ab&tok=${'A'.repeat(26)}:pw@host.example/x?q=1`), + 'SECRETS_B9_CORE_OPAQUE_NAMES', + ).toBe('https://[redacted]@host.example/x?q=1'); + }); + + // Each view of a quoted value is built once per pass, and a run of backslashes that many masks end in + // is read once. One shape per case, so that each keeps the case deadline under coverage instrumentation. + for (const [shape, input, summary] of [ + [ + 'nested keys share a run of backslashes', + `password="x\n${'token='.repeat(1_000_000)}${'\\'.repeat(1_000_000)} end`, + '[redacted]', + ], + [ + 'many values stop at a line break', + `${'password="abcdefgh\n'.repeat(400_000)}token=abcdef\\ end`, + '[redacted]', + ], + [ + 'many values precede a mask that ends in a backslash', + `${'"password": "abc", '.repeat(400_000)}\\"https://h.example/x?sig=${'a'.repeat(30)}\\" end`, + '"[redacted], "[redacted],', + ], + ] as const) { + it( + `keeps every projection linear where ${shape}`, + async () => { + expect(redactSecretShapedText(input).endsWith(' end'), 'SECRETS_B9_CORE_LINEAR_VIEWS').toBe( + true, + ); + expect( + projectSecretShapedValue({ content: input }).settled, + 'SECRETS_B9_CORE_LINEAR_VIEWS', + ).toBe(true); + expect( + (await boundForModel(input, BIG, {})).summary.startsWith(summary), + 'SECRETS_B9_CORE_LINEAR_VIEWS', + ).toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + } + + it( + "keeps a structured summary linear where a value's later closer is far away", + async () => { + const structured = { + stdout: `password="first\nsecond" ${'log line ok\n'.repeat(600_000)}echo "token=abcdef12"\n`, + }; + expect( + (await boundForModel(structured, BIG, {})).summary.startsWith( + '{"stdout":"[redacted] log line ok', + ), + 'SECRETS_B9_CORE_LINEAR_VIEWS', + ).toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); +}); + +describe('PR95 review round 10: a value whose escaped quote an earlier mask took, in a summary', () => { + it("opens a value whose escaped quote ends an earlier mask's backslashes in a summary", async () => { + // A structured result's JSON escapes each line break, so a query value at the end of a line runs on + // through the next line's key and takes the backslash of its value's escaped quote; the summary opens + // that value where the query value's mask keeps its trailing backslashes. + const env = + 'DATABASE_URL=postgres://db.example.com/app?sslmode=require\nPOSTGRES_PASSWORD="S3cr3tValue2024"\n'; + for (const [result, summary, secret] of [ + [ + { exitCode: 0, stdout: env, stderr: '', durationMs: 12 }, + '{"exitCode":0,"stdout":"DATABASE_URL=postgres://db.example.com/app?sslmode=[redacted]\\n","stderr":"","durationMs":12}', + 'S3cr3tValue2024', + ], + [ + { content: [{ type: 'text', text: env }] }, + '{"content":[{"type":"text","text":"DATABASE_URL=postgres://db.example.com/app?sslmode=[redacted]\\n"}]}', + 'S3cr3tValue2024', + ], + [ + { stdout: env.replaceAll('\n', '\r\n') }, + '{"stdout":"DATABASE_URL=postgres://db.example.com/app?sslmode=[redacted]\\r\\n"}', + 'S3cr3tValue2024', + ], + [ + { + stdout: + 'MONGO_URL=mongodb://h/db?retryWrites=true&w=majority\nMONGO_PASSWORD="M0ngoSecretPass"\n', + }, + '{"stdout":"MONGO_URL=mongodb://h/db?retryWrites=true&w=[redacted]\\n"}', + 'M0ngoSecretPass', + ], + [ + { stdout: 'GET /x?ssl_cert_reqs=required\tREDIS_PASSWORD="R3disSecretPass"' }, + '{"stdout":"GET /x?ssl_cert_reqs=[redacted]"}', + 'R3disSecretPass', + ], + ] as const) { + const bounded = await boundForModel(result, BIG, {}); + expect(bounded.summary, 'SECRETS_B10_CORE_SUMMARY_KEPT_OPENING').toBe(summary); + expect(bounded.summary.includes(secret), 'SECRETS_B10_CORE_SUMMARY_KEPT_OPENING').toBe(false); + } + // The journal projection, which is compared rather than shown, keeps its reading so that its digests do + // not change: the same JSON as a text argument still shows the value, a residual deferred-tasks.md records. + expect( + redactSecretShapedText(JSON.stringify({ stdout: env })), + 'SECRETS_B10_CORE_SUMMARY_KEPT_OPENING', + ).toBe( + '{"stdout":"DATABASE_URL=postgres://db.example.com/app?sslmode=[redacted]"S3cr3tValue2024\\"\\n"}', + ); + }); +}); diff --git a/packages/core/src/tools/bounding.ts b/packages/core/src/tools/bounding.ts index aa82048d..5ce62815 100644 --- a/packages/core/src/tools/bounding.ts +++ b/packages/core/src/tools/bounding.ts @@ -23,6 +23,7 @@ import { utf8ByteLength } from '@relavium/shared'; // while the IMPLEMENTATION lives in `@relavium/shared` — see its header for why it moved. export { utf8ByteLength }; +import { fixedPointOf, toFixedPoint } from './fixed-point.js'; import type { ToolHost, ToolResultLimits } from './types.js'; export interface BoundedResult { @@ -54,8 +55,9 @@ function countLines(text: string): number { * I3 gap where a `read_media` (or any) tool result carrying a `{ kind:'base64', data }` source would be * JSON-stringified into the `agent:tool_result.outputSummary` event / the spill / the over-cap preview — * a durable/event boundary the emit-time `deInlineMedia` choke point cannot catch, since it sees only the - * resulting flat string, not a structured media source). The MODEL-facing result value is left intact - * (the model still receives the bytes via the seam); only this text projection is redacted. Cycle-safe. + * resulting flat string, not a structured media source). Only this text projection is redacted; the + * model-facing value keeps the bytes while it fits the ceiling, and `boundForModel` counts the omitted bytes + * against that ceiling ({@link OmittedMedia}). Cycle-safe. */ /** A PLAIN object (Object/null prototype) — NOT a Date/RegExp/Map/Set/class instance, which JSON.stringify * renders natively and the walk must leave untouched (else a Date would collapse to `{}`). Narrows without an `as`. */ @@ -74,12 +76,28 @@ function isPlainObject(value: object): value is Record { * sees only a flat string). Display-only — the dispatch already ran on the real args. Cycle-safe. */ export function redactInlineMedia(value: unknown): unknown { - return redactInlineMediaForText(value, new WeakSet()); + return redactInlineMediaForText(value, new WeakSet(), { bytes: 0 }); } -function redactInlineMediaForText(value: unknown, seen: WeakSet): unknown { +/** + * The size of the inline media a text projection replaced with a marker. The model-facing value still holds + * those bytes — a builtin's media rides `attachMedia` instead (ADR-0089 §1), so media inside a dispatch + * result is ordinary text to the model and to the effect journal. A PR #96 review found a 30 MiB data URI + * measured as its 25-character marker, passed whole to the model and settled into `history.db`. + */ +interface OmittedMedia { + bytes: number; +} + +function redactInlineMediaForText( + value: unknown, + seen: WeakSet, + omitted: OmittedMedia, +): unknown { if (typeof value === 'string') { - return isBase64DataUri(value) ? '[base64 data URI omitted]' : value; + if (!isBase64DataUri(value)) return value; + omitted.bytes += utf8ByteLength(value); + return '[base64 data URI omitted]'; } if (typeof value !== 'object' || value === null) { return value; @@ -90,9 +108,10 @@ function redactInlineMediaForText(value: unknown, seen: WeakSet): unknow seen.add(value); try { if (Array.isArray(value)) { - return value.map((item) => redactInlineMediaForText(item, seen)); + return value.map((item) => redactInlineMediaForText(item, seen, omitted)); } if (isBinaryBuffer(value)) { + omitted.bytes += binaryByteLength(value); return '[binary buffer omitted]'; // a raw Uint8Array/ArrayBuffer — never JSON the decimal byte values } if (!isPlainObject(value)) { @@ -100,14 +119,16 @@ function redactInlineMediaForText(value: unknown, seen: WeakSet): unknow } if (isCanonicalBase64Source(value)) { const data = value['data']; + const base64Length = typeof data === 'string' ? data.length : 0; + omitted.bytes += base64Length; // Report the base64 CHARACTER length (≈1.33× the byte count), not "bytes" — observability only. - return { kind: 'base64', base64Length: typeof data === 'string' ? data.length : 0 }; + return { kind: 'base64', base64Length }; } const out: Record = {}; for (const [key, item] of Object.entries(value)) { // Preserve an own data key such as __proto__; assignment would invoke an inherited setter. Object.defineProperty(out, key, { - value: redactInlineMediaForText(item, seen), + value: redactInlineMediaForText(item, seen, omitted), enumerable: true, configurable: true, writable: true, @@ -120,6 +141,12 @@ function redactInlineMediaForText(value: unknown, seen: WeakSet): unknow } } +/** A binary buffer's byte length; every kind `isBinaryBuffer` admits has an own `byteLength`. */ +function binaryByteLength(value: object): number { + const length: unknown = Reflect.get(value, 'byteLength'); + return typeof length === 'number' ? length : 0; +} + // One definition builds both the complete matcher and its necessary-prefix preflight. Prefixes // have bounded width: testing their absence must never run the unbounded JWT/body alternatives. // One alternation still preserves outer spans when credential families overlap. Bounded lookaheads @@ -162,7 +189,46 @@ const JOINED_STANDALONE_CREDENTIAL = new RegExp(STANDALONE_CREDENTIAL.source.sli const SCHEME_CREDENTIAL = /\b(b[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|b[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c|t[\n\t]*o[\n\t]*k[\n\t]*e[\n\t]*n)\s+(?=[\w.~+/-]{8})[\w.~+/-]+={0,2}/gi; -type SecretSpanVisitor = (start: number, end: number, replacement: string) => void; +// A secret-ish key (bounded wrappers keep this ReDoS-safe) + `=`/`:`. The optional quote BEFORE the +// separator catches the JSON `"access_token":"…"` shape (an OAuth/egress response body), and its +// optional backslash the same shape one JSON string down (`\"access_token\":\"…\"`). Kept an +// analyzable LITERAL (not composed via `new RegExp`) so Sonar's static super-linear-runtime (S5852) +// check still covers this security-critical pattern; its keyword-alternation breadth is deliberate +// (and `apikey` is dropped — `api[_-]?key` already subsumes it). The value is read after it, in +// `visitSecretKeyValues`. +const SECRET_KEY = + /\b[\w-]{0,32}(?:password|passwd|secret|token|api[_-]?key|authorization|access[_-]?key|private[_-]?key|client[_-]?secret)[\w-]{0,16}(?:\\?["'])?\s*[=:]\s*/gi; + +/** + * One recognized span of the projection's input and the text that replaces it. Every rule reports + * its spans on the same text, and overlapping spans merge before anything is replaced + * ({@link replaceRanges}), so no rule's mask can erase the key name, scheme label, PEM marker or URL + * scheme another rule reads. + */ +interface SecretRange { + readonly start: number; + readonly end: number; + readonly replacement: string; +} + +/** + * A URL userinfo's context. `schemeStart` is where its identifying prefix starts: a removed LF/TAB + * anywhere from there on splits the credential, so a separator inside `scheme://` counts even though + * only the userinfo is replaced. Up to `ownsUntil` the userinfo owns its text: a key or query + * parameter there names the user, not a credential of its own. + */ +interface UrlUserinfo { + readonly schemeStart: number; + readonly ownsUntil: number; +} + +/** Reports one recognized span; `url` is set for a URL's userinfo. */ +type SecretSpanVisitor = ( + start: number, + end: number, + replacement: string, + url?: UrlUserinfo, +) => void; /** * A failed JWT probe must not rescan an admitted-size token run at every original boundary, @@ -244,18 +310,6 @@ function visitJoinedStandaloneCredentials( } } -function redactStandaloneCredentials(text: string): string { - const parts: string[] = []; - let copied = 0; - visitJoinedStandaloneCredentials(text, text, (start, end, replacement) => { - parts.push(text.slice(copied, start), replacement); - copied = end; - }); - if (parts.length === 0) return text; - parts.push(text.slice(copied)); - return parts.join(''); -} - /** * The finite header-label literal is the sole joined-header authority. Check ORIGINAL boundaries * before matching, so an embedded x-api-key/x-goog-api-key cannot consume an inner api-key start. @@ -290,23 +344,25 @@ function visitJoinedHeaderLabels(original: string, joined: string, mask: SecretS } /** - * LF/TAB may split finite credential shapes. Map recognized spans back onto the original text; - * ordinary spaces remain boundaries and public text is never globally normalized for identity. - * Token-character continuations after LF/TAB are intentionally part of a recognized credential. - * The two cursors advance monotonically, with no per-character offset array or recursive matcher. + * LF/TAB may split finite credential shapes. Map the spans recognized in the joined view back onto + * the original text and report the ones a removed separator actually splits; ordinary spaces remain + * boundaries and public text is never globally normalized for identity. Token-character + * continuations after LF/TAB are intentionally part of a recognized credential. The two cursors + * advance monotonically, with no per-character offset array or recursive matcher. */ -function redactJoinedCredentialSpans( +function visitSplitSpans( text: string, visit: (joined: string, mask: SecretSpanVisitor) => void, + report: SecretSpanVisitor, boundary?: RegExp, -): string { - if (!/[\n\t]/.test(text)) return text; + // URL userinfo owns a separator at either edge: one right after the scheme or before the @ splits it. + ownsEdges = false, +): void { + if (!/[\n\t]/.test(text)) return; const joined = text.replace(/[\n\t]+/g, ''); - const parts: string[] = []; const separators = /[\n\t]+/g; let next = separators.exec(text); let removed = 0; - let copied = 0; function advance(target: number, includeAtBoundary = false): number { while ( next !== null && @@ -317,37 +373,140 @@ function redactJoinedCredentialSpans( } return target + removed; } - visit(joined, (start, end, replacement) => { - const originalStart = advance(start, true); - const originalEnd = advance(end); + visit(joined, (start, end, replacement, url) => { + // A separator right before a URL's scheme is outside the URL, so it splits nothing. + const schemeStart = url === undefined ? undefined : advance(url.schemeStart, true); + const originalStart = advance(start, !ownsEdges); + const ownsUntil = url === undefined ? undefined : advance(url.ownsUntil, ownsEdges); + const originalEnd = advance(end, ownsEdges); if (boundary?.test(text[originalStart - 1] ?? '')) return; - if (!/[\n\t]/.test(text.slice(originalStart, originalEnd))) return; - parts.push(text.slice(copied, originalStart), replacement); - copied = originalEnd; + if (!/[\n\t]/.test(text.slice(schemeStart ?? originalStart, originalEnd))) return; + // A split userinfo owns nothing past its first removed separator: the next line may hold a key + // of its own (`https://\nDB_PASSWORD=P@ss`). + const split = + ownsUntil === undefined ? -1 : text.slice(originalStart, ownsUntil).search(/[\n\t]/); + report( + originalStart, + originalEnd, + replacement, + schemeStart === undefined || ownsUntil === undefined + ? undefined + : { schemeStart, ownsUntil: split < 0 ? ownsUntil : originalStart + split }, + ); }); - if (parts.length === 0) return text; - parts.push(text.slice(copied)); - return parts.join(''); } -function redactSplitCredentials(text: string): string { - if (!/[\n\t]/.test(text)) return text; - const urls = redactJoinedCredentialSpans(text, visitUrlUserinfo); - const keys = redactJoinedCredentialSpans( - urls, +/** LF/TAB-split URL userinfo and standalone credentials (`visitSplitSpans`). */ +function visitSplitCredentials(text: string, report: SecretSpanVisitor): void { + visitSplitSpans(text, visitUrlUserinfo, report, undefined, true); + visitSplitSpans( + text, (joined, mask) => { - visitJoinedStandaloneCredentials(urls, joined, mask); + visitJoinedStandaloneCredentials(text, joined, mask); }, + report, /\w/, ); - const schemes = keys.replace(SCHEME_CREDENTIAL, '$1 [redacted]'); - return redactJoinedCredentialSpans( - schemes, - (joined, mask) => { - visitJoinedHeaderLabels(schemes, joined, mask); - }, - /[A-Za-z0-9_-]/, - ); +} + +/** + * The text as a rule reads it right after the given credentials: each one is filled like an applied + * mask, so a label or credential glued to its end still starts at a word boundary. + */ +function afterMasks(text: string, ranges: readonly SecretRange[]): string { + return fillRanges(text, ranges); +} + +/** + * Join every LF/TAB-split header label, so the key=value rule reads it like an unsplit one. A label + * starts at a word boundary, and a credential or a PEM block right before it is one (`afterMasks`). + */ +function joinSplitHeaderLabels(text: string): string { + // A label ends in ':' (visitJoinedHeaderLabels) and no fill adds one, so a text without one has none. + if (!/[\n\t]/.test(text) || !text.includes(':')) return text; + const masks: SecretRange[] = []; + const firstLines: SecretRange[] = []; + const mask: SecretSpanVisitor = (start, end, replacement) => { + masks.push({ start, end, replacement }); + firstLines.push({ start, end, replacement }); + }; + // A split credential's LF/TAB continuation can take in a label at the start of the next line + // (`sk-…\napi-k\tey: `), so labels are read a second time with each split credential filled only + // up to its first removed separator. The credentials' own masks are unchanged. + visitSplitCredentials(text, (start, end, replacement) => { + masks.push({ start, end, replacement }); + const split = text.slice(start, end).search(/[\n\t]/); + firstLines.push({ start, end: split < 0 ? end : start + split, replacement }); + }); + visitPemBlocks(text, mask); + visitJoinedStandaloneCredentials(text, text, mask); + const labels: SecretRange[] = []; + for (const boundaries of new Set([afterMasks(text, masks), afterMasks(text, firstLines)])) { + visitSplitSpans( + boundaries, + (joined, label) => { + visitJoinedHeaderLabels(boundaries, joined, label); + }, + (start, end, replacement) => { + labels.push({ start, end, replacement }); + }, + /[A-Za-z0-9_-]/, + ); + } + return replaceRanges(text, labels); +} + +/** + * Replace each run of overlapping ranges once, with the replacement of the range that starts it. + * Ranges that only touch stay separate runs, so a key=value mask and the scheme label's token after + * it read `[redacted] [redacted]`. + */ +function replaceRanges(text: string, ranges: readonly SecretRange[]): string { + if (ranges.length === 0) return text; + const ordered = [...ranges].sort((left, right) => left.start - right.start); + const parts: string[] = []; + let copied = 0; + let run: { start: number; end: number; replacement: string } | undefined; + for (const range of ordered) { + if (run !== undefined && range.start < run.end) { + run.end = Math.max(run.end, range.end); + continue; + } + if (run !== undefined) { + parts.push(text.slice(copied, run.start), run.replacement); + copied = run.end; + } + run = { start: range.start, end: range.end, replacement: range.replacement }; + } + if (run === undefined) return text; + parts.push(text.slice(copied, run.start), run.replacement, text.slice(run.end)); + return parts.join(''); +} + +/** + * The same text with each run of overlapping ranges replaced by a run of the same length that reads + * like a mask (`[xx…x]`): no line break, quote, delimiter or key inside, and a word boundary at either + * end. Offsets stay valid, so a rule that reads it reports ranges of the original text. + */ +function fillRanges(text: string, ranges: readonly SecretRange[]): string { + if (ranges.length === 0) return text; + const runs: { start: number; end: number }[] = []; + for (const range of [...ranges].sort((left, right) => left.start - right.start)) { + if (range.end <= range.start) continue; + const last = runs.at(-1); + if (last !== undefined && range.start < last.end) last.end = Math.max(last.end, range.end); + else runs.push({ start: range.start, end: range.end }); + } + if (runs.length === 0) return text; + const parts: string[] = []; + let copied = 0; + for (const run of runs) { + const length = run.end - run.start; + parts.push(text.slice(copied, run.start), length < 2 ? '[' : `[${'x'.repeat(length - 2)}]`); + copied = run.end; + } + parts.push(text.slice(copied)); + return parts.join(''); } /** @@ -363,45 +522,223 @@ function redactSplitCredentials(text: string): string { * no-backtracking-RegExp / no-ReDoS posture. */ export function redactSecretShapedText(text: string): string { - const projected = redactOpaqueQueryValues(redactUrlUserinfo(redactSplitCredentials(text))) - // A PEM private-key block (multi-line, space-separated markers the `private_key` key-pattern can't see). - // The body span is bounded (`{0,20000}?`, lazy) so an unterminated block can't drive an unbounded scan. - .replace( - /-----BEGIN [A-Z0-9 ]{0,40}PRIVATE KEY-----[\s\S]{0,20000}?-----END [A-Z0-9 ]{0,40}PRIVATE KEY-----/g, - '[redacted]', - ) - // An `Authorization`-style scheme + its token: `Bearer ` / `Basic ` / `Token `. The token class - // `[\w.~+/-]` is base64url + padding (`\w` folds `[A-Za-z0-9_]`). - // For every open-ended token minimum below, a bounded lookahead enforces the same floor before - // one simple greedy run. V8 can exhaust its RegExp stack on an admitted-scale `{N,}` run; the - // equivalent `(?=class{N})class+` form keeps the whole value instead of clipping a credential. - .replace(SCHEME_CREDENTIAL, '$1 [redacted]') - // A secret-ish key (bounded wrappers keep this ReDoS-safe) + `=`/`:` + its value. The optional `["']?` - // BEFORE the separator catches the JSON `"access_token":"…"` shape (an OAuth/egress response body). The - // value has QUOTED and UNQUOTED branches. A quoted value ends at the first UNESCAPED matching quote, - // including JSON's escaped quotes/backslashes; otherwise a valid password such as `a\"secret-tail` - // left its tail exposed. The two alternatives inside each repetition are disjoint: a backslash - // consumes exactly the following non-newline character, while the ordinary class excludes it. - // No ambiguous nested repetition/backref is needed. Unquoted values retain their delimiter and - // minimum-length policy. Kept a single analyzable LITERAL (not composed via `new RegExp`) so Sonar's static - // super-linear-runtime (S5852) check still covers this security-critical pattern; its keyword-alternation - // breadth is deliberate (and `apikey` is dropped — `api[_-]?key` already subsumes it). - .replace( - /\b[\w-]{0,32}(?:password|passwd|secret|token|api[_-]?key|authorization|access[_-]?key|private[_-]?key|client[_-]?secret)[\w-]{0,16}["']?\s*[=:]\s*(?:"(?:[^"\\\r\n]|\\[^\r\n])*"|'(?:[^'\\\r\n]|\\[^\r\n])*'|(?=[^\s"',;&]{6})[^\s"',;&]+)/gi, - '[redacted]', - ); + // One pass can change the context a later pass decides on, so the projection repeats until it is + // stable and fails closed otherwise: its output is stable under another projection. + return toFixedPoint(redactSecretShapedOnce, text, '[redacted]'); +} + +/** + * The summary's projection (`makeSummary`). A summary is shown, never compared, so it decides three things + * differently. In the key rule (`visitSecretKeyValues`), a quoted value that stops at a line break is masked + * through a later quote that could close it, else to the end of its line, and one that reaches the end of + * its string or of the text without closing is masked to that end: left to the one-line pass, such a value + * lost its key to a mask before it (`token='a password="first\nsecond" done`), and JSON escaping hid it in a + * structured result; and a value whose escaped opening quote ends an earlier mask's backslashes opens where + * that mask keeps them. It also reads private-key markers that whitespace wraps (`visitPemBlocks`) and the + * rules a separator decides behind escaped separators (`visitEscapedSeparators`). + */ +function redactSecretShapedSummaryText(text: string): string { + return toFixedPoint((value) => redactSecretShapedOnce(value, true), text, '[redacted]'); +} + +function redactSecretShapedOnce(text: string, throughCloser = false): string { + const source = joinSplitHeaderLabels(text); + return replaceRanges(source, secretRanges(source, throughCloser)); +} + +/** + * Every span the projection masks in `text`; each rule is described at its visitor. `throughCloser` selects + * the summary's decisions (`redactSecretShapedSummaryText`). + */ +function secretRanges(text: string, throughCloser: boolean): readonly SecretRange[] { + const ranges: SecretRange[] = []; + // Ranges merge instead of erasing context, yet some reads see the text as already masked: a key or + // parameter in a URL userinfo's own text (`UrlUserinfo.ownsUntil`) is that userinfo's, a key's + // value runs through every span an earlier rule masks, line breaks included, and a scheme label + // may start right after a credential, as each would with the masks applied. + const owned: SecretRange[] = []; + const masked: SecretRange[] = []; + const report: SecretSpanVisitor = (start, end, replacement, url) => { + const range = { start, end, replacement }; + ranges.push(range); + masked.push(range); + if (url !== undefined) owned.push({ start, end: url.ownsUntil, replacement }); + }; + const record: SecretSpanVisitor = (start, end, replacement) => { + ranges.push({ start, end, replacement }); + }; + // The key rule's own masks, which a value that does not close is also read without. + const keyValues = new Set(); + const recordValue: SecretSpanVisitor = (start, end, replacement) => { + const range = { start, end, replacement }; + ranges.push(range); + keyValues.add(range); + }; + const schemes: SecretRange[] = []; + visitSplitCredentials(text, report); + visitUrlUserinfo(text, report); + const names = fillRanges(text, owned); + visitOpaqueQueryValues(names, text, report); + visitPemBlocks(text, report, throughCloser); + // No value runs through a scheme label's token: `authorization: Bearer ` keeps both masks. + visitSchemeCredentials(text, (start, end, replacement) => { + record(start, end, replacement); + schemes.push({ start, end, replacement }); + }); + const values = fillRanges(text, masked); + const views = quoteViews(text, masked, schemes); + const unclosed = visitSecretKeyValues(names, values, views, recordValue, throughCloser); // Well-known standalone credential shapes (OpenAI, Stripe, AWS incl. STS ASIA/ABIA, GitHub token + PAT, // GitLab PAT, Slack, Google API + OAuth, HuggingFace, npm, JWT). Kept as ONE alternation ON PURPOSE: a - // single leftmost-longest scan is required so two ADJACENT / EMBEDDED credential shapes (e.g. a JWT whose - // signature segment contains an `sk-` run, or a `xox…-glpat-…` run) are covered by the outer greedy match - // as a single span. A per-family MULTI-PASS split is NOT equivalent — an earlier pass inserting `[redacted]` - // (a non-word `[`) truncates a later family's greedy match and can leave a trailing secret-shaped substring - // EXPOSED (a real redaction regression, pinned in bounding.test.ts). The canonical fragments therefore - // construct one complete alternation, with a separate bounded prefix-absence preflight. Failed probes - // use the bounded preview before the full greedy matcher; overlap and admitted-size tests pin that path. + // single leftmost-longest scan covers two ADJACENT / EMBEDDED credential shapes (e.g. a JWT whose + // signature segment contains an `sk-` run, or a `xox…-glpat-…` run) with the outer greedy match as a + // single span (pinned in bounding.test.ts). The canonical fragments therefore construct one complete + // alternation, with a separate bounded prefix-absence preflight. Failed probes use the bounded + // preview before the full greedy matcher; overlap and admitted-size tests pin that path. // `\w` / `[\w-]` fold only the classes that are EXACTLY `[A-Za-z0-9_]` / `[A-Za-z0-9_-]`; // the tighter `[A-Za-z0-9]` / `[A-Za-z0-9-]` families keep their narrower class (no `_`). - return redactStandaloneCredentials(projected); + const standalone: SecretRange[] = []; + visitJoinedStandaloneCredentials(text, text, (start, end, replacement) => { + record(start, end, replacement); + standalone.push({ start, end, replacement }); + }); + // A scheme label glued to the end of another credential starts one of its own, even where a key's + // value runs on over it: labels are read again with every mask filled but key values and scheme + // tokens (`afterMasks`). Anything else glued to a mask is read by the projection's next pass. With no + // mask, that view is the text the labels were already read in. + if (masked.length > 0 || standalone.length > 0) { + visitSchemeCredentials(afterMasks(text, [...masked, ...standalone]), record); + } + // A key's quoted value that does not close on its line is decided last, when every other range of the + // pass is known (`visitUnclosedValues`). + if (unclosed.length > 0) visitUnclosedValues(text, values, ranges, keyValues, unclosed, record); + // A summary also reads the line breaks and tabs that JSON escaping hides (`visitEscapedSeparators`). + if (throughCloser) visitEscapedSeparators(text, record); + return ranges; +} + +/** + * The ranges without the run of backslashes each ends with in `text`, or `ranges` itself when none ends + * with one. Filled, they leave a quote right after a mask the escape it has in the text (`laterClosing`). + * Ranges that share a backslash run read it once: each run's start is remembered at every end it serves. + */ +function keepingEscapes(text: string, ranges: readonly SecretRange[]): readonly SecretRange[] { + const runStarts = new Map(); + const runStart = (end: number): number => { + let index = end; + while (text[index - 1] === '\\') { + const known = runStarts.get(index); + if (known !== undefined) { + index = known; + break; + } + index -= 1; + } + for (let position = end; position > index && text[position - 1] === '\\'; position -= 1) { + if (runStarts.has(position)) break; + runStarts.set(position, index); + } + return index; + }; + let changed = false; + const kept = ranges.map((range) => { + if (text[range.end - 1] !== '\\') return range; + changed = true; + return { ...range, end: Math.max(runStart(range.end), range.start) }; + }); + return changed ? kept : ranges; +} + +/** Where a quoted value ends (`quotedValueEnd`). */ +type QuotedValueEnd = ReturnType; + +/** + * Of two readings of a quoted value, the one that closes it later; a reading that closes beats one that + * does not. A quoted value is read in several views of its text (`quoteViews`, `visitUnclosedValues`): + * with the masks before or inside it filled, as the next pass reads them, and with each mask keeping the + * backslashes it ends with, as the text has them; and with the scheme labels' tokens filled too, as the + * key rule read them before this projection merged its rules. A mask can take the backslash that escapes a + * later quote, which then closes the value early in one reading + * (`password="Bearer\nabcdefgh token=xyz\"tail" more`), or the backslash before a quote that is the + * value's only closer (`password="say \"…?sig=<24 or more>\"`), which closes it in that reading alone; a + * scheme token can take the line break before the value's closer. The latest closing of all of them + * masks each of these. + */ +function laterClosing(first: QuotedValueEnd, second: QuotedValueEnd): QuotedValueEnd { + return second.closed && (!first.closed || second.end > first.end) ? second : first; +} + +/** + * The views a quoted value is also read in beside `values` (`laterClosing`), when one of the earlier masks + * ends with a backslash: `plain`, which is `escapes`, those masks keeping the backslashes they end with; + * and `schemed`, `escapes` with the scheme labels' tokens filled too, when there is one, which only moves + * a closing that a plain reading reaches. With no such backslash every view reads like `values`, so none + * is built: a scheme token cannot move a closing that `values` itself reaches on its line. `escapes` also + * gives the summary's closers their second reading (`throughNearestCloser`). + */ +interface QuoteViews { + readonly escapes: string | undefined; + readonly plain: readonly string[]; + readonly schemed: readonly string[]; +} + +function quoteViews( + text: string, + masked: readonly SecretRange[], + schemes: readonly SecretRange[], +): QuoteViews { + const kept = keepingEscapes(text, masked); + if (kept === masked) return { escapes: undefined, plain: [], schemed: [] }; + const escapes = fillRanges(text, kept); + const schemed = schemes.length > 0 ? [fillRanges(text, [...kept, ...schemes])] : []; + return { escapes, plain: [escapes], schemed }; +} + +/** + * A summary's second reading, for the line breaks and tabs that JSON escaping hides: a structured result's + * JSON writes each as two characters, so a credential or a scheme label right after one has no boundary, + * and a credential split by one is not joined. The rules that a separator decides read a view of the + * same length in which each such escape is its separator twice (`escapedSeparatorView`); their ranges + * merge with those of the JSON reading, which reads a quoted value or a PEM block across the escapes. + */ +function visitEscapedSeparators(text: string, report: SecretSpanVisitor): void { + const view = escapedSeparatorView(text); + if (view === text) return; + visitSplitCredentials(view, report); + visitJoinedStandaloneCredentials(view, view, report); + visitSchemeCredentials(view, report); +} + +/** The separator each escape letter stands for. */ +const ESCAPED_SEPARATORS: ReadonlyMap = new Map([ + ['n', '\n'], + ['r', '\r'], + ['t', '\t'], +]); + +/** + * `text` with each escaped line feed, carriage return or tab (`\n`, `\r`, `\t`) replaced by that separator + * twice, so offsets stay valid. The backslash before the letter may end a longer run, as in a JSON text + * inside a member (`\\n`) or a path (`C:\new`): the view only adds masks, and a mask needs a credential + * after the separator. Each run of backslashes is read once. + */ +function escapedSeparatorView(text: string): string { + const parts: string[] = []; + let copied = 0; + let index = text.indexOf('\\'); + while (index >= 0) { + let end = index; + while (text[end] === '\\') end += 1; + const separator = ESCAPED_SEPARATORS.get(text[end] ?? ''); + if (separator !== undefined) { + parts.push(text.slice(copied, end - 1), separator, separator); + copied = end + 1; + } + index = text.indexOf('\\', end); + } + if (parts.length === 0) return text; + parts.push(text.slice(copied)); + return parts.join(''); } /** @@ -410,41 +747,456 @@ export function redactSecretShapedText(text: string): string { * mask the whole value: a fixed-length prefix leaves a permanent credential equality oracle. * Scan delimiters separately instead of matching a multi-megabyte repeated value in V8's * RegExp stack. Both cursors advance; short unrecognized values add only bounded overlap. + * Parameters are read in `names`, which hides each URL userinfo's username (`UrlUserinfo.ownsUntil`); + * a value is measured in `text`, so a mask inside it neither ends nor extends it. */ -function redactOpaqueQueryValues(text: string): string { - if (!text.includes('?') && !text.includes('&')) return text; +function visitOpaqueQueryValues(names: string, text: string, mask: SecretSpanVisitor): void { + if (!names.includes('?') && !names.includes('&')) return; const prefix = /[?&][\w.-]{1,64}=/g; const delimiter = /[\s&#"',}\]]/g; - const parts: string[] = []; - let cursor = 0; - for (let match = prefix.exec(text); match !== null; match = prefix.exec(text)) { + for (let match = prefix.exec(names); match !== null; match = prefix.exec(names)) { const valueStart = prefix.lastIndex; delimiter.lastIndex = valueStart; const valueEnd = delimiter.exec(text)?.index ?? text.length; if (valueEnd - valueStart < 24) continue; - parts.push(text.slice(cursor, valueStart), '[redacted]'); - cursor = valueEnd; + mask(valueStart, valueEnd, '[redacted]'); prefix.lastIndex = valueEnd; } - if (parts.length === 0) return text; - parts.push(text.slice(cursor)); - return parts.join(''); +} + +/** A PEM block's body is bounded, so an unterminated block cannot drive an unbounded scan. */ +const PEM_BODY_MAX = 20_000; + +/** + * A PEM private-key block (multi-line, space-separated markers the `private_key` key-pattern can't see): + * a BEGIN marker and the first END marker that starts at most PEM_BODY_MAX characters after it, which + * is what one regex with a lazy bounded body finds. That regex rescanned the whole bounded body after + * every unterminated BEGIN marker; here one END search serves every BEGIN marker before the END it + * found, so the scan stays linear. A summary also reads a marker whose words a line break, a tab or more + * than one space separates (`wrapped`), as a block wrapped at a fixed width can split one: its pass + * before normalization must not mask part of a marker that its one-line pass would pair. + */ +function visitPemBlocks(text: string, mask: SecretSpanVisitor, wrapped = false): void { + if (!text.includes('-----BEGIN')) return; + const begin = wrapped + ? /-----BEGIN\s[A-Z0-9\s]{0,40}PRIVATE\s{1,8}KEY-----/g + : /-----BEGIN [A-Z0-9 ]{0,40}PRIVATE KEY-----/g; + const end = wrapped + ? /-----END\s[A-Z0-9\s]{0,40}PRIVATE\s{1,8}KEY-----/g + : /-----END [A-Z0-9 ]{0,40}PRIVATE KEY-----/g; + let found: RegExpExecArray | null | undefined; + for (let marker = begin.exec(text); marker !== null; marker = begin.exec(text)) { + const body = begin.lastIndex; + if (found === undefined || (found !== null && found.index < body)) { + end.lastIndex = body; + found = end.exec(text); + } + if (found === null) return; + if (found.index - body > PEM_BODY_MAX) { + // Like the regex, try the next start: a later BEGIN marker may be close enough to this END. + begin.lastIndex = marker.index + 1; + continue; + } + const blockEnd = found.index + found[0].length; + mask(marker.index, blockEnd, '[redacted]'); + begin.lastIndex = blockEnd; + } +} + +/** + * An `Authorization`-style scheme + its token: `Bearer ` / `Basic ` / `Token `. The token class + * `[\w.~+/-]` is base64url + padding (`\w` folds `[A-Za-z0-9_]`). The range starts after the label, so + * the label stays and one space separates it from the mask: `Bearer [redacted]`. The scan resumes + * after the label rather than after the token: a label inside the token (`Bearer a/Bearer `) starts + * a credential that reaches past this one. + * For every open-ended token minimum below, a bounded lookahead enforces the same floor before + * one simple greedy run. V8 can exhaust its RegExp stack on an admitted-scale `{N,}` run; the + * equivalent `(?=class{N})class+` form keeps the whole value instead of clipping a credential. + */ +function visitSchemeCredentials(text: string, mask: SecretSpanVisitor): void { + // Whitespace follows every label, so a text without any has no scheme credential. + if (!/\s/.test(text)) return; + const scheme = new RegExp(SCHEME_CREDENTIAL); + for (let match = scheme.exec(text); match !== null; match = scheme.exec(text)) { + const labelEnd = match.index + (match[1]?.length ?? 0); + mask(labelEnd, scheme.lastIndex, ' [redacted]'); + scheme.lastIndex = labelEnd; + } +} + +/** + * Where the quoted value that opens at `start` ends: just past its closing quote on its line (`closed`), + * at a `lineBreak` it does not cross (it may still close on a later line), or, without closing, at the + * end of the string around it or of the text. A value opened by a plain quote (`"…"`, `'…'`) closes at + * the first UNESCAPED matching quote: a backslash consumes exactly the next character, as in JSON's + * escaped quotes and backslashes (otherwise `a\"secret-tail` left its tail exposed). A value opened by an + * escaped quote (`\"…\"`, as JSON inside a JSON string writes it) is read one decoding level down: each + * backslash pair is one character, an escaped backslash escapes the next one, an escaped `\n`/`\r` is a + * line break, and only its escaped quote closes it. The raw quote of its kind or the raw line break, + * behind a backslash or not, ends the string around it, which the value cannot cross. A raw quote of the + * other kind does not end it: an apostrophe is ordinary text in a JSON string. A plain scan, because a + * RegExp repeat over the value keeps backtracking state per character and exhausts V8's stack on an + * admitted-size value. + */ +function quotedValueEnd( + text: string, + start: number, +): { readonly end: number; readonly closed: boolean; readonly lineBreak: boolean } { + const escaped = text[start] === '\\'; + const quote = text[escaped ? start + 1 : start]; + // A raw line break stops a plain value; one JSON string down it ends the string around the value. + const stop = (end: number): { end: number; closed: boolean; lineBreak: boolean } => ({ + end, + closed: false, + lineBreak: !escaped, + }); + let escapeNext = false; + for (let index = escaped ? start + 2 : start + 1; index < text.length; index += 1) { + const char = text[index]; + if (char === '\r' || char === '\n') return stop(index); + if (char === quote) + return escaped ? stop(index) : { end: index + 1, closed: true, lineBreak: false }; + if (char !== '\\') { + escapeNext = false; + continue; + } + const next = text[index + 1]; + if (next === undefined) break; + if (next === '\n' || next === '\r') return stop(index + 1); + if (escaped && (next === 'n' || next === 'r')) { + return { end: index, closed: false, lineBreak: true }; + } + index += 1; + if (!escaped) continue; + if (escapeNext) escapeNext = false; + else if (next === quote) return { end: index + 1, closed: true, lineBreak: false }; + else escapeNext = next === '\\'; + } + return { end: text.length, closed: false, lineBreak: false }; +} + +/** Where an unquoted value, and the run of a quoted one that does not close, ends: whitespace, a quote, `,`, `;`, `&`. */ +function valueDelimiter(): RegExp { + return /[\s"',;&]/g; +} + +/** + * A secret-ish key (`SECRET_KEY`) and its value: a whole quoted string (`quotedValueEnd`, also one JSON + * string down), or an unquoted run of at least six characters up to whitespace, a quote, `,`, `;` or + * `&`. A quoted value that does not close on its line, or one JSON string down before the end of the + * string around it, is returned for the journal projection to decide once the pass's other ranges are + * known (`visitUnclosedValues`). The summary's projection (`throughCloser`) masks a value that one of its + * readings stops at a line break through a later quote instead (`throughNearestCloser`), and through + * the closing of a reading that closes too; it also opens a value whose escaped opening quote ends an + * earlier mask's backslashes in the views that keep them (`?sslmode=require\nPOSTGRES_PASSWORD=\"…\"` in a + * structured result, where the query value runs on through the next line's key). Keys are read in + * `keys`, which hides each URL userinfo's username (`UrlUserinfo.ownsUntil`): a key there names the user + * (`https://x-access-token:@github.com/o/r`), and its value would run on through the public host and + * path. Values are read in `values`, where every earlier mask is one run, as an applied mask would be, + * and a quoted value also in the other `views` (`quoteViews`); it closes at the latest of them + * (`laterClosing`). The key scan resumes at each value rather than after it, so a key inside a value is + * read too: `token=a>password: ` has a second value past the first one's end. One delimiter scan + * serves every unquoted value, and a quoted value is scanned no further than its line in each view: a + * nested quoted value can only open at the closing quote of the one around it, and one that does not + * close leaves no quote of its kind on its line, so the scan stays linear. + */ +function visitSecretKeyValues( + keys: string, + values: string, + views: QuoteViews, + report: SecretSpanVisitor, + throughCloser: boolean, +): readonly UnclosedValue[] { + // Every key ends in `=` or `:`, so names without either have no value to read. + if (!keys.includes('=') && !keys.includes(':')) return []; + const key = new RegExp(SECRET_KEY); + const delimiter = valueDelimiter(); + let delimiterAt = -1; + const masked: SecretRange[] = []; + const open: OpenValue[] = []; + const unclosed: UnclosedValue[] = []; + const mask = (start: number, end: number): void => { + masked.push({ start, end, replacement: '[redacted]' }); + report(start, end, '[redacted]'); + }; + for (let match = key.exec(keys); match !== null; match = key.exec(keys)) { + const valueStart = escapedValueStart(values, key.lastIndex); + // In a summary, a value whose escaped opening quote ends an earlier mask's backslashes (a query value + // that ran on through a JSON-escaped line break and the next line's key) opens where that mask keeps + // them, and is read only in the views that keep them. + const kept = + throughCloser && values[valueStart] !== '\\' && views.escapes?.[valueStart] === '\\' + ? views.escapes + : undefined; + const opener = kept ?? values; + const escaped = opener[valueStart] === '\\'; + const opening = escaped ? opener[valueStart + 1] : opener[valueStart]; + if (opening === '"' || opening === "'") { + const plain = views.plain.map((view) => quotedValueEnd(view, valueStart)); + const readings = kept === undefined ? [quotedValueEnd(values, valueStart), ...plain] : plain; + // A scheme label's token can take the line break before a closer that a reading reaches early; a + // value that no reading closes is decided once every range of the pass is known instead. + // `readings` holds the value's own reading first; the fallback states that here. + const first = readings + .slice(1) + .reduce( + (later, reading) => laterClosing(later, reading), + readings[0] ?? quotedValueEnd(opener, valueStart), + ); + const read = first.closed + ? views.schemed + .map((view) => quotedValueEnd(view, valueStart)) + .reduce((later, reading) => laterClosing(later, reading), first) + : first; + const kind = escaped ? `\\${opening}` : opening; + if (!throughCloser) { + if (read.closed) mask(match.index, read.end); + else unclosed.push({ start: match.index, quote: valueStart, end: read.end }); + continue; + } + const stopped = readings.find((reading) => !reading.closed && reading.lineBreak); + if (read.closed || stopped === undefined) mask(match.index, read.end); + if (stopped !== undefined) open.push({ start: match.index, end: stopped.end, quote: kind }); + continue; + } + if (delimiterAt < valueStart) { + delimiter.lastIndex = valueStart; + delimiterAt = delimiter.exec(values)?.index ?? values.length; + } + if (delimiterAt - valueStart >= 6) mask(match.index, delimiterAt); + } + for (const range of throughNearestCloser(values, views.escapes, masked, open)) { + report(range.start, range.end, '[redacted]'); + } + return unclosed; +} + +/** A key's quoted value that does not close on its line: from its key, where its quote opens, where it stops. */ +interface UnclosedValue { + readonly start: number; + readonly quote: number; + readonly end: number; +} + +/** + * The journal projection's masks for key values that do not close on their line (`visitSecretKeyValues`). + * Each is read again where every other range of the pass is filled: a mask that takes its line break (a + * scheme label's token, a nested key's value) joins its lines as the next pass would read them, so a + * value that then closes is masked whole, as it was before this rule. It is read with and without the + * key rule's own masks (`keyValues`), since another key's value can take its closing quote or end at its + * opening one, and in each with every range keeping the backslashes it ends with too; it closes at the + * latest of these readings (`laterClosing`). A reading whose view fills the value's own quote is skipped: + * there another range decides that quote. Otherwise the value is read as an unquoted run from after its + * quote, in `values`, no further than where it stopped: its quote may instead close a string the key sits + * in (`"token=" in prose`), and masking on past the run would give calls that differ only in that later + * text one projection. A run starts right after a quote, where the delimiter scan of every earlier run + * stops, so no two runs share a scan; a reading stops at the next quote that could close it in its view, + * so the readings of one kind of quote share none either. + */ +function visitUnclosedValues( + text: string, + values: string, + ranges: readonly SecretRange[], + keyValues: ReadonlySet, + unclosed: readonly UnclosedValue[], + report: SecretSpanVisitor, +): void { + const others = ranges.filter((range) => !keyValues.has(range)); + const views: string[] = []; + for (const list of others.length === ranges.length ? [ranges] : [ranges, others]) { + views.push(fillRanges(text, list)); + const kept = keepingEscapes(text, list); + if (kept !== list) views.push(fillRanges(text, kept)); + } + const delimiter = valueDelimiter(); + for (const value of unclosed) { + const from = value.quote + (values[value.quote] === '\\' ? 2 : 1); + const quote = values.slice(value.quote, from); + let reread: QuotedValueEnd | undefined; + for (const view of views) { + if (view.slice(value.quote, from) !== quote) continue; + const reading = quotedValueEnd(view, value.quote); + reread = reread === undefined ? reading : laterClosing(reread, reading); + } + if (reread?.closed === true) { + report(value.start, reread.end, '[redacted]'); + continue; + } + delimiter.lastIndex = from; + const end = Math.min(delimiter.exec(values)?.index ?? values.length, value.end); + if (end - from >= 6) report(value.start, end, '[redacted]'); + } } +/** + * Where a key's value starts: right after its separator, or one JSON string down behind escaped tabs and + * line breaks (`password:\t\"…\"`, `password:\n '…'`) when they end at an escaped quote or at an + * apostrophe, which JSON leaves unescaped; that quote then opens the value. A plain scan over the run, + * like `quotedValueEnd`'s. + */ +function escapedValueStart(text: string, start: number): number { + let index = start; + for (;;) { + const char = text[index]; + const next = text[index + 1]; + if (char === ' ' || char === '\t') index += 1; + else if (char === '\\' && (next === 't' || next === 'n' || next === 'r')) index += 2; + else break; + } + const char = text[index]; + const opening = text[index + 1]; + const quoted = char === "'" || (char === '\\' && (opening === '"' || opening === "'")); + return index > start && quoted ? index : start; +} + +/** A key's quoted value that stops at a line break, from its key; its quote is `\"` one JSON string down. */ +interface OpenValue { + readonly start: number; + readonly end: number; + readonly quote: string; +} + +/** + * The kind of open value the quote at `index` could close, as `quotedValueEnd` decodes it: after an even + * run of backslashes the quote is unescaped and may close a plain value; after a run of 4k+1 it closes a + * value one JSON string down. Each run is read once, before the quote it escapes. + */ +function closingKind(text: string, index: number): string | undefined { + const char = text[index]; + if (char !== '"' && char !== "'") return undefined; + let backslashes = 0; + while (text[index - 1 - backslashes] === '\\') backslashes += 1; + if (backslashes % 2 === 0) return char; + return backslashes % 4 === 1 ? `\\${char}` : undefined; +} + +/** + * The summary's masks for the `open` values (`redactSecretShapedSummaryText`). A summary is shown, never + * compared, so each value is masked through the nearest later quote that could close it outside the + * `masked` values, or, when every such quote is inside one, through the last of those, in its string or + * not, and only else to the end of its line. Masked values that take every later quote leave what lies + * between them uncertain (`token='\ntoken='''`), so the summary masks through all of it. Like a + * value that closes, it is read twice, with every mask filled and with each mask keeping the backslashes + * it ends with (`escapes` for the earlier masks), and masked through the later of the two + * (`laterClosing`). One backward scan, from the end of the text to the earliest end of an open value; + * each run of backslashes is read once, before the quote it escapes. + */ +function throughNearestCloser( + values: string, + escapes: string | undefined, + masked: readonly SecretRange[], + open: readonly OpenValue[], +): readonly { readonly start: number; readonly end: number }[] { + if (open.length === 0) return []; + const byEnd = open.toSorted((left, right) => right.end - left.end); + const filled = fillRanges(values, masked); + const base = escapes ?? values; + const kept = keepingEscapes(base, masked); + const keptFilled = escapes === undefined && kept === masked ? undefined : fillRanges(base, kept); + // Where the nearest quote of each kind outside the masked values ends, and where the last one does, in + // each reading. + const outside = new Map(); + const last = new Map(); + const keptOutside = new Map(); + const keptLast = new Map(); + const ranges: { start: number; end: number }[] = []; + let next = 0; + for (let index = values.length - 1; next < byEnd.length; index -= 1) { + for (let value = byEnd[next]; value !== undefined && value.end > index; value = byEnd[next]) { + next += 1; + const end = outside.get(value.quote) ?? last.get(value.quote) ?? value.end; + const keptEnd = keptOutside.get(value.quote) ?? keptLast.get(value.quote) ?? value.end; + ranges.push({ start: value.start, end: Math.max(end, keptEnd) }); + } + const kind = closingKind(filled, index); + if (kind !== undefined) outside.set(kind, index + 1); + const raw = closingKind(values, index); + if (raw !== undefined && !last.has(raw)) last.set(raw, index + 1); + if (keptFilled === undefined) continue; + const keptKind = closingKind(keptFilled, index); + if (keptKind !== undefined) keptOutside.set(keptKind, index + 1); + const keptRaw = closingKind(base, index); + if (keptRaw !== undefined && !keptLast.has(keptRaw)) keptLast.set(keptRaw, index + 1); + } + return ranges; +} + +/** A host-shaped prefix: a dotted name (or IPv4), `localhost` or a bracketed IP literal, with an optional port. */ +const HOST_SHAPED_AUTHORITY = + /^(?:[a-z\d-]+(?:\.[a-z\d-]+)+\.?|localhost|\[[\da-f:.]+\])(?::\d{1,5})?$/i; + +/** A DNS name has at most 253 characters (254 with a trailing dot), and a `:port` adds at most six. */ +const HOST_SHAPED_MAX_LENGTH = 260; + /** * One whole authority, with its last `@` separating userinfo from the host. Prefix-only * matches can mask the first password fragment and leave another `@` plus a credential * tail in the durable digest. Paths and JSON delimiters stop the scan; queries/fragments - * after an existing userinfo separator remain public. Before that separator, raw ?/# in a - * colon-bearing password are masked, except an explicit public host:port prefix; a short colonless vanity handle retains the existing non-secret policy. + * after an existing userinfo separator remain public. Before that separator, a raw ?/# + * begins public query/fragment text only when the text before it is host-shaped **and** an + * `=` assignment follows it; anything else may be a malformed username or password. This is the + * same predicate as the LLM diagnostic backstop (`packages/llm/src/scrub-secrets.ts`). For such an + * ambiguous userinfo the backstop masks through the authority's end unless a host-shaped name follows + * its last `@`. This projection masks the userinfo and reads every key and parameter in it but its + * username's; their values keep their own extent and the rules' ranges merge + * (`https://gateway?password=P@ss.word1` projects to `https://[redacted]`). + * The projection differs from that backstop by design: its authority also ends at `"`, `'`, + * `}` and `]`, the delimiters of the JSON and quoted texts it reads, so userinfo containing one of + * them (an already-masked `[REDACTED]` password among them) is left as it is; it recognizes any + * scheme of up to 32 characters rather than a finite list; and a userinfo with no password is a + * vanity handle below 16 characters and masked from 16 on, under every scheme. A `,` or `/` in a + * password is read on to its `@` ({@link passwordPastDelimiter}). + */ +function isPublicQueryContact(userinfo: string, query: number): boolean { + // A longer prefix cannot be a hostname, and an unbounded dotted run would exhaust V8's RegExp + // backtrack stack at admitted size; such a prefix stays userinfo, which fails closed. + if (query > HOST_SHAPED_MAX_LENGTH) return false; + return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query)) && userinfo.includes('=', query + 1); +} + +/** + * The `@` of a password that holds a `,` or `/`, or -1. The authority scan stops at either, so + * `https://user:pa,ss@host` and `https://user:pa/ss@host` left the password's tail public (a PR #96 + * review finding; RFC 3986 admits a raw `,` in userinfo, and a raw `/` is a malformed URL a + * credential can still sit in). Reads on past them only, to the first `@` within 512 characters. A + * port (`host:8080/a@b`, `host:8080,bob@x.io`), a public query contact as the authority scan reads + * one ({@link isPublicQueryContact}), another scheme, and the delimiters the authority keeps + * (whitespace, quotes, `}`, `]`) keep the text public. + */ +function passwordPastDelimiter(text: string, start: number, end: number): number { + if (text[end] !== ',' && text[end] !== '/') return -1; + const authority = text.slice(start, end); + const colon = authority.indexOf(':'); + if (colon < 0 || /^\d*$/.test(authority.slice(colon + 1))) return -1; + const query = authority.search(/[?#]/); + if (query >= 0 && isPublicQueryContact(authority, query)) return -1; + PASSWORD_TAIL.lastIndex = end; + const tail = PASSWORD_TAIL.exec(text)?.[0]; + if (tail === undefined || tail.includes('://')) return -1; + return end + tail.length - 1; +} + +/** + * Mask a userinfo whose password holds a `,` or `/` ({@link passwordPastDelimiter}); its username owns the + * text before the first `:`, as in {@link visitUrlUserinfo}. Returns where the scheme scan resumes: after + * the `@` it masked through, or at `start` when there is none. */ -function isPublicHostPort(text: string): boolean { - const colon = text.lastIndexOf(':'); - if (colon < 0 || !/^\d{1,5}$/.test(text.slice(colon + 1))) return false; - const hostname = text.slice(0, colon); - return hostname.includes('.') || hostname === 'localhost' || /^\[[\da-f:]+\]$/i.test(hostname); +function maskPasswordPastDelimiter( + text: string, + schemeStart: number, + start: number, + end: number, + mask: SecretSpanVisitor, +): number { + const passwordAt = passwordPastDelimiter(text, start, end); + if (passwordAt < 0) return start; + mask(start, passwordAt, '[redacted]', { schemeStart, ownsUntil: text.indexOf(':', start) }); + return passwordAt + 1; } +/** A password's tail past a `,` or `/`, through its `@`; bounded so a long run is never rescanned. */ +const PASSWORD_TAIL = /[^\s"'}\]?#@]{0,512}@/y; + function visitUrlUserinfo(text: string, mask: SecretSpanVisitor): void { if (!text.includes('://')) return; const scheme = /[a-z][\w+.-]{0,31}:\/\//gi; @@ -457,34 +1209,32 @@ function visitUrlUserinfo(text: string, mask: SecretSpanVisitor): void { const firstQuery = authority.search(/[?#]/); const authorityAt = firstQuery < 0 ? -1 : authority.lastIndexOf('@', firstQuery - 1); const at = authorityAt >= 0 ? authorityAt : authority.lastIndexOf('@'); - if (at < 0) continue; + if (at < 0) { + scheme.lastIndex = maskPasswordPastDelimiter(text, match.index, start, end, mask); + continue; + } const userinfo = authority.slice(0, at); const colon = userinfo.indexOf(':'); const query = userinfo.search(/[?#]/); - // Raw ?/# can belong to a malformed password. An explicit host:port prefix instead - // introduces a public query/fragment; its contact values must not become userinfo. - if (query >= 0 && (colon < 0 || colon > query || isPublicHostPort(userinfo.slice(0, query)))) - continue; + // Raw ?/# can belong to a malformed username or password. A host-shaped prefix followed by + // an assignment instead introduces a public query/fragment whose contact must not become userinfo. + if (query >= 0 && isPublicQueryContact(userinfo, query)) continue; const hasPassword = colon >= 0 && colon < userinfo.length - 1; if (!hasPassword && userinfo.length < 16) continue; - mask(start, start + at, '[redacted]'); + // Only a username owns its text: the part before the first ':' when no raw ?/# comes first. A key + // there names the user (`https://x-access-token:@github.com/o/r` keeps its host). A key in the + // password, after a raw ?/# or in a userinfo with no username is read by the key/value and + // opaque-query rules, which mask its value to its own end (`jdbc:…;password=P@ss;…`). + const ownsUntil = colon >= 0 && (query < 0 || colon < query) ? colon : 0; + mask(start, start + at, '[redacted]', { + schemeStart: match.index, + ownsUntil: start + ownsUntil, + }); // Skip the recognized userinfo without hiding a later scheme in the public host/tail. scheme.lastIndex = start + at + 1; } } -function redactUrlUserinfo(text: string): string { - const parts: string[] = []; - let cursor = 0; - visitUrlUserinfo(text, (start, end, replacement) => { - parts.push(text.slice(cursor, start), replacement); - cursor = end; - }); - if (parts.length === 0) return text; - parts.push(text.slice(cursor)); - return parts.join(''); -} - /** * Walk an arbitrary value applying {@link redactSecretShapedText} to every string, leaving structure intact — * the toolInput twin of the summary scrub. Applied to the `agent:tool_call.toolInput` event field so a @@ -494,7 +1244,116 @@ function redactUrlUserinfo(text: string): string { * position is redacted, closing that leak path too). Cycle-safe; model/dispatch values stay unchanged. */ export function redactSecretShapedValue(value: unknown): unknown { - return redactSecretShapedWalk(value, new WeakSet()); + return redactSecretShapedWalk(value, new WeakSet(), redactSecretShapedText); +} + +/** + * The journal projection of a tool's arguments (`redactSecretShapedValue`) and whether it settled. A string + * whose projection still changes on its third pass becomes `[redacted]`, and two member names of one object + * that project alike leave one member, either of which would give different calls one journal identity, so + * the registry refuses such an effect before it is prepared + * ([effect-journal.md](../../../../docs/reference/shared-core/effect-journal.md) §11). + */ +export function projectSecretShapedValue(value: unknown): { + readonly value: unknown; + readonly settled: boolean; +} { + let settled = true; + const projected = redactSecretShapedWalk( + value, + new WeakSet(), + (text) => { + const fixed = fixedPointOf(redactSecretShapedOnce, text); + if (fixed !== undefined) return fixed; + settled = false; + return '[redacted]'; + }, + () => { + settled = false; + }, + ); + return { value: projected, settled }; +} + +/** How the walk projects each string: the display projection, or the journal's settle-checking one. */ +type TextProjection = (text: string) => string; + +/** + * What the walk puts in place of a value it masks whole, a credential member or a credential flag's + * argument, given that value and its `mask` (`[redacted]`, or a canonical scheme label and `[redacted]`). + */ +type WholeMask = (value: unknown, mask: string) => string; + +/** The journal's and the display's whole mask: the mask itself. */ +const maskWhole: WholeMask = (_value, mask) => mask; + +/** + * The text a non-string result's summary is made from (`toText`): its JSON with every value the walk masks + * whole (`redactSecretShapedWalk`) masked, and read once by the text rules before those masks, so a quoted + * value or a PEM block that runs into or out of such a value keeps the context it is read in, as the JSON + * of the unwalked result would give it. The walk leaves a private-use placeholder for each such value; the + * value's own JSON goes back where its placeholder is, and its mask is one more range of that reading. A + * result whose JSON already holds a placeholder character, or one that has no JSON form, keeps the walk's + * masks in place and skips the reading. + */ +function structuredSummaryText(safe: unknown): string { + const holes: { readonly value: unknown; readonly mask: string }[] = []; + const marked = jsonText(() => + redactSecretShapedWalk( + safe, + new WeakSet(), + (text) => text, + undefined, + (value, mask) => `\uE000${holes.push({ value, mask }) - 1}\uE001`, + ), + ); + if (holes.length === 0) return marked; + const placeholders = [...marked.matchAll(/\uE000(\d+)\uE001/g)]; + if (placeholders.length !== holes.length || marked.split('\uE000').length - 1 !== holes.length) { + return jsonText(() => redactSecretShapedWalk(safe, new WeakSet(), (text) => text)); + } + const parts: string[] = []; + const masks: SecretRange[] = []; + let length = 0; + let copied = 0; + for (const placeholder of placeholders) { + const hole = holes[Number(placeholder[1])]; + if (hole === undefined) continue; + // A placeholder that is a whole JSON string stands for any value; one inside a string (a + // `--password=` argument) stands for a string's text there. + let start = placeholder.index; + let end = start + placeholder[0].length; + let json: string; + let mask: string; + if (marked[start - 1] === '"' && marked[end] === '"') { + start -= 1; + end += 1; + // A value with no JSON form (a function, a bigint, a cycle) is read as its mask. + json = jsonValue(hole.value) ?? JSON.stringify(hole.mask); + mask = JSON.stringify(hole.mask); + } else { + json = JSON.stringify(typeof hole.value === 'string' ? hole.value : hole.mask).slice(1, -1); + mask = JSON.stringify(hole.mask).slice(1, -1); + } + const before = marked.slice(copied, start); + parts.push(before, json); + length += before.length; + masks.push({ start: length, end: length + json.length, replacement: mask }); + length += json.length; + copied = end; + } + parts.push(marked.slice(copied)); + const text = parts.join(''); + return replaceRanges(text, [...secretRanges(text, true), ...masks]); +} + +/** The JSON of one value, or `undefined` where it has none or its serialization throws. */ +function jsonValue(value: unknown): string | undefined { + try { + return JSON.stringify(value); + } catch { + return undefined; + } } /** @@ -534,12 +1393,18 @@ function isSecretishKey(key: string): boolean { * answers one with the other's recorded result — the exact failure the digest exists to prevent. */ function keyWords(key: string): readonly string[] { - return key + return nameWords(key).map((word) => + word.endsWith('s') && word.length > 3 ? word.slice(0, -1) : word, + ); +} + +/** A name's lowercase words as written, before {@link keyWords} folds plurals. */ +function nameWords(name: string): readonly string[] { + return name .replace(/([a-z0-9])([A-Z])/g, '$1 $2') .split(/[^A-Za-z0-9]+/) .filter((word) => word.length > 0) - .map((word) => word.toLowerCase()) - .map((word) => (word.endsWith('s') && word.length > 3 ? word.slice(0, -1) : word)); + .map((word) => word.toLowerCase()); } /** Names that mean "this is a credential" on their own. */ @@ -578,40 +1443,74 @@ const KEY_QUALIFIERS: ReadonlySet = new Set([ * FOLLOWING a secret-ish flag is redacted; the flag itself is kept, because which flag was passed is * diagnostic and is not the secret. */ -function redactArgVector(items: readonly unknown[], seen: WeakSet): unknown[] { +function redactArgVector( + items: readonly unknown[], + seen: WeakSet, + project: TextProjection, + collide: (() => void) | undefined, + whole: WholeMask, +): unknown[] { const out: unknown[] = []; let redactNext: string | undefined; for (const item of items) { if (redactNext !== undefined) { - out.push(redactCredentialArgument(redactNext, item)); + const masked = redactCredentialArgument(redactNext, item, whole); + out.push(masked ?? redactSecretShapedWalk(item, seen, project, collide, whole)); redactNext = undefined; continue; } - if (typeof item === 'string') { - const separator = item.indexOf('='); - const flag = separator > 0 ? item.slice(0, separator) : ''; - if (isSecretishFlag(flag)) { - const value = item.slice(separator + 1); - out.push(`${flag}=${redactCredentialArgument(flag, value)}`); - continue; - } - if (isSecretishFlag(item)) redactNext = item; + const inline = typeof item === 'string' ? redactInlineCredential(item, whole) : undefined; + if (inline !== undefined) { + out.push(inline); + continue; } - out.push(redactSecretShapedWalk(item, seen)); + if (typeof item === 'string' && isCredentialFlag(item)) redactNext = item; + out.push(redactSecretShapedWalk(item, seen, project, collide, whole)); } return out; } -/** One credential-value rule for both argv spellings; only ambiguous -p may name a public port. */ -function redactCredentialArgument(flag: string, value: unknown): string { +/** + * One argument that carries its own credential, masked: a credential flag's `--flag=value` spelling or curl's + * glued `-ualice:secret`. `undefined` when the argument is neither, so the vector walks it as usual. + */ +function redactInlineCredential(item: string, whole: WholeMask): string | undefined { + const separator = item.indexOf('='); + const flag = separator > 0 ? item.slice(0, separator) : ''; + if (isCredentialFlag(flag)) { + const masked = redactCredentialArgument(flag, item.slice(separator + 1), whole); + if (masked !== undefined) return `${flag}=${masked}`; + } + const glued = gluedUserPassword(item); + return glued === undefined ? undefined : `-u${whole(glued, '[redacted]')}`; +} + +/** + * One credential-value rule for both argv spellings; only ambiguous -p may name a public port. `undefined` + * means the value is not a credential after all (a `-u`/`--user` name without a password) and is walked as + * any other element. + */ +function redactCredentialArgument( + flag: string, + value: unknown, + whole: WholeMask, +): string | undefined { if (flag.toLowerCase() === '-p' && typeof value === 'string' && looksLikePort(value)) { return value; } - return '[redacted]'; + if (isUserPasswordFlag(flag)) { + return looksLikeUserPassword(value) ? whole(value, '[redacted]') : undefined; + } + return whole(value, '[redacted]'); +} + +/** A flag whose next argument (or `=` value) is a credential, or a `user:password` pair. */ +function isCredentialFlag(flag: string): boolean { + return isSecretishFlag(flag) || isUserPasswordFlag(flag); } /** - * A CLI flag that introduces a credential — `--token`, `--password`, `-p`. + * A CLI flag that introduces a credential — `--token`, `--password`, `--auth-token`, `-p`. * * `-p` is deliberately kept AND deliberately qualified. It really is the password flag for `mysql` and * `psql`, and dropping it would leak those verbatim — but it is also the port flag for `ssh` and the @@ -620,7 +1519,27 @@ function redactCredentialArgument(flag: string, value: unknown): string { * `-p` ambiguity retains that exception; explicitly named credential flags do not. Both separate and * `--password=…` argument spellings use the entire value, rather than a partial text-pattern match. * See {@link looksLikePort}. + * + * Besides the anchored names, a flag counts when its LAST word names a credential, read with the object-key + * rule's words ({@link SECRET_WORDS}, or `key` qualified by the word before it): `--auth-token`, + * `--access-token`, `--db-password`, `--pwd`, `--x-api-key`. A PR #96 review found those passing verbatim + * while the key rule masked the same names as object keys. The last word, not any word, because a flag + * that only MENTIONS a credential is usually a boolean or names a file — `--password-stdin`, `--token-file` + * — and masking the element after it would make calls that differ only there share one journal identity. + * For the same reason the word is read as written, without the key rule's plural folding (`--max-tokens` + * is a count), with `credentials` the one plural admitted, and a `no-` flag (`--no-password`) never counts. */ +function isSecretishFlag(item: string): boolean { + if (SECRETISH_FLAG.test(item)) return true; + const name = /^--?([A-Za-z][\w-]{0,63})$/.exec(item)?.[1]; + if (name === undefined) return false; + const words = nameWords(name); + if (words[0] === 'no') return false; + const last = words.at(-1) ?? ''; + if (SECRET_WORDS.has(last) || last === 'credentials') return true; + return last === 'key' && KEY_QUALIFIERS.has(words.at(-2) ?? ''); +} + /** * The flag names, as one anchored alternation. * @@ -631,8 +1550,31 @@ function redactCredentialArgument(flag: string, value: unknown): string { const SECRETISH_FLAG = /^-{1,2}(?:p|pw|password|passwd|secret|token|api[_-]?key|auth|access[_-]?key|private[_-]?key|client[_-]?secret|credential|passphrase|pin)$/i; // NOSONAR — the metric is counting NAMES; see above -function isSecretishFlag(item: string): boolean { - return SECRETISH_FLAG.test(item); +/** + * curl's `-u`/`--user` carries `user:password` in ONE argument, so its value is a credential only when it + * holds a `:`. A `uid[:gid]` (`docker run -u 1000:1000`) and a URL (`pip install --user git+https://…`) + * are public. + */ +function isUserPasswordFlag(flag: string): boolean { + return flag === '-u' || flag === '--user'; +} + +function looksLikeUserPassword(value: unknown): value is string { + return ( + typeof value === 'string' && + value.includes(':') && + !value.includes('://') && + !/^\d+(?::\d+)?$/.test(value) + ); +} + +/** The value of curl's glued `-ualice:secret` spelling, when it is a `user:password` pair. */ +function gluedUserPassword(item: string): string | undefined { + if (!item.startsWith('-u') || item.length < 3 || item[2] === '-' || item[2] === '=') { + return undefined; + } + const value = item.slice(2); + return looksLikeUserPassword(value) ? value : undefined; } /** Port-shaped values disambiguate `-p`; an explicit credential flag has no port exception. */ @@ -647,32 +1589,46 @@ function looksLikePort(value: string): boolean { * remaining text is public. Do not traverse a credential container or carry any partially scrubbed value * into the journal. Only an anchored, known auth scheme may survive as a fixed diagnostic label. */ -function redactUnderKey(key: string, item: unknown, seen: WeakSet): unknown { +function redactUnderKey( + key: string, + item: unknown, + seen: WeakSet, + project: TextProjection, + collide: (() => void) | undefined, + whole: WholeMask, +): unknown { if (!isSecretishKey(key) || item === null || item === undefined) - return redactSecretShapedWalk(item, seen); + return redactSecretShapedWalk(item, seen, project, collide, whole); if (typeof item === 'string') { // Match only the bounded label and the start of a nonempty remainder, never return that remainder. // Canonical labels also keep caller-controlled casing out of the credential projection. const scheme = /^(bearer|basic|token)[ \t]+\S/i.exec(item)?.[1]?.toLowerCase(); switch (scheme) { case 'bearer': - return 'Bearer [redacted]'; + return whole(item, 'Bearer [redacted]'); case 'basic': - return 'Basic [redacted]'; + return whole(item, 'Basic [redacted]'); case 'token': - return 'Token [redacted]'; + return whole(item, 'Token [redacted]'); } } - return '[redacted]'; + return whole(item, '[redacted]'); } -function redactSecretShapedWalk(value: unknown, seen: WeakSet): unknown { - if (typeof value === 'string') return redactSecretShapedText(value); +function redactSecretShapedWalk( + value: unknown, + seen: WeakSet, + project: TextProjection, + // Reports two member names of one object that project alike, which leaves only the later member. + collide?: () => void, + whole: WholeMask = maskWhole, +): unknown { + if (typeof value === 'string') return project(value); if (typeof value !== 'object' || value === null) return value; if (seen.has(value)) return '[cyclic]'; // break the cycle (mirrors redactInlineMedia) — never re-emit the ref seen.add(value); try { - if (Array.isArray(value)) return redactArgVector(value, seen); + if (Array.isArray(value)) return redactArgVector(value, seen, project, collide, whole); if (!isPlainObject(value)) return value; // Date/RegExp/Map/… — leave for native handling const out: Record = {}; // Scrub the KEY too (a secret-shaped key is redacted; a normal name is unchanged — see the doc above). A @@ -682,8 +1638,10 @@ function redactSecretShapedWalk(value: unknown, seen: WeakSet): unknown // inspects strings by shape — so a numeric PIN, a nested credential object, or a high-entropy opaque // token with no recognisable prefix all survive it. `null`/`undefined` are left alone so an absent // field does not become a phantom `[redacted]` in a diagnostic. - Object.defineProperty(out, redactSecretShapedText(key), { - value: redactUnderKey(key, item, seen), + const name = project(key); + if (Object.hasOwn(out, name)) collide?.(); + Object.defineProperty(out, name, { + value: redactUnderKey(key, item, seen, project, collide, whole), enumerable: true, configurable: true, writable: true, @@ -695,33 +1653,67 @@ function redactSecretShapedWalk(value: unknown, seen: WeakSet): unknown } } -/** Render any result to the text the model would see (a string is itself; else compact JSON). Inline - * media bytes are redacted first so the summary/spill/preview can never carry base64 (I3). */ -function toText(result: unknown): string { +/** + * Render any result to the text the model would see (a string is itself; else compact JSON) and to the text + * its summary is made from. Inline media bytes are redacted first so the summary/spill/preview can never + * carry base64 (I3). A non-string result's summary text is its JSON with every member under a + * credential-named key and every argument after a credential flag masked whole (`structuredSummaryText`): + * the text rules read those values once in the unwalked JSON, with the whole masks as ranges of that same + * pass, and the later passes read only the masks. The text rules read the JSON as one text (`makeSummary`), + * a value one JSON string down included, so a PEM block or a quoted value that runs across members keeps + * the context it is read in; projecting each string alone would mask a key or a PEM marker that a later + * member's text needs. + */ +function toText(result: unknown): { + readonly text: string; + readonly summaryText: string; + /** The inline media bytes the texts replaced with markers but the model-facing `result` still holds. */ + readonly omittedBytes: number; +} { + const omitted: OmittedMedia = { bytes: 0 }; if (typeof result === 'string') { - return isBase64DataUri(result) ? '[base64 data URI omitted]' : result; + const text = isBase64DataUri(result) ? '[base64 data URI omitted]' : result; + return { text, summaryText: text, omittedBytes: text === result ? 0 : utf8ByteLength(result) }; } if (result === undefined) { - return ''; + return { text: '', summaryText: '', omittedBytes: 0 }; } + let safe: unknown; try { // Always run the redaction walk for a non-string result (inside the try so a throwing getter/proxy // degrades to '[unserializable]' rather than escaping): it strips inline base64 sources, base64 data: // URIs, AND raw binary buffers at any nesting before JSON-serialisation, so no media bytes (base64 or // decimal) can ride the summary/spill/preview (I3). A non-media object is returned as a structural clone - // (same JSON). JSON.stringify returns undefined for a value with no JSON form (a bare function/symbol); - // the `??` keeps a string without an `[object Object]`. - const safe = redactInlineMediaForText(result, new WeakSet()); - return JSON.stringify(safe) ?? '[unserializable]'; + // (same JSON). Both texts are made from this one walk. + safe = redactInlineMediaForText(result, new WeakSet(), omitted); + } catch { + return { text: '[unserializable]', summaryText: '[unserializable]', omittedBytes: 0 }; // throwing getter during redaction + } + return { + text: jsonText(() => safe), + summaryText: structuredSummaryText(safe), + omittedBytes: omitted.bytes, + }; +} + +/** + * JSON of a value, or `[unserializable]` for one that has no JSON form (JSON.stringify returns undefined for + * a bare function/symbol; the `??` keeps a string without an `[object Object]`) or whose serialization throws + * (a circular reference, a throwing toJSON). + */ +function jsonText(value: () => unknown): string { + try { + return JSON.stringify(value()) ?? '[unserializable]'; } catch { - return '[unserializable]'; // circular reference / throwing toJSON / throwing getter during redaction + return '[unserializable]'; } } function makeSummary(text: string): string { // Recognize LF/TAB-split shapes in the complete materialized text before one-line normalization. Clip only after recognition: a closing quote or URL // authority separator can be beyond the preview. Raw model/spill text stays unchanged. - const oneLine = redactSecretShapedText(redactSplitCredentials(text).replace(/\s+/g, ' ').trim()); + const recognized = /[\n\t]/.test(text) ? redactSecretShapedSummaryText(text) : text; + const oneLine = redactSecretShapedSummaryText(recognized.replace(/\s+/g, ' ').trim()); return oneLine.length <= SUMMARY_MAX ? oneLine : `${oneLine.slice(0, SUMMARY_MAX)}…`; } @@ -849,9 +1841,11 @@ export async function boundForModel( host: ToolHost, signal?: AbortSignalLike, ): Promise { - const text = toText(result); - const summary = makeSummary(text); - const bytes = utf8ByteLength(text); + const { text, summaryText, omittedBytes } = toText(result); + const summary = makeSummary(summaryText); + // The ceiling is the model-facing value's, so inline media the texts omit still counts; the spill and the + // preview below are made from the media-free text, so an over-ceiling result never hands them base64. + const bytes = utf8ByteLength(text) + omittedBytes; const lines = countLines(text); if (bytes <= limits.maxBytes && lines <= limits.maxLines) { @@ -878,5 +1872,10 @@ export async function boundForModel( ref === undefined ? `\n\n[… truncated: ${bytes} bytes / ${lines} lines; full output unavailable (${unavailableNote}) …]` : `\n\n[… truncated: ${bytes} bytes / ${lines} lines; full output at ${ref} …]`; - return { value: makePreview(text, limits) + marker, truncated: true, summary }; + // Over the ceiling only by the media it omits, the text itself is the preview, not a doubled head and tail. + const preview = + utf8ByteLength(text) <= limits.maxBytes && lines <= limits.maxLines + ? text + : makePreview(text, limits); + return { value: preview + marker, truncated: true, summary }; } diff --git a/packages/core/src/tools/fixed-point.test.ts b/packages/core/src/tools/fixed-point.test.ts new file mode 100644 index 00000000..df9ac902 --- /dev/null +++ b/packages/core/src/tools/fixed-point.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest'; + +import { fixedPointOf, toFixedPoint } from './fixed-point.js'; + +describe('toFixedPoint', () => { + const failed = '[failed]'; + const passes = (outputs: readonly string[]) => { + const calls: string[] = []; + const pass = (text: string) => { + calls.push(text); + return outputs[calls.length - 1] ?? text; + }; + return { pass, calls }; + }; + + it('returns an input the first pass leaves unchanged after one pass', () => { + const { pass, calls } = passes(['a']); + expect(toFixedPoint(pass, 'a', failed)).toBe('a'); + expect(calls).toEqual(['a']); + }); + + it('returns the first output once a second pass confirms it', () => { + const { pass, calls } = passes(['b', 'b']); + expect(toFixedPoint(pass, 'a', failed)).toBe('b'); + expect(calls).toEqual(['a', 'b']); + }); + + it('returns the second output once a third pass confirms it', () => { + const { pass, calls } = passes(['b', 'c', 'c']); + expect(toFixedPoint(pass, 'a', failed)).toBe('c'); + expect(calls).toEqual(['a', 'b', 'c']); + }); + + it('fails closed when the third pass still changes the text', () => { + const { pass, calls } = passes(['b', 'c', 'd']); + expect(toFixedPoint(pass, 'a', failed)).toBe(failed); + expect(calls).toEqual(['a', 'b', 'c']); + }); +}); + +describe('fixedPointOf', () => { + it('reports a settled output and an unsettled one apart', () => { + let calls = 0; + const settles = (text: string) => (text === 'a' ? 'b' : text); + const drifts = (text: string) => { + calls += 1; + return `${text}+`; + }; + expect(fixedPointOf(settles, 'a')).toBe('b'); + expect(fixedPointOf(drifts, 'a')).toBeUndefined(); + expect(calls).toBe(3); + }); +}); diff --git a/packages/core/src/tools/fixed-point.ts b/packages/core/src/tools/fixed-point.ts new file mode 100644 index 00000000..526c5185 --- /dev/null +++ b/packages/core/src/tools/fixed-point.ts @@ -0,0 +1,21 @@ +/** + * Repeats one projection pass until its output stops changing, at most three passes. A pass can + * change the context the next pass decides on, so a settled output is a fixed point; an output that + * still changes on the third pass has not settled, and this returns `undefined` for it. + */ +export function fixedPointOf(pass: (text: string) => string, text: string): string | undefined { + const once = pass(text); + if (once === text) return once; + const twice = pass(once); + if (twice === once) return once; + return pass(twice) === twice ? twice : undefined; +} + +/** + * The pass's fixed point (`fixedPointOf`), failing closed to `failed` when it has not settled. Callers + * choose a `failed` value the pass leaves unchanged, so the result is stable under another call + * either way. + */ +export function toFixedPoint(pass: (text: string) => string, text: string, failed: string): string { + return fixedPointOf(pass, text) ?? failed; +} diff --git a/packages/core/src/tools/registry.test.ts b/packages/core/src/tools/registry.test.ts index 4671b574..0a424e7d 100644 --- a/packages/core/src/tools/registry.test.ts +++ b/packages/core/src/tools/registry.test.ts @@ -1,6 +1,8 @@ import type { AbortSignalLike, Scope, ToolPolicy } from '@relavium/shared'; import { describe, expect, it, vi } from 'vitest'; +import { z } from 'zod'; +import { projectSecretShapedValue } from './bounding.js'; import { BUILTIN_TOOLS, BUILTIN_TOOL_IDS } from './builtins.js'; import { ToolArgsInvalidError, @@ -21,11 +23,19 @@ import type { ToolApprovalDecision, ToolApprovalRequest, ToolCallPart, + ToolDef, ToolDispatchContext, ToolHost, } from './types.js'; import { createInMemoryEffectJournal } from '../engine/execution-host.js'; +// Observe the journal projection, retaining its actual implementation, so a case can force a value that +// does not settle on arguments of its choosing; another uses a real input that does not settle. +vi.mock('./bounding.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, projectSecretShapedValue: vi.fn(actual.projectSecretShapedValue) }; +}); + /* --- helpers --- */ function call(name: string, args?: unknown, providerExecuted?: boolean): ToolCallPart { @@ -429,6 +439,29 @@ describe('ToolRegistry — git_commit human-gate', () => { expect(prepare).toHaveBeenCalledTimes(1); expect(settle).toHaveBeenCalledTimes(1); }); + + it('settles a bounded preview, not a 30 MiB data URI, into the effect journal', async () => { + // A PR #96 review: the ceiling was measured on the data URI's text marker, so the whole value reached + // the model and `settleCommitted` wrote it to `history.db`. + const uri = `data:image/png;base64,${'A'.repeat(30 * 1024 * 1024)}`; + const spawn = vi.fn['spawn']>(() => + Promise.resolve({ exitCode: 0, stdout: uri, stderr: '', durationMs: 1 }), + ); + const context = ctx({ toolPolicy: { allowedCommands: ['ls'] } }); + const settle = vi.fn(context.effects.settle); + const out = await registry(stubHost({ process: { spawn } })).dispatch( + call('run_command', { command: 'ls' }), + { + ...context, + effects: { ...context.effects, settle }, + }, + ); + expect(out.events.result.success).toBe(true); + expect(settle).toHaveBeenCalledTimes(1); + const settled = JSON.stringify(settle.mock.calls[0]?.[3]); + expect(settled).not.toContain('AAAAAAAA'); + expect(settled.length).toBeLessThan(64 * 1024); + }); }); /* --- per-tool approval (ADR-0057 EA3) --- */ @@ -1653,3 +1686,190 @@ describe('ToolRegistry — remaining hardening regressions (TG-2, TG-4)', () => expect(err.reason).toBe('insecure_url'); }); }); + +describe('ToolRegistry — journal identity of credential-shaped arguments (PR95 round 7)', () => { + it('refuses a tiered effect whose journal projection does not settle, before preparing it', async () => { + const actual = await vi.importActual('./bounding.js'); + vi.mocked(projectSecretShapedValue).mockImplementation((value) => ({ + value: actual.projectSecretShapedValue(value).value, + settled: !JSON.stringify(value).includes('UNSETTLED'), + })); + try { + const writeFile = vi.fn['writeFile']>((path, data) => + Promise.resolve({ path, bytesWritten: data.length }), + ); + const host = stubHost(); + if (host.fs === undefined) throw new Error('missing stub fs'); + const context = ctx(); + const prepare = vi.fn(context.effects.prepare); + const err: unknown = await registry(stubHost({ fs: { ...host.fs, writeFile } })) + .dispatch( + call('write_file', { path: './out.txt', content: 'UNSETTLED body', append: true }), + { + ...context, + effects: { ...context.effects, prepare }, + }, + ) + .catch((error: unknown) => error); + expect(err, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toBeInstanceOf(ToolArgsInvalidError); + if (!(err instanceof ToolArgsInvalidError)) throw new Error('missing refusal'); + expect(err.fields, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toEqual(['content']); + expect(err.message.includes('UNSETTLED'), 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toBe(false); + expect(prepare, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').not.toHaveBeenCalled(); + expect(writeFile, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').not.toHaveBeenCalled(); + } finally { + vi.mocked(projectSecretShapedValue).mockImplementation(actual.projectSecretShapedValue); + } + }); + + it('names the arguments it refuses through the same masker', async () => { + const actual = await vi.importActual('./bounding.js'); + vi.mocked(projectSecretShapedValue).mockImplementation((value) => ({ + value: actual.projectSecretShapedValue(value).value, + settled: !JSON.stringify(value).includes('UNSETTLED'), + })); + try { + // An MCP tool's open schema lets the model choose the argument names themselves. + const dispatch = vi.fn(() => Promise.resolve({ receipt: 'published' })); + const tool: ToolDef = { + id: 'open_publish', + source: 'mcp', + description: 'Offline open-schema fixture', + llmVisibleParams: { type: 'object' }, + parseArgs: (raw) => z.record(z.string(), z.unknown()).parse(raw), + policy: { fsScoped: false, spawnsProcess: false, requiresGateApproval: false }, + effect: () => 3, + dispatch, + }; + const context = ctx({ grantedToolIds: new Set([tool.id]) }); + const prepare = vi.fn(context.effects.prepare); + const err: unknown = await createToolRegistry({ tools: [tool], host: stubHost() }) + .dispatch( + call('open_publish', { + content: 'UNSETTLED body', + ghp_abcdefghijklmnopqrstuvwxyz0123UNSETTLED: 'x', + 'sk-proj-abcdefghijklmnopqrstuvUNSETTLED': 'y', + }), + { ...context, effects: { ...context.effects, prepare } }, + ) + .catch((error: unknown) => error); + expect(err, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toBeInstanceOf(ToolArgsInvalidError); + if (!(err instanceof ToolArgsInvalidError)) throw new Error('missing refusal'); + expect(err.fields, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toEqual(['content', '[redacted]']); + for (const raw of ['ghp_', 'sk-proj', 'UNSETTLED']) { + expect(err.message.includes(raw), 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toBe(false); + } + expect(prepare, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').not.toHaveBeenCalled(); + expect(dispatch, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').not.toHaveBeenCalled(); + } finally { + vi.mocked(projectSecretShapedValue).mockImplementation(actual.projectSecretShapedValue); + } + }); + + it('names every argument when only the whole projection does not settle', async () => { + const actual = await vi.importActual('./bounding.js'); + vi.mocked(projectSecretShapedValue).mockImplementation((value) => ({ + value: actual.projectSecretShapedValue(value).value, + settled: !(typeof value === 'object' && value !== null && Object.keys(value).length > 1), + })); + try { + const writeFile = vi.fn['writeFile']>((path, data) => + Promise.resolve({ path, bytesWritten: data.length }), + ); + const host = stubHost(); + if (host.fs === undefined) throw new Error('missing stub fs'); + const context = ctx(); + const prepare = vi.fn(context.effects.prepare); + const err: unknown = await registry(stubHost({ fs: { ...host.fs, writeFile } })) + .dispatch(call('write_file', { path: './out.txt', content: 'body', append: true }), { + ...context, + effects: { ...context.effects, prepare }, + }) + .catch((error: unknown) => error); + expect(err, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toBeInstanceOf(ToolArgsInvalidError); + if (!(err instanceof ToolArgsInvalidError)) throw new Error('missing refusal'); + expect(err.fields, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toEqual([ + 'append', + 'content', + 'path', + ]); + expect(prepare, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').not.toHaveBeenCalled(); + expect(writeFile, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').not.toHaveBeenCalled(); + } finally { + vi.mocked(projectSecretShapedValue).mockImplementation(actual.projectSecretShapedValue); + } + }); + + it('refuses an open-schema effect whose argument names collide after masking', async () => { + // Two names that project alike would leave one member for two argument sets. + const dispatch = vi.fn(() => Promise.resolve({ receipt: 'published' })); + const tool: ToolDef = { + id: 'open_publish', + source: 'mcp', + description: 'Offline open-schema fixture', + llmVisibleParams: { type: 'object' }, + parseArgs: (raw) => z.record(z.string(), z.unknown()).parse(raw), + policy: { fsScoped: false, spawnsProcess: false, requiresGateApproval: false }, + effect: () => 3, + dispatch, + }; + const context = ctx({ grantedToolIds: new Set([tool.id]) }); + const prepare = vi.fn(context.effects.prepare); + const err: unknown = await createToolRegistry({ tools: [tool], host: stubHost() }) + .dispatch( + call('open_publish', { + 'sk-AAAAAAAAAAAAAAAAAAAA': 'rm -rf build', + 'sk-BBBBBBBBBBBBBBBBBBBB': 'echo done', + }), + { ...context, effects: { ...context.effects, prepare } }, + ) + .catch((error: unknown) => error); + expect(err, 'SECRETS_B8_CORE_KEY_COLLISION').toBeInstanceOf(ToolArgsInvalidError); + if (!(err instanceof ToolArgsInvalidError)) throw new Error('missing refusal'); + expect(err.fields, 'SECRETS_B8_CORE_KEY_COLLISION').toEqual(['[redacted]']); + expect(err.message.includes('sk-'), 'SECRETS_B8_CORE_KEY_COLLISION').toBe(false); + expect(prepare, 'SECRETS_B8_CORE_KEY_COLLISION').not.toHaveBeenCalled(); + expect(dispatch, 'SECRETS_B8_CORE_KEY_COLLISION').not.toHaveBeenCalled(); + }); + + it('refuses a real tiered effect whose arguments never settle', async () => { + // Four glued ids need four passes, one more than the projection's bound. + const ids = 'AKIAAAAAAAAAAAAAAAAAASIABBBBBBBBBBBBBBBB'.repeat(2); + const writeFile = vi.fn['writeFile']>((path, data) => + Promise.resolve({ path, bytesWritten: data.length }), + ); + const host = stubHost(); + if (host.fs === undefined) throw new Error('missing stub fs'); + const context = ctx(); + const prepare = vi.fn(context.effects.prepare); + const err: unknown = await registry(stubHost({ fs: { ...host.fs, writeFile } })) + .dispatch(call('write_file', { path: './keys.txt', content: `${ids}\n`, append: true }), { + ...context, + effects: { ...context.effects, prepare }, + }) + .catch((error: unknown) => error); + expect(err, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toBeInstanceOf(ToolArgsInvalidError); + if (!(err instanceof ToolArgsInvalidError)) throw new Error('missing refusal'); + expect(err.fields, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toEqual(['content']); + expect(prepare, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').not.toHaveBeenCalled(); + expect(writeFile, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').not.toHaveBeenCalled(); + }); + + it('projects an exactly admitted-size quoted write after the write lands', async () => { + const size = 8 * 1024 * 1024; + const writeFile = vi.fn['writeFile']>((path, data) => + Promise.resolve({ path, bytesWritten: data.length }), + ); + const host = stubHost(); + if (host.fs === undefined) throw new Error('missing stub fs'); + const out = await registry(stubHost({ fs: { ...host.fs, writeFile } })).dispatch( + call('write_file', { path: './out.txt', content: `password="${'a'.repeat(size - 11)}"` }), + ctx(), + ); + expect(writeFile, 'SECRETS_B7_CORE_QUOTED_SCAN').toHaveBeenCalledTimes(1); + expect(out.events.call.toolInput, 'SECRETS_B7_CORE_QUOTED_SCAN').toEqual({ + path: './out.txt', + content: '[redacted]', + }); + }, 15_000); +}); diff --git a/packages/core/src/tools/registry.ts b/packages/core/src/tools/registry.ts index 51454c0b..13bdb635 100644 --- a/packages/core/src/tools/registry.ts +++ b/packages/core/src/tools/registry.ts @@ -19,6 +19,7 @@ import { import { boundForModel, + projectSecretShapedValue, redactInlineMedia, redactSecretShapedText, redactSecretShapedValue, @@ -338,13 +339,24 @@ async function prepareEffect( effective: Readonly>, ctx: ToolDispatchContext, ): Promise { - try { - const args = [ - ctx.effectSlot, + const journal = projectInput(def, effective, ctx.secretArgKeys, 'journal'); + if (journal.unsettled.length > 0) { + // A value whose redacted projection never settles projects to `[redacted]`, which any other such + // value shares, and two member names that project alike leave one member: either way the digest + // could replay one call's result for a different call (§4), so the effect is refused before + // anything is journaled. The model can correct the argument. An open schema lets the model choose + // an argument's name, so the names are masked like any other text. + const fields = [...new Set(journal.unsettled.map(redactSecretShapedText))]; + throw new ToolArgsInvalidError( def.id, - tier, - sanitizeInput(def, effective, ctx.secretArgKeys, 'journal'), - ] as const; + fields, + `tool \`${def.id}\`: credential-shaped text in ${fields.join( + ', ', + )} has no stable, distinct redacted form, so this effect cannot be given a journal identity`, + ); + } + try { + const args = [ctx.effectSlot, def.id, tier, journal.input] as const; return await (ctx.effectAttempt === undefined ? ctx.effects.prepare(...args) : ctx.effects.prepare(...args, undefined, ctx.effectAttempt)); @@ -1061,6 +1073,19 @@ function sanitizeInput( secretArgKeys: ReadonlySet | undefined, purpose: 'display' | 'journal', ): Readonly> { + return projectInput(def, effective, secretArgKeys, purpose).input; +} + +/** + * `sanitizeInput`'s projection, with the top-level arguments whose journal projection did not settle + * (always none for the display projection, which needs no identity). + */ +function projectInput( + def: ToolDef, + effective: Readonly>, + secretArgKeys: ReadonlySet | undefined, + purpose: 'display' | 'journal', +): { readonly input: Readonly>; readonly unsettled: readonly string[] } { const out: Record = { ...effective }; for (const key of def.configOnlyParams ?? []) { delete out[key]; @@ -1094,12 +1119,27 @@ function sanitizeInput( // A display preview may omit a tail; the journal's collision guard may not. Both still exclude // declared secrets and credential/media-shaped values BEFORE the host computes its digest. // Recognition needs the complete value: clipping may remove a quote or URL authority separator. - const walked = redactSecretShapedValue(redactInlineMedia(out)); + const media = redactInlineMedia(out); + const journal = purpose === 'journal' ? projectSecretShapedValue(media) : undefined; + const walked = journal === undefined ? redactSecretShapedValue(media) : journal.value; const projection = purpose === 'display' ? boundArgsForDisplay(walked) : walked; + // Name the arguments that did not settle (the walk itself reports only that one did not). + const unsettled = + journal === undefined || journal.settled || !isPlainRecord(media) + ? [] + : Object.keys(media) + .filter((key) => !projectSecretShapedValue({ [key]: media[key] }).settled) + .toSorted(byCodeUnit); // The walk preserves plain-object shape, so this is the same record with scrubbed members; the guard is // a boundary check rather than a cast, and the fallback can only be reached if the walk ever stops // returning an object for an object input. - return isPlainRecord(projection) ? projection : out; + return { + input: isPlainRecord(projection) ? projection : out, + unsettled: + journal?.settled === false && unsettled.length === 0 + ? Object.keys(out).toSorted(byCodeUnit) + : unsettled, + }; } /** diff --git a/packages/db/src/media-store-marker-acquisition.test.ts b/packages/db/src/media-store-marker-acquisition.test.ts index f9dd7bac..368d7958 100644 --- a/packages/db/src/media-store-marker-acquisition.test.ts +++ b/packages/db/src/media-store-marker-acquisition.test.ts @@ -104,7 +104,9 @@ function claim(): string { const old = new Date(Date.now() - 25 * 60 * 60 * 1000); utimesSync(blob, old, old); const info = statSync(blob, { bigint: true }); - const name = `media-claim-v2.${absentPid}.${randomUUID()}.${info.dev.toString(36)}.${info.ino.toString(36)}.${absentPid}.${randomUUID()}`; + // Production's token: Node reports a top-bit file ID as a negative bigint, encoded unsigned. + const token = (value: bigint): string => BigInt.asUintN(64, value).toString(36); + const name = `media-claim-v2.${absentPid}.${randomUUID()}.${token(info.dev)}.${token(info.ino)}.${absentPid}.${randomUUID()}`; const directory = join(root, name); renameSync(staging, directory); return directory; diff --git a/packages/db/src/media-store-native-identity.test.ts b/packages/db/src/media-store-native-identity.test.ts new file mode 100644 index 00000000..876ed9e3 --- /dev/null +++ b/packages/db/src/media-store-native-identity.test.ts @@ -0,0 +1,339 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + renameSync, + rmSync, + statSync, + symlinkSync, + utimesSync, + writeFileSync, + type BigIntStats, + type StatOptions, +} from 'node:fs'; +import type { FileHandle } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { basename, join } from 'node:path'; + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { FilesystemMediaStore } from './media-store.js'; +import { MediaStoreFilesystemError, MediaStorePaths } from './media-store-paths.js'; +import { scanMediaFile } from './media-store-read.js'; + +interface IdentityFixture { + readonly fakes: Map; + readonly mkdirs: string[]; + beforeLstat: ((path: string) => void) | undefined; + afterOpen: ((path: string, file: FileHandle) => void) | undefined; + substitute: (info: unknown, exact: BigIntStats | undefined) => void; +} + +/** + * POSIX inodes stay far below 2^53, so these cases substitute every identity the store reads: each real + * (dev, ino) becomes 2^60 + k. Distinct substitutes stay distinct as bigints but share one Number (the + * spacing there is 256), so a read without `{ bigint: true }` or a comparison through Number lets a real + * replacement compare equal, as a 64-bit Windows file ID can. + */ +const fixture = vi.hoisted((): IdentityFixture => { + const fakes = new Map(); + return { + fakes, + mkdirs: [], + beforeLstat: undefined, + afterOpen: undefined, + substitute: (info, exact) => { + if (exact === undefined || typeof info !== 'object' || info === null || !('ino' in info)) + return; + const key = `${exact.dev}:${exact.ino}`; + const value = fakes.get(key) ?? (1n << 60n) + BigInt(fakes.size); + fakes.set(key, value); + Object.defineProperty(info, 'ino', { + value: typeof info.ino === 'bigint' ? value : Number(value), + }); + }, + }; +}); +vi.mock('node:fs', async (original) => { + const native = await original(); + return { + ...native, + lstatSync: (...args: Parameters) => { + fixture.beforeLstat?.(String(args[0])); + const info = native.lstatSync(...args); + fixture.substitute(info, native.lstatSync(args[0], { bigint: true, throwIfNoEntry: false })); + return info; + }, + statSync: (...args: Parameters) => { + const info = native.statSync(...args); + fixture.substitute(info, native.statSync(args[0], { bigint: true, throwIfNoEntry: false })); + return info; + }, + }; +}); +vi.mock('node:fs/promises', async (original) => { + const native = await original(); + return { + ...native, + open: async (...args: Parameters) => { + const file = await native.open(...args); + const stat = file.stat.bind(file); + Object.defineProperty(file, 'stat', { + configurable: true, + value: async (options?: StatOptions) => { + const info = await stat(options); + fixture.substitute(info, await stat({ bigint: true })); + return info; + }, + }); + fixture.afterOpen?.(String(args[0]), file); + return file; + }, + mkdir: (...args: Parameters) => { + fixture.mkdirs.push(String(args[0])); + return native.mkdir(...args); + }, + }; +}); + +const bytes = Buffer.from('exact native identity media fixtures'); +const digest = createHash('sha256').update(bytes).digest('hex'); +let base: string; +let root: string; +beforeEach(() => { + fixture.fakes.clear(); + fixture.mkdirs.length = 0; + base = realpathSync.native(mkdtempSync(join(tmpdir(), 'relavium-media-native-identity-'))); + root = join(base, 'store'); +}); +afterEach(() => { + fixture.beforeLstat = undefined; + fixture.afterOpen = undefined; + vi.restoreAllMocks(); + rmSync(base, { recursive: true, force: true }); +}); + +/** The fixture must make the original and its replacement collide as Numbers and only as Numbers. */ +function expectNumberCollision(original: bigint, replacement: bigint): void { + expect(replacement).not.toBe(original); + expect(Number(replacement)).toBe(Number(original)); +} + +function refusal(action: () => unknown): string | undefined { + try { + action(); + return undefined; + } catch (error) { + if (error instanceof MediaStoreFilesystemError) return error.reason; + throw error; + } +} + +async function outcome(action: Promise): Promise { + return action.then( + () => 'completed', + (error: unknown) => (error instanceof MediaStoreFilesystemError ? error.reason : 'other'), + ); +} + +/** An old producer temporary whose owner the sweep sees as gone. */ +function abandonedTemporary(): string { + mkdirSync(root, { mode: 0o700 }); + const source = join(root, `media-v1.${process.pid}.${randomUUID()}.tmp`); + writeFileSync(source, new Uint8Array([5, 6]), { mode: 0o600 }); + age(source); + return source; +} +function age(path: string): void { + const old = (Date.now() - 2 * 24 * 60 * 60 * 1000) / 1000; + utimesSync(path, old, old); +} +function absentOwners(onCheck: () => void = () => undefined): void { + vi.spyOn(process, 'kill').mockImplementation(() => { + onCheck(); + throw Object.assign(new Error('fixture absent owner'), { code: 'ESRCH' }); + }); +} + +describe('the media store root pin compares exact native identities', () => { + it('refuses a replaced root directory whose identity collides as a Number', () => { + const paths = new MediaStorePaths(root); + expect(paths.root(true)).toBe(root); + const original = statSync(root, { bigint: true }).ino; + // Keep the original alive so the platform cannot recycle its inode for the replacement. + renameSync(root, `${root}-old`); + mkdirSync(root); + expectNumberCollision(original, statSync(root, { bigint: true }).ino); + expect( + refusal(() => paths.root(false)), + 'CAS_ROOT_PIN_PRECISION', + ).toBe('unsafe_path'); + }); + + it('refuses an unchanged root alias whose target was replaced (the canonical identity)', () => { + const alias = join(base, 'alias'); + mkdirSync(root); + symlinkSync(root, alias, 'junction'); + const paths = new MediaStorePaths(alias); + expect(paths.root(false)).toBe(root); + const original = statSync(root, { bigint: true }).ino; + const entry = lstatSync(alias, { bigint: true }).ino; + renameSync(root, `${root}-old`); + mkdirSync(root); + expectNumberCollision(original, statSync(root, { bigint: true }).ino); + expect(lstatSync(alias, { bigint: true }).ino).toBe(entry); + expect( + refusal(() => paths.root(false)), + 'CAS_ROOT_PIN_PRECISION', + ).toBe('unsafe_path'); + }); + + it('refuses a replaced root alias whose target is unchanged (the entry identity)', () => { + const alias = join(base, 'alias'); + mkdirSync(root); + symlinkSync(root, alias, 'junction'); + const paths = new MediaStorePaths(alias); + expect(paths.root(false)).toBe(root); + const canonical = statSync(root, { bigint: true }).ino; + const original = lstatSync(alias, { bigint: true }).ino; + renameSync(alias, `${alias}-old`); + symlinkSync(root, alias, 'junction'); + expectNumberCollision(original, lstatSync(alias, { bigint: true }).ino); + expect(statSync(root, { bigint: true }).ino).toBe(canonical); + expect( + refusal(() => paths.root(false)), + 'CAS_ROOT_PIN_PRECISION', + ).toBe('unsafe_path'); + }); +}); + +describe('media ownership compares exact native identities', () => { + it('refuses to publish a producer temporary replaced after its exclusive open', async () => { + const store = new FilesystemMediaStore(root); + const moved = join(base, 'producer-original'); + let temp: string | undefined; + let identities: { original: bigint; replacement: bigint } | undefined; + fixture.afterOpen = (path) => { + if (!basename(path).startsWith('media-v1.')) return; + fixture.afterOpen = undefined; + temp = path; + // The swap lands after the producer's exclusive open and before its first ownership check. + renameSync(path, moved); + writeFileSync(path, 'replacement', { mode: 0o600 }); + identities = { + original: lstatSync(moved, { bigint: true }).ino, + replacement: lstatSync(path, { bigint: true }).ino, + }; + }; + const result = await outcome(store.put(bytes, 'audio/L16')); + if (temp === undefined || identities === undefined) + throw new Error('the producer was not swapped'); + expectNumberCollision(identities.original, identities.replacement); + const left = existsSync(temp) ? readFileSync(temp, 'utf8') : undefined; + const published = existsSync(join(root, digest.slice(0, 2), digest.slice(2))); + expect({ result, left, published }, 'CAS_OWNED_IDENTITY_PRECISION').toEqual({ + result: 'unsafe_path', + left: 'replacement', + published: false, + }); + }); + + it('refuses to claim into a replaced claim directory', async () => { + const source = abandonedTemporary(); + const moved = join(base, 'claim-original'); + let identities: { original: bigint; replacement: bigint } | undefined; + // The sweep's first ownership check of its source runs after the claim directory is pinned. + fixture.beforeLstat = (path) => { + if (path !== source) return; + const name = readdirSync(root).find((entry) => entry.startsWith('media-claim-v2.')); + if (name === undefined) return; + fixture.beforeLstat = undefined; + const dir = join(root, name); + const original = lstatSync(dir, { bigint: true }).ino; + renameSync(dir, moved); + mkdirSync(dir, { mode: 0o700 }); + identities = { original, replacement: lstatSync(dir, { bigint: true }).ino }; + }; + absentOwners(); + const result = await new FilesystemMediaStore(root).sweepTemporaries(); + if (identities === undefined) throw new Error('the claim directory was not swapped'); + expectNumberCollision(identities.original, identities.replacement); + expect( + { result, source: existsSync(source), moved: readdirSync(moved) }, + 'MEDIA_TEMP_SAME_PRECISION', + ).toEqual({ + result: { removed: 0, retained: 1, reasons: { unsafe_entry: 1 } }, + source: true, + moved: [], + }); + }); + + it('refuses a producer temporary replaced before its claim begins', async () => { + const source = abandonedTemporary(); + const moved = join(base, 'producer-original'); + let identities: { original: bigint; replacement: bigint } | undefined; + // The owner check runs after the sweep's snapshot of the temporary and before its claim. + absentOwners(() => { + if (identities !== undefined) return; + renameSync(source, moved); + writeFileSync(source, 'replacement', { mode: 0o600 }); + age(source); + identities = { + original: lstatSync(moved, { bigint: true }).ino, + replacement: lstatSync(source, { bigint: true }).ino, + }; + }); + const result = await new FilesystemMediaStore(root).sweepTemporaries(); + if (identities === undefined) throw new Error('the producer was not swapped'); + expectNumberCollision(identities.original, identities.replacement); + // nativeFile refuses before any claim directory exists; the later owned-file check would not. + expect( + { + result, + claims: fixture.mkdirs.filter((path) => basename(path).startsWith('media-claim-v2.')), + left: readFileSync(source, 'utf8'), + }, + 'MEDIA_TEMP_NATIVE_FILE_PRECISION', + ).toEqual({ + result: { removed: 0, retained: 1, reasons: { unsafe_entry: 1 } }, + claims: [], + left: 'replacement', + }); + }); +}); + +describe('a media read compares its descriptor stats exactly', () => { + it('refuses a second descriptor stat whose identity differs only beyond a Number (no real filesystem does this)', async () => { + const store = new FilesystemMediaStore(root); + expect(await store.put(bytes, 'audio/L16')).toBe(`media://sha256-${digest}`); + const path = join(root, digest.slice(0, 2), digest.slice(2)); + let identities: { first: bigint; second: bigint } | undefined; + fixture.afterOpen = (opened, file) => { + if (opened !== path) return; + fixture.afterOpen = undefined; + const stat = file.stat.bind(file); + let first: bigint | undefined; + Object.defineProperty(file, 'stat', { + value: async (options?: StatOptions) => { + const info = await stat(options); + if (typeof info.ino !== 'bigint') return info; + if (first === undefined) first = info.ino; + else { + Object.defineProperty(info, 'ino', { value: first + 1n }); + identities = { first, second: first + 1n }; + } + return info; + }, + }); + }; + const result = await outcome(scanMediaFile(new MediaStorePaths(root), digest, false)); + if (identities === undefined) throw new Error('the second stat was not reached'); + expectNumberCollision(identities.first, identities.second); + expect(result, 'MEDIA_FD_IDENTITY_PRECISION').toBe('unsafe_path'); + }); +}); diff --git a/packages/db/src/media-store-paths.test.ts b/packages/db/src/media-store-paths.test.ts index 3b06059d..8f7a13d2 100644 --- a/packages/db/src/media-store-paths.test.ts +++ b/packages/db/src/media-store-paths.test.ts @@ -1,4 +1,4 @@ -import { createHash } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; import { existsSync, linkSync, @@ -184,6 +184,24 @@ it('owns put bytes before awaiting and preserves concurrent atomic/idempotent pu expect((await store.listHandles()).map((entry) => entry.handle)).toEqual([handle]); }); +it('retains a regular file under a claim name as an unsafe entry', async () => { + const name = `media-claim-v2.2147483647.${randomUUID()}.1.2.2147483647.${randomUUID()}`; + writeFileSync(join(root, name), 'not a claim directory'); + vi.spyOn(process, 'kill').mockImplementation(() => { + throw Object.assign(new Error('fixture absent owner'), { code: 'ESRCH' }); + }); + try { + expect(await new FilesystemMediaStore(root).sweepTemporaries()).toEqual({ + removed: 0, + retained: 1, + reasons: { unsafe_entry: 1 }, + }); + } finally { + vi.restoreAllMocks(); + } + expect(readFileSync(join(root, name), 'utf8')).toBe('not a claim directory'); +}); + it('preserves never-created read/GC semantics without provisioning a root', async () => { const missing = join(base, 'not-yet-created'); const store = new FilesystemMediaStore(missing); diff --git a/packages/db/src/media-store-paths.ts b/packages/db/src/media-store-paths.ts index 7fb5c534..040066f6 100644 --- a/packages/db/src/media-store-paths.ts +++ b/packages/db/src/media-store-paths.ts @@ -1,6 +1,13 @@ import { RelaviumError } from '@relavium/shared'; -import { lstatSync, mkdirSync, realpathSync, statSync, type Stats } from 'node:fs'; +import { + lstatSync, + mkdirSync, + realpathSync, + statSync, + type BigIntStats, + type Stats, +} from 'node:fs'; import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { transferStorageFailure, withOwnedFilesystem } from './storage-failure-facts.js'; @@ -27,13 +34,49 @@ function within(root: string, path: string): boolean { interface RootPin { readonly canonical: string; - readonly dev: number; - readonly ino: number; - readonly entryDev: number; - readonly entryIno: number; + readonly dev: bigint; + readonly ino: bigint; + readonly entryDev: bigint; + readonly entryIno: bigint; readonly linked: boolean; } +/** + * One stat with an exact native identity. A 64-bit file ID (NTFS, overlayfs xino) exceeds a Number's + * precision, so `dev`/`ino` stay bigint; `nlink` and `size` are exact as Numbers for every value the + * guards admit, and `mtimeMs` repeats Node's number formula so it keeps its sub-millisecond fraction. + */ +export interface ExactStats { + readonly dev: bigint; + readonly ino: bigint; + readonly nlink: number; + readonly size: number; + readonly mtimeMs: number; + isFile(): boolean; + isDirectory(): boolean; + isSymbolicLink(): boolean; +} + +const NS_PER_SECOND = 1_000_000_000n; + +/** Project one bigint stat; a second stat of a path could observe a different file. */ +export function exactStats(info: BigIntStats): ExactStats { + const file = info.isFile(); + const directory = info.isDirectory(); + const link = info.isSymbolicLink(); + const seconds = info.mtimeNs / NS_PER_SECOND - (info.mtimeNs % NS_PER_SECOND < 0n ? 1n : 0n); + return Object.freeze({ + dev: info.dev, + ino: info.ino, + nlink: Number(info.nlink), + size: Number(info.size), + mtimeMs: Number(seconds) * 1000 + Number(info.mtimeNs - seconds * NS_PER_SECOND) / 1e6, + isFile: () => file, + isDirectory: () => directory, + isSymbolicLink: () => link, + }); +} + /** * Pin the root on its first admitted operation. Reads/GC do not provision a never-used store. * Once pinned, absence/replacement/retargeting revokes it; no later operation silently re-pins. @@ -50,18 +93,18 @@ export class MediaStorePaths { root(create: boolean): string | undefined { try { - let entry: Stats; + let entry: BigIntStats; try { - entry = withOwnedFilesystem(() => lstatSync(this.#lexical)); + entry = withOwnedFilesystem(() => lstatSync(this.#lexical, { bigint: true })); } catch (error) { if (!missing(error)) throw error; if (this.#pin !== undefined) throw new MediaStoreFilesystemError('unsafe_path'); if (!create) return undefined; withOwnedFilesystem(() => mkdirSync(this.#lexical, { recursive: true, mode: 0o700 })); - entry = withOwnedFilesystem(() => lstatSync(this.#lexical)); + entry = withOwnedFilesystem(() => lstatSync(this.#lexical, { bigint: true })); } const canonical = withOwnedFilesystem(() => realpathSync(this.#lexical)); - const info = withOwnedFilesystem(() => statSync(canonical)); + const info = withOwnedFilesystem(() => statSync(canonical, { bigint: true })); if (!info.isDirectory()) throw new MediaStoreFilesystemError('unsafe_path'); const current = { canonical, @@ -155,7 +198,7 @@ export class MediaStorePaths { } } - file(path: string): Stats | undefined { + file(path: string): ExactStats | undefined { const info = this.fileSnapshot(path); // fileSnapshot already excludes additional links; ownership also excludes detachment. if (info !== undefined && info.nlink === 0) throw new MediaStoreFilesystemError('unsafe_path'); @@ -167,10 +210,10 @@ export class MediaStorePaths { * between the kernel's lookup and attribute read. Accept that zero-link snapshot only * for non-owning reads/publication checks; temporary ownership and deletion stay strict. */ - fileSnapshot(path: string): Stats | undefined { + fileSnapshot(path: string): ExactStats | undefined { this.assertParent(path); try { - const info = withOwnedFilesystem(() => lstatSync(path)); + const info = exactStats(withOwnedFilesystem(() => lstatSync(path, { bigint: true }))); if (!info.isFile() || info.isSymbolicLink() || (info.nlink !== 0 && info.nlink !== 1)) throw new MediaStoreFilesystemError('unsafe_path'); return info; @@ -180,7 +223,7 @@ export class MediaStorePaths { } } - assertOwnedFile(path: string, opened: Stats): void { + assertOwnedFile(path: string, opened: ExactStats): void { const current = this.file(path); if ( !opened.isFile() || @@ -202,7 +245,7 @@ export class MediaStorePaths { * owner-controlled-parent assumption; it is not a descriptor-relative parent-swap jail. * Temporary-file ownership and destructive operations must not use this read-only check. */ - assertReadableFile(path: string, opened: Stats): void { + assertReadableFile(path: string, opened: ExactStats): void { const current = this.fileSnapshot(path); if ( !opened.isFile() || diff --git a/packages/db/src/media-store-read.ts b/packages/db/src/media-store-read.ts index 4f24e63d..1f7c2604 100644 --- a/packages/db/src/media-store-read.ts +++ b/packages/db/src/media-store-read.ts @@ -10,7 +10,7 @@ import { type DurableMediaMeta, } from '@relavium/shared'; -import { MediaStoreFilesystemError, MediaStorePaths } from './media-store-paths.js'; +import { exactStats, MediaStoreFilesystemError, MediaStorePaths } from './media-store-paths.js'; /** Internal typed cancellation; caller-provided abort reasons never become diagnostics. */ class MediaReadCancelledError extends Error { @@ -66,9 +66,11 @@ export async function scanMediaFile( let cleanupError: MediaStoreFilesystemError | undefined; try { checkAbort(); - const opened = await file.stat().catch((error: unknown) => { - throw paths.error(error); - }); + const opened = exactStats( + await file.stat({ bigint: true }).catch((error: unknown) => { + throw paths.error(error); + }), + ); if (opened.dev !== admitted.dev) throw new MediaStoreFilesystemError('unsafe_path'); paths.assertReadableFile(path, opened); // This check precedes even the fixed read buffer and every whole-body allocation. @@ -118,9 +120,11 @@ export async function scanMediaFile( } total += bytesRead; } - const finished = await file.stat().catch((error: unknown) => { - throw paths.error(error); - }); + const finished = exactStats( + await file.stat({ bigint: true }).catch((error: unknown) => { + throw paths.error(error); + }), + ); checkAbort(); if (finished.dev !== opened.dev || finished.ino !== opened.ino) throw new MediaStoreFilesystemError('unsafe_path'); diff --git a/packages/db/src/media-store-temporaries.test.ts b/packages/db/src/media-store-temporaries.test.ts index 1d01da05..038ad89a 100644 --- a/packages/db/src/media-store-temporaries.test.ts +++ b/packages/db/src/media-store-temporaries.test.ts @@ -232,15 +232,22 @@ async function exactIdentityClaims(): Promise { result: { removed: 0, retained: 1, reasons: { identity_changed: 1 } }, exists: true, }); - for (const [index, value] of [ - BigInt(Number.MAX_SAFE_INTEGER), - first, - first + 1n, - 11193352636847878000n, - (1n << 64n) - 1n, - ].entries()) { + // Literal tokens: an encoding that maps a negative identity onto another value's token fails here. + const exact: ReadonlyArray = [ + [BigInt(Number.MAX_SAFE_INTEGER), '2gosa7pa2gv'], + [first, '2gosa7pa2gw'], + [first + 1n, '2gosa7pa2gx'], + [11193352636847878000n, '2d1i8ct3mhd9s'], + [(1n << 64n) - 1n, '3w5e11264sgsf'], + // Node's signed bigint stats: a 64-bit file ID with its top bit set reads negative, and its + // token is the unsigned value the filesystem reports (NI). + [-1n, '3w5e11264sgsf'], + [-(1n << 63n), '1y2p0ij32e8e8'], + ]; + for (const [index, [value, token]] of exact.entries()) { const label = `exact-${index}`; const claim = await makeClaim(label, value); + expect(basename(claim.dir).split('.')[4], 'MEDIA_TEMP_NATIVE_IDENTITY_TOKEN').toBe(token); expect( await new FilesystemMediaStore(join(base, label)).sweepTemporaries(), 'MEDIA_TEMP_NATIVE_IDENTITY_RECOVERY', diff --git a/packages/db/src/media-store-temporaries.ts b/packages/db/src/media-store-temporaries.ts index 2af51f62..ed924a9a 100644 --- a/packages/db/src/media-store-temporaries.ts +++ b/packages/db/src/media-store-temporaries.ts @@ -1,11 +1,16 @@ import { RelaviumError } from '@relavium/shared'; import { randomUUID } from 'node:crypto'; -import { constants, lstatSync, readdirSync, realpathSync, type Stats } from 'node:fs'; +import { constants, lstatSync, readdirSync, realpathSync } from 'node:fs'; import { mkdir, open, rename, rmdir, unlink } from 'node:fs/promises'; import { join } from 'node:path'; -import { MediaStoreFilesystemError, MediaStorePaths } from './media-store-paths.js'; +import { + exactStats, + MediaStoreFilesystemError, + MediaStorePaths, + type ExactStats, +} from './media-store-paths.js'; import { withOwnedFilesystem, withOwnedFilesystemAsync } from './storage-failure-facts.js'; export type MediaTemporaryRetentionReason = @@ -45,6 +50,7 @@ const CLAIM = new RegExp( const LEGACY_ROOT = new RegExp(`^stream\\.${UUID}\\.tmp$`); const LEGACY_SHARD = new RegExp(`^[0-9a-f]{62}\\.${UUID}\\.tmp$`); const MIN_AGE = 24 * 60 * 60 * 1000; +const MIN_NATIVE_IDENTITY = -(1n << 63n); const MAX_NATIVE_IDENTITY = (1n << 64n) - 1n; const MAX_NATIVE_TOKEN = MAX_NATIVE_IDENTITY.toString(36); @@ -53,8 +59,11 @@ interface NativeIdentity { readonly ino: string; } function nativeIdentityToken(value: bigint): string { - if (value < 0n || value > MAX_NATIVE_IDENTITY) throw new MediaStoreFilesystemError('unsafe_path'); - return value.toString(36); + if (value < MIN_NATIVE_IDENTITY || value > MAX_NATIVE_IDENTITY) + throw new MediaStoreFilesystemError('unsafe_path'); + // Node fills bigint stats from a signed array, so a 64-bit file ID with its top bit set reads + // negative; the token is the unsigned value the filesystem reports. + return BigInt.asUintN(64, value).toString(36); } function canonicalNativeToken(field: string): boolean { return ( @@ -80,10 +89,10 @@ function live(pid: string): Liveness { function code(error: unknown): unknown { return typeof error === 'object' && error !== null && 'code' in error ? error.code : undefined; } -function same(a: Stats, b: Stats): boolean { +function same(a: ExactStats, b: ExactStats): boolean { return a.dev === b.dev && a.ino === b.ino; } -function age(info: Stats, now: number): MediaTemporaryRetentionReason | undefined { +function age(info: ExactStats, now: number): MediaTemporaryRetentionReason | undefined { if (!Number.isFinite(info.mtimeMs) || info.mtimeMs > now) return 'future_timestamp'; return now - info.mtimeMs < MIN_AGE ? 'too_young' : undefined; } @@ -203,7 +212,7 @@ export class MediaStoreTemporaries { const marker = join(root, `media-maintenance-v1.${process.pid}.${randomUUID()}`); const identity = await this.createMarker(marker); try { - const candidates: Array<{ path: string; info: Stats }> = []; + const candidates: Array<{ path: string; info: ExactStats }> = []; // The marker is visible before this scan. New producers recheck after exclusive open; // old admitted producers remain visible here until publication/cleanup. for (const name of this.entries(root)) { @@ -262,7 +271,7 @@ export class MediaStoreTemporaries { throw this.paths.error(error); } } - private async createMarker(path: string): Promise { + private async createMarker(path: string): Promise { this.paths.assertParent(path); return this.acquireMarker( path, @@ -273,9 +282,9 @@ export class MediaStoreTemporaries { /** Keep captured fd identity available when close or the post-close ownership check fails. */ private async acquireMarker( path: string, - assertOwned: (identity: Stats) => void, - cleanup: (identity: Stats) => Promise, - ): Promise { + assertOwned: (identity: ExactStats) => void, + cleanup: (identity: ExactStats) => Promise, + ): Promise { const file = await withOwnedFilesystemAsync(() => open( path, @@ -289,14 +298,18 @@ export class MediaStoreTemporaries { ).catch((error: unknown) => { throw this.paths.error(error); }); - let identity: Stats | undefined; + let identity: ExactStats | undefined; let acquired = false; let cleanupError: MediaStoreFilesystemError | undefined; try { try { - identity = await withOwnedFilesystemAsync(() => file.stat()).catch((error: unknown) => { - throw this.paths.error(error); - }); + identity = exactStats( + await withOwnedFilesystemAsync(() => file.stat({ bigint: true })).catch( + (error: unknown) => { + throw this.paths.error(error); + }, + ), + ); } finally { try { await withOwnedFilesystemAsync(() => file.close()); @@ -327,16 +340,16 @@ export class MediaStoreTemporaries { } } } - private async removeOwned(path: string, identity: Stats): Promise { + private async removeOwned(path: string, identity: ExactStats): Promise { this.paths.assertOwnedFile(path, identity); await withOwnedFilesystemAsync(() => unlink(path)).catch((error: unknown) => { throw this.paths.error(error); }); } - private directory(root: string, name: string, identity?: Stats): Stats { + private directory(root: string, name: string, identity?: ExactStats): ExactStats { this.paths.root(false); const path = join(root, name); - const info = withOwnedFilesystem(() => lstatSync(path)); + const info = exactStats(withOwnedFilesystem(() => lstatSync(path, { bigint: true }))); if ( !info.isDirectory() || info.isSymbolicLink() || @@ -347,10 +360,17 @@ export class MediaStoreTemporaries { this.paths.root(false); return info; } - private child(root: string, name: string, identity: Stats, child: string): Stats | undefined { + private child( + root: string, + name: string, + identity: ExactStats, + child: string, + ): ExactStats | undefined { this.directory(root, name, identity); try { - const info = withOwnedFilesystem(() => lstatSync(join(root, name, child))); + const info = exactStats( + withOwnedFilesystem(() => lstatSync(join(root, name, child), { bigint: true })), + ); if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1) throw new MediaStoreFilesystemError('unsafe_path'); return info; @@ -360,14 +380,14 @@ export class MediaStoreTemporaries { } } /** Exact claim identity supplements, rather than replaces, the existing cooperative path floor. */ - private nativeFile(path: string, expected: Stats): NativeIdentity { + private nativeFile(path: string, expected: ExactStats): NativeIdentity { const info = withOwnedFilesystem(() => lstatSync(path, { bigint: true })); if ( !info.isFile() || info.isSymbolicLink() || info.nlink !== 1n || - Number(info.dev) !== expected.dev || - Number(info.ino) !== expected.ino + info.dev !== expected.dev || + info.ino !== expected.ino ) throw new MediaStoreFilesystemError('unsafe_path'); return { dev: nativeIdentityToken(info.dev), ino: nativeIdentityToken(info.ino) }; @@ -375,7 +395,7 @@ export class MediaStoreTemporaries { private async claim( root: string, source: string, - info: Stats, + info: ExactStats, pid: string, nonce: string, ): Promise { @@ -513,9 +533,9 @@ export class MediaStoreTemporaries { private assertRecoveryMarker( root: string, name: string, - identity: Stats, + identity: ExactStats, marker: string, - markerInfo: Stats, + markerInfo: ExactStats, ): void { const own = this.child(root, name, identity, marker); if ( @@ -530,9 +550,9 @@ export class MediaStoreTemporaries { private async removeRecoveryMarker( root: string, name: string, - identity: Stats, + identity: ExactStats, marker: string, - markerInfo: Stats, + markerInfo: ExactStats, ): Promise { this.assertRecoveryMarker(root, name, identity, marker, markerInfo); await withOwnedFilesystemAsync(() => unlink(join(root, name, marker))); @@ -540,9 +560,9 @@ export class MediaStoreTemporaries { private async cleanupRecovery( root: string, name: string, - identity: Stats, + identity: ExactStats, marker: string, - markerInfo: Stats, + markerInfo: ExactStats, ): Promise { await this.removeRecoveryMarker(root, name, identity, marker, markerInfo); try { diff --git a/packages/db/src/media-store.ts b/packages/db/src/media-store.ts index 0a6d7655..4dc71425 100644 --- a/packages/db/src/media-store.ts +++ b/packages/db/src/media-store.ts @@ -1,9 +1,14 @@ import { createHash } from 'node:crypto'; -import { constants, type Stats } from 'node:fs'; +import { constants } from 'node:fs'; import { open, readdir, rename, rm, type FileHandle } from 'node:fs/promises'; import { join } from 'node:path'; -import { MediaStoreFilesystemError, MediaStorePaths } from './media-store-paths.js'; +import { + exactStats, + MediaStoreFilesystemError, + MediaStorePaths, + type ExactStats, +} from './media-store-paths.js'; import { assertMediaNotAborted, scanMediaFile } from './media-store-read.js'; import { MediaStoreTemporaries, @@ -245,15 +250,17 @@ export class FilesystemMediaStore implements MediaStore { ).catch((error: unknown) => { throw this.#paths.error(error); }); - let identity: Stats | undefined; + let identity: ExactStats | undefined; let closed = false; let published = false; let result: string | undefined; let cleanupError: MediaStoreFilesystemError | undefined; try { - identity = await file.stat().catch((error: unknown) => { - throw this.#paths.error(error); - }); + identity = exactStats( + await file.stat({ bigint: true }).catch((error: unknown) => { + throw this.#paths.error(error); + }), + ); this.#paths.assertOwnedFile(tmp, identity); this.#temporaries.assertProducerAllowed(); const handle = await write(file); diff --git a/packages/llm/src/fixed-point.test.ts b/packages/llm/src/fixed-point.test.ts new file mode 100644 index 00000000..b8dd9759 --- /dev/null +++ b/packages/llm/src/fixed-point.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; + +import { toFixedPoint } from './fixed-point.js'; + +describe('toFixedPoint', () => { + const failed = '[FAILED]'; + const passes = (outputs: readonly string[]) => { + const calls: string[] = []; + const pass = (text: string) => { + calls.push(text); + return outputs[calls.length - 1] ?? text; + }; + return { pass, calls }; + }; + + it('returns an input the first pass leaves unchanged after one pass', () => { + const { pass, calls } = passes(['a']); + expect(toFixedPoint(pass, 'a', failed)).toBe('a'); + expect(calls).toEqual(['a']); + }); + + it('returns the first output once a second pass confirms it', () => { + const { pass, calls } = passes(['b', 'b']); + expect(toFixedPoint(pass, 'a', failed)).toBe('b'); + expect(calls).toEqual(['a', 'b']); + }); + + it('returns the second output once a third pass confirms it', () => { + const { pass, calls } = passes(['b', 'c', 'c']); + expect(toFixedPoint(pass, 'a', failed)).toBe('c'); + expect(calls).toEqual(['a', 'b', 'c']); + }); + + it('fails closed when the third pass still changes the text', () => { + const { pass, calls } = passes(['b', 'c', 'd']); + expect(toFixedPoint(pass, 'a', failed)).toBe(failed); + expect(calls).toEqual(['a', 'b', 'c']); + }); +}); diff --git a/packages/llm/src/fixed-point.ts b/packages/llm/src/fixed-point.ts new file mode 100644 index 00000000..e8dde636 --- /dev/null +++ b/packages/llm/src/fixed-point.ts @@ -0,0 +1,14 @@ +/** + * Repeats one masking pass until its output stops changing, at most three passes. A pass can glue a + * marker to the text after another rule's token, which the next pass reads anew, so a settled output + * is a fixed point, and an output that still changes on the third pass fails closed to `failed`. + * Callers choose a `failed` value the pass leaves unchanged, so the result is stable under another + * call either way. Dependency-free, so the pure `scrub-secrets` leaf can load it. + */ +export function toFixedPoint(pass: (text: string) => string, text: string, failed: string): string { + const once = pass(text); + if (once === text) return once; + const twice = pass(once); + if (twice === once) return once; + return pass(twice) === twice ? twice : failed; +} diff --git a/packages/llm/src/scrub-secrets.test.ts b/packages/llm/src/scrub-secrets.test.ts index 31aac2bb..3b8fd17e 100644 --- a/packages/llm/src/scrub-secrets.test.ts +++ b/packages/llm/src/scrub-secrets.test.ts @@ -1,13 +1,33 @@ -import { describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { toFixedPoint } from './fixed-point.js'; import { scrubSecrets as rootScrubSecrets } from './index.js'; import { makeLlmError, scrubSecrets as errorScrubSecrets } from './llm-error.js'; import { scrubSecrets } from './scrub-secrets.js'; +// Observe the scrubber's bounded fixed point, retaining its actual implementation, so one case can +// make it fail and see which value the scrubber fails closed with. +vi.mock('./fixed-point.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, toFixedPoint: vi.fn(actual.toFixedPoint) }; +}); + +// The observation must not keep admitted-size inputs and outputs alive for the whole file. +afterEach(() => { + vi.mocked(toFixedPoint).mockClear(); +}); + // Synthetic credentials assembled from fragments, never real provider material. const key = ['sk-', 'ant-', 'api03-', 'AbCdEf0123456789xyz'].join(''); const googleKey = ['AI', 'za', 'SyA1234567890abcdefghijklmnopqrstuv'].join(''); +/** + * Eight-MiB cases take about a second. Their bound is this case deadline, far below the minutes a + * quadratic scan would need, rather than a wall-clock assertion that a loaded runner can trip. It is + * also the largest case budget the secrets causal-control phases admit (`CAUSAL_TIMEOUT_BUDGET`). + */ +const ADMITTED_SIZE_CASE_TIMEOUT_MS = 15_000; + const compatibleDiagnostics = [ { input: `invalid ${key}; retry later`, expected: 'invalid [REDACTED]; retry later' }, { input: `invalid ${googleKey}; retry later`, expected: 'invalid [REDACTED]; retry later' }, @@ -83,8 +103,10 @@ describe('finite diagnostic credential backstop regressions (PR #95 M2)', () => expected: `connect ${scheme.toUpperCase()}://[REDACTED]@provider.invalid/v1 failed`, }, ]), - // A scheme glued to a preceding word character was masked before this PR; a `\b` on the rule - // let it through (PR #95 review, MEDIUM-1). Adjacent and LF/TAB-split forms are both pinned. + // A scheme glued to a preceding word character: the pre-PR matcher masked it for http(s), and a `\b` on + // the rule let it through (PR #95 review, MEDIUM-1). ws/wss/ftp/ftps are this PR's additions and get + // the same word-adjacency, so those cases pin new behaviour, not parity. Adjacent and LF/TAB-split + // forms are both pinned. ...['http', 'https', 'ws', 'wss', 'ftp', 'ftps'].flatMap((scheme) => [ { input: `x${scheme}://alice:opaque-password@provider.invalid/v1`, @@ -99,6 +121,29 @@ describe('finite diagnostic credential backstop regressions (PR #95 M2)', () => input: `xhttps://ali${separator}ce:opaque-password@provider.invalid/v1`, expected: '[REDACTED]', })), + // A glued, split credential that also contains a raw `@` used to leave a partial mask with the rest + // of the credential visible: the scanner skipped starts preceded by a word character (security + // sitting, PR #95 post-merge review). A clean-prefix twin was already redacted whole. + { + input: 'x\u0000https://u@e.com:pw\nxyz@provider.invalid/v1', + expected: '[REDACTED]', + }, + { + input: 'xhttps://user:p@ss\tword@provider.invalid/v1', + expected: '[REDACTED]', + }, + { + input: '_https://alice:opaque-password@provider.invalid/v1', + expected: '_https://[REDACTED]@provider.invalid/v1', + }, + { + input: 'xht\ntps://alice:opaque-password@provider.invalid/v1', + expected: '[REDACTED]', + }, + { + input: 'a http://u1:p1@h1.invalid/x and xhttps://u2:p2@h2.invalid/y', + expected: 'a http://[REDACTED]@h1.invalid/x and xhttps://[REDACTED]@h2.invalid/y', + }, ...['api-key', 'x-api-key', 'x-goog-api-key'].flatMap((name) => [ { input: `header ${name}: opaque-header-Z29 rejected`, @@ -116,9 +161,12 @@ describe('finite diagnostic credential backstop regressions (PR #95 M2)', () => ...['\n', '\t'].flatMap((separator) => [ { input: `Bear${separator}er opaque-header-Z29 rejected`, - expected: 'Bearer [REDACTED] rejected', + expected: `Bear${separator}er [REDACTED] rejected`, + }, + { + input: `Bas${separator}ic dXNlcjpwYXNz rejected`, + expected: `Bas${separator}ic [REDACTED] rejected`, }, - { input: `Bas${separator}ic dXNlcjpwYXNz rejected`, expected: 'Basic [REDACTED] rejected' }, { input: `wss://ali${separator}ce:opaque@password@provider.invalid/v1`, expected: '[REDACTED]', @@ -145,12 +193,33 @@ describe('finite diagnostic credential backstop regressions (PR #95 M2)', () => expect(error.code).toBe(expected); }); + // Documents, and does not endorse, two consequences of a URL rule with no word boundary. Both fail + // closed: they blank or widen a diagnostic rather than reveal one. The whitespace-stripped residual probe + // glues the prose before a URL to its scheme, and an unlisted longer scheme name matches by its suffix. + it.each([ + { + input: 'Error: ECONNREFUSED http://127.0.0.1:11434 contact admin@corp', + expected: '[REDACTED]', + }, + { + input: 'MCP server at ws://127.0.0.1:8080 refused; email support@vendor.com', + expected: '[REDACTED]', + }, + { + input: 'Error: connect ECONNREFUSED postgres://db.example:5432 contact dba@corp', + expected: '[REDACTED]', + }, + { input: 'gftp://scm@code.example/x', expected: 'gftp://[REDACTED]@code.example/x' }, + ])('fails closed on the documented trade-off $input', ({ input, expected }) => { + expect(scrubSecrets(input)).toBe(expected); + }); + it('protects each internal LF/TAB split position in the finite Bearer/Basic labels', () => { for (const label of ['Bearer', 'Basic']) { for (const separator of ['\n', '\t', '\n\t']) { for (let position = 1; position < label.length; position += 1) { - const input = `${label.slice(0, position)}${separator}${label.slice(position)} opaque-header-Z29`; - expect(scrubSecrets(input)).toBe(`${label} [REDACTED]`); + const split = `${label.slice(0, position)}${separator}${label.slice(position)}`; + expect(scrubSecrets(`${split} opaque-header-Z29`)).toBe(`${split} [REDACTED]`); } } } @@ -222,6 +291,9 @@ describe('PR95 B2 split provider/header and malformed userinfo diagnostics', () for (let index = 1; index < name.length; index += 1) { const input = `header ${name.slice(0, index)}${separator}${name.slice(index)}: opaque-Z29; retry later`; expectCredentialDiagnostic(scrubSecrets(input)).toBe('[REDACTED]'); + // JSON-encoded JSON; the inner mask keeps the whitespace-stripped residual from answering. + const escaped = `{\\"${name.slice(0, index)}${separator}${name.slice(index)}\\":\\"[REDACTED]opaque-Z29\\"}`; + expectCredentialDiagnostic(scrubSecrets(escaped)).toBe('[REDACTED]'); } } } @@ -282,16 +354,38 @@ describe('PR95 B2 finite joined-view boundaries', () => { }, ); - it('bounds admitted-size failed header and URL preparation', () => { - const started = performance.now(); - for (const fragment of ['wordx-\tapi-\nkey ', 'https:\n//provider.invalid/']) { - const input = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); - const output = scrubSecrets(input); - expect(output.length, 'SECRETS_B2_HEADER_PREPARATION').toBe(input.length); - expect(output === input, 'SECRETS_B2_HEADER_PREPARATION').toBe(true); - } - expect(performance.now() - started, 'SECRETS_B2_HEADER_PREPARATION').toBeLessThan(5_000); - }); + it( + 'bounds admitted-size failed header and URL preparation', + () => { + for (const fragment of ['wordx-\tapi-\nkey ', 'https:\n//provider.invalid/']) { + const input = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); + const output = scrubSecrets(input); + expect(output.length, 'SECRETS_B2_HEADER_PREPARATION').toBe(input.length); + expect(output === input, 'SECRETS_B2_HEADER_PREPARATION').toBe(true); + } + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it( + 'reaches the split URL scan over admitted-size letter-heavy text', + () => { + const letters = 8 * 1024 * 1024 - 256; + for (const filler of ['a', 'hwf']) { + const tail = filler.repeat(Math.floor(letters / filler.length)); + for (const [input, expected] of [ + [`a\nhttps://u@h ${tail}`, `a\nhttps://[REDACTED]@h ${tail}`], + [`${tail}\nhttps://u@h`, `${tail}\nhttps://[REDACTED]@h`], + ] as const) { + const output = scrubSecrets(input); + // Bounded evidence: a regression must not serialize eight MiB into its report. + expect(output.length, 'SECRETS_B2_URL_SCAN').toBe(expected.length); + expect(output === expected, 'SECRETS_B2_URL_SCAN').toBe(true); + } + } + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); it.each([ ['sk-proj-', '\n'], @@ -306,17 +400,19 @@ describe('PR95 B2 finite joined-view boundaries', () => { expect(error.code, 'SECRETS_B2_ORIGINAL_BOUNDARY').toBe('[REDACTED]'); }); - it('bounds admitted-size embedded prefixes before a valid original boundary', () => { - const fragment = `wordsk-proj-${'A'.repeat(52)}\n`; - const publicPrefix = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); - const input = `${publicPrefix}sk-proj-${'B'.repeat(32)}\t${'C'.repeat(32)}; public=A`; - const started = performance.now(); - const output = scrubSecrets(input); - // Keep removal evidence bounded: a regression must not serialize eight MiB into its report. - expect(output.length, 'SECRETS_B2_ORIGINAL_BOUNDARY').toBe('[REDACTED]'.length); - expect(output, 'SECRETS_B2_ORIGINAL_BOUNDARY').toBe('[REDACTED]'); - expect(performance.now() - started, 'SECRETS_B2_ORIGINAL_BOUNDARY').toBeLessThan(5_000); - }); + it( + 'bounds admitted-size embedded prefixes before a valid original boundary', + () => { + const fragment = `wordsk-proj-${'A'.repeat(52)}\n`; + const publicPrefix = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); + const input = `${publicPrefix}sk-proj-${'B'.repeat(32)}\t${'C'.repeat(32)}; public=A`; + const output = scrubSecrets(input); + // Keep removal evidence bounded: a regression must not serialize eight MiB into its report. + expect(output.length, 'SECRETS_B2_ORIGINAL_BOUNDARY').toBe('[REDACTED]'.length); + expect(output, 'SECRETS_B2_ORIGINAL_BOUNDARY').toBe('[REDACTED]'); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); it('does not invent a key boundary in a public word with a separate newline', () => { const input = `wordsk-proj-${'A'.repeat(32)};\nordinary`; @@ -349,3 +445,1000 @@ describe('PR95 B2 finite joined-view boundaries', () => { ).toBe('[REDACTED]\n request; retry later'); }); }); + +const COLONLESS_SCHEMES = ['http', 'https', 'ws', 'wss', 'ftp', 'ftps']; +const CONNECTION_SCHEMES = [ + 'postgres', + 'postgresql', + 'mysql', + 'mariadb', + 'redis', + 'rediss', + 'mongodb', + 'mongodb+srv', + 'amqp', + 'amqps', + 'ssh', + 'sftp', + 'ldap', + 'ldaps', + 'smtp', + 'smtps', +]; + +describe('PR95 header values that begin with a Bearer/Basic label', () => { + const token = ['abcdef', 'ghijkl', '123456'].join(''); + const headerSchemeLabel = (value: unknown) => expect(value, 'SECRETS_B2_HEADER_SCHEME_LABEL'); + + it.each(['api-key', 'x-api-key', 'x-goog-api-key', 'X-Api-Key'])( + 'masks the token after a scheme label in the %s header', + (name) => { + for (const label of ['Bearer', 'Basic', 'bearer', 'BASIC']) { + for (const [input, expected] of [ + [`${name}:${label} ${token}`, `${name}:${label} [REDACTED]`], + [ + `header ${name}: ${label} ${token} rejected`, + `header ${name}: ${label} [REDACTED] rejected`, + ], + [`${name}:\t${label}\t${token}`, `${name}:\t${label}\t[REDACTED]`], + // An opaque value after a label ends only at whitespace, so in compact JSON it fails closed + // to the end of the run, exactly as a standalone `"authorization":"Bearer …"` does. + [ + `{"${name}":"${label} ${token}","ordinary":"kept"}`, + `{"${name}":"${label} [REDACTED]"}`, + ], + ] as const) { + const output = scrubSecrets(input); + headerSchemeLabel(output).toBe(expected); + headerSchemeLabel(output.includes(token)).toBe(false); + } + } + }, + ); + + it('masks every LF/TAB split position inside a scheme label after a header', () => { + for (const label of ['Bearer', 'Basic']) { + for (const separator of ['\n', '\t', '\n\t']) { + for (let position = 1; position < label.length; position += 1) { + const split = `${label.slice(0, position)}${separator}${label.slice(position)}`; + headerSchemeLabel(scrubSecrets(`api-key:${split} ${token}`)).toBe( + `api-key:${split} [REDACTED]`, + ); + } + } + } + }); + + it('masks the next token, failing closed, when a scheme label ends the line', () => { + headerSchemeLabel(scrubSecrets('api-key: Bearer\nrequest failed')).toBe( + 'api-key: Bearer\n[REDACTED] failed', + ); + // A bare label is no token: it stays visible rather than become the masked value. + expect(scrubSecrets('api-key: Bearer')).toBe('api-key: Bearer'); + }); + + it('keeps a token masked when the scheme label precedes the header name', () => { + expect(scrubSecrets(`Bearer api-key: ${token}`)).toBe('Bearer [REDACTED] [REDACTED]'); + }); +}); + +describe('PR95 raw ?/# userinfo: public only after a host-shaped prefix and an assignment', () => { + const userinfoContact = (value: unknown) => expect(value, 'SECRETS_B2_USERINFO_CONTACT'); + + it.each([ + ['https://us?er:TOKEN@h.example/', 'https://[REDACTED]@h.example/'], + ['http://#TOKEN@h.example', 'http://[REDACTED]@h.example'], + ['https://us?er=1:TOKEN@h.example/', 'https://[REDACTED]@h.example/'], + ])('masks userinfo whose raw ?/# follows no host-shaped prefix: %s', (input, expected) => { + userinfoContact(scrubSecrets(input)).toBe(expected); + }); + + it.each([ + ['https://my.user:1234?TOKEN@h.example/', 'https://[REDACTED]@h.example/'], + ['https://host.example#sec@x.example', 'https://[REDACTED]@x.example'], + ])('masks a host-shaped prefix that no assignment follows: %s', (input, expected) => { + userinfoContact(scrubSecrets(input)).toBe(expected); + }); + + it.each(['?', '#'])('retains public %s contact text after a host-shaped prefix', (separator) => { + for (const host of [ + 'provider.invalid', + 'provider.invalid:8443', + 'provider.invalid.', + '10.0.0.1', + 'localhost:3000', + '[::1]', + '[::1]:8443', + '[::ffff:10.0.0.1]', + ]) { + const input = `connect https://${host}${separator}contact=alice@example.invalid failed`; + expect(scrubSecrets(input)).toBe(input); + } + }); + + it('masks a single-label host query, failing closed', () => { + expect(scrubSecrets('https://intranet?contact=alice@example.invalid')).toBe( + 'https://[REDACTED]@example.invalid', + ); + }); + + it.each(['\n', '\t'])('keeps the predicate across an LF/TAB split (%j)', (separator) => { + userinfoContact(scrubSecrets(`https://us?e${separator}r:TOKEN@h.example/`)).toBe('[REDACTED]'); + const contact = `https://provider.invalid?con${separator}tact=alice@example.invalid`; + expect(scrubSecrets(contact)).toBe(contact); + }); +}); + +describe('PR95 connection-string URL schemes', () => { + const connectionScheme = (value: unknown) => expect(value, 'SECRETS_B2_CONNECTION_SCHEME'); + + it.each(CONNECTION_SCHEMES)('masks colon-bearing %s userinfo and keeps a bare user', (scheme) => { + for (const spelling of [scheme, scheme.toUpperCase()]) { + connectionScheme(scrubSecrets(`connect ${spelling}://user:opaque@db.example/x failed`)).toBe( + `connect ${spelling}://[REDACTED]@db.example/x failed`, + ); + connectionScheme(scrubSecrets(`${spelling}://:opaque@db.example`)).toBe( + `${spelling}://[REDACTED]@db.example`, + ); + connectionScheme(scrubSecrets(`x${spelling}://user:opaque@db.example/x`)).toBe( + `x${spelling}://[REDACTED]@db.example/x`, + ); + const bareUser = `${spelling}://admin@db.example/x`; + expect(scrubSecrets(bareUser)).toBe(bareUser); + } + }); + + it('masks a multi-host connection string and keeps a git+ssh user', () => { + connectionScheme(scrubSecrets('mongodb://user:opaque@h1.example,h2.example/db')).toBe( + 'mongodb://[REDACTED]@h1.example,h2.example/db', + ); + expect(scrubSecrets('git+ssh://git@github.com/org/repo')).toBe( + 'git+ssh://git@github.com/org/repo', + ); + }); + + it('keeps masking colonless userinfo for the original schemes', () => { + for (const scheme of COLONLESS_SCHEMES) { + expect(scrubSecrets(`${scheme}://opaque@provider.invalid/x`)).toBe( + `${scheme}://[REDACTED]@provider.invalid/x`, + ); + } + }); +}); + +describe('PR95 joined-view and glued-scheme witnesses', () => { + // An inner pre-masked span splits the whitespace-stripped residual probe, so only the joined + // scanner can see each split credential below. + it.each([...COLONLESS_SCHEMES, ...CONNECTION_SCHEMES])( + 'masks a split %s credential around an inner masked span', + (scheme) => { + for (const separator of ['\n', '\t']) { + expect( + scrubSecrets(`${scheme}://u${separator}ser:pa[REDACTED]ss@h.example/x`), + 'SECRETS_B2_URL_JOINED', + ).toBe('[REDACTED]'); + } + }, + ); + + it.each(['?', '#'])( + 'masks a split credential holding a raw %s and an inner masked span', + (raw) => { + for (const separator of ['\n', '\t']) { + expect( + scrubSecrets(`https://u${separator}ser:pa${raw}ss[REDACTED]x@h.example/v1`), + 'SECRETS_B2_URL_JOINED', + ).toBe('[REDACTED]'); + } + }, + ); + + it.each(['\n', '\t'])( + 'masks a glued scheme whose split credential holds a raw @ (%j)', + (separator) => { + for (const scheme of ['https', 'postgres']) { + expect( + scrubSecrets(`x${scheme}://user:p@ss${separator}word@provider.invalid/v1`), + 'SECRETS_B2_URL_GLUED_SCHEME', + ).toBe('[REDACTED]'); + } + }, + ); +}); + +describe('PR95 review round 1: label stacking, host bounds and masked userinfo', () => { + const token = ['abcdef', 'ghijkl', '123456'].join(''); + const headerSchemeLabel = (value: unknown) => expect(value, 'SECRETS_B2_HEADER_SCHEME_LABEL'); + const userinfoContact = (value: unknown) => expect(value, 'SECRETS_B2_USERINFO_CONTACT'); + + it('masks the token after stacked or quoted scheme labels in a header value', () => { + for (const [input, expected] of [ + [`api-key: Bearer Bearer ${token}`, 'api-key: Bearer Bearer [REDACTED]'], + [`x-api-key: Basic Bearer ${token}`, 'x-api-key: Basic Bearer [REDACTED]'], + [`{"x-api-key":"Bearer Bearer ${token}"}`, '{"x-api-key":"Bearer Bearer [REDACTED]"}'], + [`x-goog-api-key: Bearer\nBearer ${token}`, 'x-goog-api-key: Bearer\nBearer [REDACTED]'], + [`api-key:Bearer B\tearer ${token}`, 'api-key:Bearer B\tearer [REDACTED]'], + [`api-key: Bearer "${token}"`, 'api-key: Bearer "[REDACTED]"'], + [`api-key: Basic '${token}'`, "api-key: Basic '[REDACTED]'"], + ] as const) { + const output = scrubSecrets(input); + headerSchemeLabel(output).toBe(expected); + headerSchemeLabel(output.includes(token)).toBe(false); + } + }); + + it('masks the token after more than one whitespace character following a scheme label', () => { + for (const [input, expected] of [ + [`api-key: Bearer ${token} rejected`, 'api-key: Bearer [REDACTED] rejected'], + [`x-api-key:Basic \t${token}`, 'x-api-key:Basic \t[REDACTED]'], + [`api-key:Bearer \r\n${token}`, 'api-key:Bearer \r\n[REDACTED]'], + ] as const) { + const output = scrubSecrets(input); + headerSchemeLabel(output).toBe(expected); + headerSchemeLabel(output.includes(token)).toBe(false); + } + }); + + it('masks the token after stacked labels in a standalone scheme value', () => { + // The last label masks the token: no label is taken as another label's token. + for (const [input, expected] of [ + [ + `Authorization: Bearer Bearer ${token} rejected`, + 'Authorization: Bearer Bearer [REDACTED] rejected', + ], + [`Authorization: Basic Basic ${token}`, 'Authorization: Basic Basic [REDACTED]'], + [`Authorization: Bearer Basic ${token}`, 'Authorization: Bearer Basic [REDACTED]'], + [`Authorization: Bearer B\nearer ${token}`, 'Authorization: Bearer B\nearer [REDACTED]'], + ] as const) { + expect(scrubSecrets(input)).toBe(expected); + } + expect(scrubSecrets('Authorization: Bearer Bearerish rejected')).toBe( + 'Authorization: Bearer [REDACTED] rejected', + ); + }); + + it.each([ + 'https://john.doe:s3cret?x=1@h.example/', + 'https://localhostuser:pw?a=b@h.example/', + 'https://user:pw.evil.com?x=1@h.example/', + 'https://h.example:123456?x=1@h.example/', + ])('masks userinfo whose prefix only resembles a host: %s', (input) => { + userinfoContact(scrubSecrets(input)).toBe('https://[REDACTED]@h.example/'); + }); + + it('retains a public contact after a host with a five-digit port', () => { + const input = 'connect https://h.example:65535?contact=alice@example.invalid failed'; + expect(scrubSecrets(input)).toBe(input); + }); + + it( + 'masks an admitted-size dotted userinfo prefix without exhausting the RegExp stack', + () => { + const labels = 'a.'.repeat(Math.floor((8 * 1024 * 1024 - 256 - 64) / 2)); + const input = `upstream said https://${labels}a?x=1@h.example failed`; + let output = ''; + expect(() => { + output = scrubSecrets(input); + }, 'SECRETS_B2_USERINFO_HOST_LENGTH').not.toThrow(); + expect( + output === 'upstream said https://[REDACTED]@h.example failed', + 'SECRETS_B2_USERINFO_HOST_LENGTH', + ).toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it('keeps colonless connection-string userinfo visible even with a raw ?/#', () => { + for (const input of ['redis://#TOKEN@h.example', 'postgres://us?er@db.example/x']) { + expect(scrubSecrets(input)).toBe(input); + } + }); + + it('leaves already-masked userinfo as a fixed point and masks a literal around a mask', () => { + for (const input of [ + 'postgres://app:[REDACTED]@db.example/app', + 'redis://:[REDACTED]@cache.example:6379', + 'https://[REDACTED]:[REDACTED]@h.example/', + 'https://:[REDACTED]@h.example/', + ]) { + expect(scrubSecrets(input)).toBe(input); + } + expect(scrubSecrets('https://app:[REDACTED]@h.example/')).toBe('https://[REDACTED]@h.example/'); + expect(scrubSecrets('postgres://app:pa[REDACTED]ss@db.example')).toBe( + 'postgres://[REDACTED]@db.example', + ); + // Only an exactly masked password is kept: text glued after the marker, or an empty password, is not. + for (const [input, expected] of [ + ['postgres://app:[REDACTED]tail@db.example/x', 'postgres://[REDACTED]@db.example/x'], + ['redis://:[REDACTED]SECRETTAIL@cache:6379', 'redis://[REDACTED]@cache:6379'], + ['https://:[REDACTED]tail@h.example/', 'https://[REDACTED]@h.example/'], + ['https://[REDACTED]:@h.example/', 'https://[REDACTED]@h.example/'], + ] as const) { + expect(scrubSecrets(input), 'SECRETS_B5_MASKED_USERINFO').toBe(expected); + } + }); +}); + +describe('PR95 review round 2: quoted labels, unbounded stacking and fixed points', () => { + const token = ['abcdef', 'ghijkl', '123456'].join(''); + const stacked = (value: unknown) => expect(value, 'SECRETS_B2_STACKED_LABEL'); + + it('masks quoted and escaped tokens beside header scheme labels', () => { + for (const [input, expected] of [ + [ + `request headers: {'x-api-key': '"Bearer ${token}"'}`, + `request headers: {'x-api-key': '"Bearer [REDACTED]"'}`, + ], + [`{"x-api-key":"Bearer \\"${token}\\""}`, '{"x-api-key":"Bearer \\"[REDACTED]\\""}'], + [ + `{"x-goog-api-key":"Basic Basic \\"${token}\\""}`, + '{"x-goog-api-key":"Basic Basic \\"[REDACTED]\\""}', + ], + [ + `{"x-api-key":"\\\\\\"Bearer ${token}\\\\\\""}`, + '{"x-api-key":"\\\\\\"Bearer [REDACTED]\\\\\\""}', + ], + [`{"x-api-key":"\\"${token}\\""}`, '{"x-api-key":"\\"[REDACTED]\\""}'], + // A delimiter-led token after a label: the whitespace-stripped residual probe blanks it all. + [`api-key: Bearer ,${token}`, '[REDACTED]'], + ] as const) { + const output = scrubSecrets(input); + expect(output, 'SECRETS_B2_HEADER_SCHEME_LABEL').toBe(expected); + expect(output.includes(token), 'SECRETS_B2_HEADER_SCHEME_LABEL').toBe(false); + } + }); + + it('masks a header token after up to eight stacked labels and fails closed beyond them', () => { + for (let count = 1; count <= 12; count += 1) { + const labels = Array.from({ length: count }, (_, index) => (index % 2 ? 'Basic' : 'Bearer')); + const output = scrubSecrets(`api-key: ${labels.join(' ')} ${token}`); + stacked(output).toBe(count <= 8 ? `api-key: ${labels.join(' ')} [REDACTED]` : '[REDACTED]'); + } + }); + + it('masks the token after any number of stacked standalone labels', () => { + for (let count = 1; count <= 12; count += 1) { + for (const label of ['Bearer', 'Basic']) { + const kept = `${label} `.repeat(count - 1); + stacked(scrubSecrets(`Authorization: ${kept}${label} ${token}`)).toBe( + `Authorization: ${kept}${label} [REDACTED]`, + ); + } + } + }); + + it( + 'keeps admitted-size label runs linear and fails closed', + () => { + const run = 'Bearer '.repeat(Math.floor((8 * 1024 * 1024 - 256 - 64) / 7)); + let header = ''; + let standalone = ''; + expect(() => { + header = scrubSecrets(`api-key: ${run}${token}`); + standalone = scrubSecrets(`Authorization: ${run}${token}`); + }, 'SECRETS_B2_HEADER_LABEL_BOUND').not.toThrow(); + expect(header === '[REDACTED]', 'SECRETS_B2_HEADER_LABEL_BOUND').toBe(true); + expect( + standalone.endsWith('Bearer [REDACTED]') && !standalone.includes(token), + 'SECRETS_B2_HEADER_LABEL_BOUND', + ).toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it('keeps already-masked scheme tokens as fixed points', () => { + for (const input of [ + 'x-api-key: Bearer [REDACTED]', + 'x-api-key: Basic [REDACTED]', + '{"x-api-key":"Bearer [REDACTED]"}', + '{"x-api-key":"Basic [REDACTED]"}', + "{'api-key': 'Bearer [REDACTED]'}", + 'Authorization: Bearer "[REDACTED]"', + // Round 3: quoted and backslash-escaped masked tokens after a label or a header separator. + 'x-api-key: Bearer "[REDACTED]"', + "x-api-key: Basic '[REDACTED]'", + 'api-key: \\"[REDACTED]\\"', + 'api-key: Bearer \\"[REDACTED]\\"', + '{"x-api-key":"Bearer \\"[REDACTED]\\""}', + ]) { + expect(scrubSecrets(input), 'SECRETS_B2_MASKED_FIXED_POINT').toBe(input); + } + }); + + it('bounds the host-shaped prefix at exactly 260 characters, case-insensitively', () => { + const host = (length: number) => `${'a'.repeat(length - 2)}.b`; + const retained = `connect https://${host(260)}?contact=alice@example.invalid failed`; + expect(scrubSecrets(retained), 'SECRETS_B2_USERINFO_HOST_LENGTH').toBe(retained); + expect( + scrubSecrets(`https://${host(261)}?contact=alice@example.invalid`), + 'SECRETS_B2_USERINFO_HOST_LENGTH', + ).toBe('https://[REDACTED]@example.invalid'); + const upper = 'connect https://PROVIDER.INVALID:8443?contact=alice@example.invalid failed'; + expect(scrubSecrets(upper)).toBe(upper); + }); +}); + +describe('PR95 review round 3: merged masking, glued labels and nested headers', () => { + const token = ['abcdef', 'ghijkl', '123456'].join(''); + const cases = (marker: string, pairs: readonly (readonly [string, string])[]) => { + for (const [input, expected] of pairs) { + const output = scrubSecrets(input); + expect(output, marker).toBe(expected); + expect(output.includes(token) || output.includes('SECRET'), marker).toBe(false); + expect(scrubSecrets(output), marker).toBe(output); + } + }; + + it("masks a tail glued to a masked token, including the scrubber's own masks", () => { + cases('SECRETS_B3_MASKED_TAIL', [ + [ + 'Authorization: Bearer sk-abcdefghijklmnopqrstuv;TENANTSECRET99', + 'Authorization: Bearer [REDACTED]', + ], + [ + 'Authorization: Basic AIzaSyABCDEFGHIJKLMNOPQRSTUVWX,TAILSECRET', + 'Authorization: Basic [REDACTED]', + ], + ["Authorization: Bearer [REDACTED]'TAILSECRET", 'Authorization: Bearer [REDACTED]'], + ['Authorization: Bearer [REDACTED]TAILSECRET', 'Authorization: Bearer [REDACTED]'], + ["Authorization: Basic [REDACTED]'TAILSECRET", 'Authorization: Basic [REDACTED]'], + ['Authorization: Basic [REDACTED]TAILSECRET', 'Authorization: Basic [REDACTED]'], + [ + '{"authorization":"Bearer sk-proj-abcdefghijklmnopqrst","api_key":"SECRETVALUE77"}', + '{"authorization":"Bearer [REDACTED]"}', + ], + // A header token continues through a backslash, so a marker glued to it is no fixed point. + ['api-key: [REDACTED]\\TAILSECRET', 'api-key: [REDACTED]'], + ['api-key: sk-abcdefghijklmnopqrstuv\\TAILSECRET', 'api-key: [REDACTED]'], + ]); + }); + + it('masks the token after a label glued behind punctuation or inside another token', () => { + cases('SECRETS_B3_GLUED_LABEL', [ + [`api-key: \\"(Bearer ${token}`, 'api-key: \\"(Bearer [REDACTED]'], + [`x-api-key:""(Bearer ${token}`, 'x-api-key:""(Bearer [REDACTED]'], + [`api-key: \\"=Basic ${token}`, 'api-key: \\"=Basic [REDACTED]'], + [`api-key: \\"[REDACTED]Bearer ${token}`, 'api-key: \\"[REDACTED] [REDACTED]'], + [ + `Authorization: Bearer Bearer (Bearer ${token}`, + 'Authorization: Bearer Bearer (Bearer [REDACTED]', + ], + [`api-key: abc(Bearer ${token}`, 'api-key: [REDACTED] [REDACTED]'], + [`Bearer x"Bearer ${token}`, 'Bearer [REDACTED]"Bearer [REDACTED]'], + [`Authorization: Basic x(Basic ${token}`, 'Authorization: Basic [REDACTED]Basic [REDACTED]'], + [`Authorization: Basic (Basic ${token}`, 'Authorization: Basic (Basic [REDACTED]'], + [`Basic x"Basic ${token}`, 'Basic [REDACTED]"Basic [REDACTED]'], + // A terminal control inside the run makes the raw and normalized views disagree: fail closed. + [`Bearer \u001b[0m""Bearer \n${token}`, '[REDACTED]'], + ]); + }); + + it('masks the token after a label closed by its own quote', () => { + cases('SECRETS_B3_QUOTED_LABEL', [ + [`x-api-key: "Bearer" ${token}`, 'x-api-key: "Bearer" [REDACTED]'], + [`api-key: 'Bearer' ${token}`, "api-key: 'Bearer' [REDACTED]"], + [`{"x-api-key": "Bearer" "${token}"}`, '{"x-api-key": "Bearer" "[REDACTED]"}'], + [`Authorization: Bearer "Bearer" ${token}`, 'Authorization: Bearer "Bearer" [REDACTED]'], + [`Authorization: Bearer "Basic" ${token}`, 'Authorization: Bearer "Basic" [REDACTED]'], + [ + `Authorization: Basic \\"Bearer\\" ${token}`, + 'Authorization: Basic \\"Bearer\\" [REDACTED]', + ], + [`Authorization: "Basic" ${token}`, 'Authorization: "Basic" [REDACTED]'], + [`Authorization: Bearer 'Basic' ${token}`, "Authorization: Bearer 'Basic' [REDACTED]"], + ]); + // A word that only begins with a label is a token; a label at the end of the text has none. + expect(scrubSecrets('Authorization: Basic Basicx'), 'SECRETS_B3_QUOTED_LABEL').toBe( + 'Authorization: Basic [REDACTED]', + ); + expect(scrubSecrets('Authorization: Basic Basic'), 'SECRETS_B3_QUOTED_LABEL').toBe( + 'Authorization: Basic Basic', + ); + }); + + it('masks a header token after any run of quotes or escapes, also in JSON-encoded JSON', () => { + cases('SECRETS_B3_HEADER_ESCAPES', [ + [`x-api-key: '""${token}`, `x-api-key: '""[REDACTED]`], + [`{"x-api-key":"\\"\\"${token}"}`, '{"x-api-key":"\\"\\"[REDACTED]"}'], + [`{"x-api-key":"\\\\\\"${token}\\\\\\""}`, '{"x-api-key":"\\\\\\"[REDACTED]\\\\\\""}'], + [ + `{"body":"{\\"x-api-key\\":\\"${token}\\"}"}`, + '{"body":"{\\"x-api-key\\":\\"[REDACTED]\\"}"}', + ], + [ + `{"body":"{\\"x-api-key\\":\\"Bearer ${token}\\"}"}`, + '{"body":"{\\"x-api-key\\":\\"Bearer [REDACTED]\\"}"}', + ], + ]); + }); + + it('keeps every header value and label token apart from a nested header name', () => { + cases('SECRETS_B3_NESTED_HEADER', [ + [`api-key:""api-key":${token}`, 'api-key:""api-key":[REDACTED]'], + [`api-key: x(api-key: ${token}`, 'api-key: [REDACTED]api-key: [REDACTED]'], + [`api-key:(api-key:[REDACTED]${token}`, 'api-key:[REDACTED]api-key:[REDACTED]'], + [`Bearer api-key: ${token}`, 'Bearer [REDACTED] [REDACTED]'], + [`Bearer api-key:,${token}`, 'Bearer [REDACTED]'], + ]); + }); + + it('treats a whitespace-separated labels-only header value alike in every context', () => { + for (const input of ['api-key: Bearer Bearer', 'x api-key: Bearer Bearer', 'api-key: Basic']) { + expect(scrubSecrets(input), 'SECRETS_B3_LABEL_CONTEXT').toBe(input); + } + const masked = scrubSecrets('sk-proj-ABCDEFGHIJKLMNOPQRSTUV api-key: Bearer Bearer'); + expect(masked, 'SECRETS_B3_LABEL_CONTEXT').toBe('[REDACTED] api-key: Bearer Bearer'); + expect(scrubSecrets(masked), 'SECRETS_B3_LABEL_CONTEXT').toBe(masked); + }); + + it( + 'keeps admitted-size escape and concatenated-label runs linear and total', + () => { + const size = 8 * 1024 * 1024 - 256 - 64; + const escapes = `api-key: ${'\\\\'.repeat(size)}`; + const quoted = `api-key: ${'"Bearer'.repeat(Math.floor(size / 7))} ${token}`; + const standalone = `Bearer ${'"Bearer'.repeat(Math.floor(size / 7))} ${token}`; + const basic = `Basic ${'"Basic'.repeat(Math.floor(size / 6))} ${token}`; + let results: string[] = []; + expect(() => { + results = [escapes, quoted, standalone, basic].map((input) => scrubSecrets(input)); + }, 'SECRETS_B3_ADMITTED_RUNS').not.toThrow(); + // A run of escapes holds no token, and a glued-label run past eight labels blanks it all. + expect(results[0] === escapes, 'SECRETS_B3_ADMITTED_RUNS').toBe(true); + expect(results.slice(1), 'SECRETS_B3_ADMITTED_RUNS').toEqual(Array(3).fill('[REDACTED]')); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); +}); + +describe('PR95 review round 4: concatenated labels, ambiguous authorities and a fixed point', () => { + const token = ['abcdef', 'ghijkl', '123456'].join(''); + const cases = (marker: string, pairs: readonly (readonly [string, string])[]) => { + for (const [input, expected] of pairs) { + const output = scrubSecrets(input); + expect(output, marker).toBe(expected); + expect(output.includes(token) || output.includes('SECRET'), marker).toBe(false); + expect(scrubSecrets(output), marker).toBe(output); + } + }; + const kept = (marker: string, inputs: readonly string[]) => { + for (const input of inputs) expect(scrubSecrets(input), marker).toBe(input); + }; + + it('masks the token after a concatenated label run in every context', () => { + cases('SECRETS_B4_CONCAT_LABEL', [ + [ + `Authorization: Bearer BearerBearer ${token}`, + 'Authorization: Bearer BearerBearer [REDACTED]', + ], + [`Authorization: BearerBearer ${token}`, 'Authorization: BearerBearer [REDACTED]'], + [`Authorization: BasicBearer ${token}`, 'Authorization: BasicBearer [REDACTED]'], + [`Authorization: Bearer BasicBasic ${token}`, 'Authorization: Bearer BasicBasic [REDACTED]'], + [`Basic bearerbasic ${token}`, 'Basic bearerbasic [REDACTED]'], + [ + `Authorization: Bearer\tBearerBearer ${token}`, + 'Authorization: Bearer\tBearerBearer [REDACTED]', + ], + [`Authorization: B\nearerBasic ${token}`, 'Authorization: B\nearerBasic [REDACTED]'], + [ + `Authorization: Bearer "BearerBearer" ${token}`, + 'Authorization: Bearer "BearerBearer" [REDACTED]', + ], + [ + `{"authorization":"Bearer BearerBearer ${token}"}`, + '{"authorization":"Bearer BearerBearer [REDACTED]"}', + ], + [`api-key: BearerBearer ${token}`, 'api-key: BearerBearer [REDACTED]'], + [`see api-key: BearerBearer ${token}`, 'see api-key: BearerBearer [REDACTED]'], + [`curl -H x-api-key: BearerBearer ${token}`, 'curl -H x-api-key: BearerBearer [REDACTED]'], + [`api-key: BearerBearer [REDACTED]${token}`, 'api-key: BearerBearer [REDACTED]'], + [`x-api-key: }Bearer BearerBearer ${token}`, 'x-api-key: }Bearer BearerBearer [REDACTED]'], + // A token ends before a concatenated run glued to it behind punctuation. + [ + `Authorization: Bearer x(BasicBearer ${token}`, + 'Authorization: Bearer [REDACTED]BasicBearer [REDACTED]', + ], + [ + `Authorization: Basic x(BearerBasic ${token}`, + 'Authorization: Basic [REDACTED]BearerBasic [REDACTED]', + ], + // A terminal control inside the header name reveals the value only once normalized. + [`-H X\u0007-API-KEY: BearerBearer ${token}`, '[REDACTED]'], + ]); + // A concatenated run is never a token: labels alone stay visible. + kept('SECRETS_B4_CONCAT_LABEL', [ + 'Authorization: Bearer BearerBearer', + 'Authorization: Basic BasicBasic', + 'Authorization: BearerBearer', + 'api-key: BearerBearer', + ]); + }); + + it('fails closed on nine concatenated labels and keeps eight in place', () => { + const bound = (value: unknown) => expect(value, 'SECRETS_B4_LABEL_RUN_BOUND'); + for (let count = 1; count <= 12; count += 1) { + for (const label of ['Bearer', 'Basic']) { + const run = label.repeat(count); + bound(scrubSecrets(`Authorization: Bearer ${run} ${token}`)).toBe( + count <= 8 ? `Authorization: Bearer ${run} [REDACTED]` : '[REDACTED]', + ); + bound(scrubSecrets(`api-key: ${run} ${token}`)).toBe( + count <= 8 ? `api-key: ${run} [REDACTED]` : '[REDACTED]', + ); + } + } + // Quotes or escapes between the labels still join them, and a run with no token fails alike. + for (const input of [ + `Authorization: ${'"Bearer'.repeat(9)} ${token}`, + `Authorization: ${'\\"Basic'.repeat(9)} ${token}`, + `Authorization: Bearer ${'Basic'.repeat(9)}`, + ]) { + bound(scrubSecrets(input)).toBe('[REDACTED]'); + } + // Any case, and labels split by LF or TAB, count alike. + for (const label of ['bearer', 'BASIC', 'B\nearer', 'B\tasic']) { + for (const count of [8, 9]) { + const run = label.repeat(count); + bound(scrubSecrets(`Authorization: Bearer ${run} ${token}`)).toBe( + count <= 8 ? `Authorization: Bearer ${run} [REDACTED]` : '[REDACTED]', + ); + bound(scrubSecrets(`api-key: ${run} ${token}`)).toBe( + count <= 8 ? `api-key: ${run} [REDACTED]` : '[REDACTED]', + ); + } + } + }); + + it( + 'keeps admitted-size label runs total, concatenated or separate', + () => { + const size = 8 * 1024 * 1024 - 256 - 64; + const concatenated = `Authorization: Bearer ${'Bearer'.repeat(Math.floor(size / 6))} ${token}`; + const separate = `Authorization: ${'Basic '.repeat(Math.floor(size / 6))}${token}`; + let results: string[] = []; + expect(() => { + results = [concatenated, separate].map((input) => scrubSecrets(input)); + }, 'SECRETS_B4_LABEL_RUN_BOUND').not.toThrow(); + expect(results[0] === '[REDACTED]', 'SECRETS_B4_LABEL_RUN_BOUND').toBe(true); + expect( + results[1]?.endsWith('Basic [REDACTED]') === true && !results[1].includes(token), + 'SECRETS_B4_LABEL_RUN_BOUND', + ).toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it('ends a header token at a closing quote or brace', () => { + cases('SECRETS_B4_HEADER_STOP', [ + [`{api-key:${token}}`, '{api-key:[REDACTED]}'], + [`api-key: ${token}}`, 'api-key: [REDACTED]}'], + [`{'api-key':'${token}'}`, "{'api-key':'[REDACTED]'}"], + [`{'x-api-key': '${token}', 'b': 1}`, "{'x-api-key': '[REDACTED]', 'b': 1}"], + ]); + }); + + it('masks a quoted standalone token after either label', () => { + cases('SECRETS_B4_LABEL_LEAD', [ + [`Authorization: Bearer "${token}"`, 'Authorization: Bearer "[REDACTED]"'], + [`Authorization: Bearer '${token}'`, "Authorization: Bearer '[REDACTED]'"], + [`{"authorization":"Bearer \\"${token}\\""}`, '{"authorization":"Bearer \\"[REDACTED]\\""}'], + [`Authorization: Basic "${token}"`, 'Authorization: Basic "[REDACTED]"'], + [`Authorization: Basic '${token}'`, "Authorization: Basic '[REDACTED]'"], + [`{"authorization":"Basic \\"${token}\\""}`, '{"authorization":"Basic \\"[REDACTED]\\""}'], + ]); + }); + + it('masks the whole of a range that contains a later one', () => { + // A later pass re-masks a truncated range wherever its rule matches again; behind an ambiguous + // authority no `@` is left, so only the merged extent keeps a key's neighbour masked. + cases('SECRETS_B4_MERGE_EXTENT', [ + ['https://svc?a=b@.sk-abcdefghijklmnopqrstuv.TAILSECRET', 'https://[REDACTED]'], + [ + 'https://svc?a=b@.AIzaSyA1234567890abcdefghijklmn.TAILSECRET failed', + 'https://[REDACTED] failed', + ], + ['{"u":"https://svc#x@.sk-abcdefghijklmnopqrstuv.TAILSECRET"}', '{"u":"https://[REDACTED]"}'], + [ + `https://user:sk-abcdefghijklmnopqrstuv:more${token}@h.example/`, + 'https://[REDACTED]@h.example/', + ], + [`postgres://u:AIzaSyA1234567890abcdefghijklmn:${token}@db/x`, 'postgres://[REDACTED]@db/x'], + ]); + }); + + it('keeps the documented edges of the label and header rules', () => { + cases('SECRETS_B4_RULE_EDGES', [[`api-key : ${token} retry`, 'api-key : [REDACTED] retry']]); + // A label glued to a preceding word character is no label. + kept('SECRETS_B4_RULE_EDGES', [`xBearer ${token}`, `xBasic ${token}`]); + }); + + it('masks a literal query value in the same pass as every other rule', () => { + cases('SECRETS_B4_QUERY_MERGE', [ + [ + '{\\"x-api-key\\":\\"https://h.example/cb?token=abc;SECRETTAIL\\"}', + '{\\"x-api-key\\":\\"[REDACTED]', + ], + ['x-api-key: https://h.example/cb?token=abc;SECRETTAIL', 'x-api-key: [REDACTED]'], + ]); + }); + + it('masks a secret query value that holds a raw @ after a single-label host', () => { + for (const [input, expected, tail] of [ + [ + 'request to https://svc?password=P@ssw0rd123 failed', + 'request to https://[REDACTED] failed', + 'ssw0rd123', + ], + ['https://gateway?api_key=Ab3@Zx9QrT7', 'https://[REDACTED]', 'Zx9QrT7'], + ['postgres://db?options=a:b&password=P@ssw0rd99', 'postgres://[REDACTED]', 'ssw0rd99'], + ['https://svc?token=abc@def&x=1', 'https://[REDACTED]', 'def'], + ] as const) { + const output = scrubSecrets(input); + expect(output, 'SECRETS_B4_AMBIGUOUS_QUERY').toBe(expected); + expect(output.includes(tail), 'SECRETS_B4_AMBIGUOUS_QUERY').toBe(false); + } + }); + + it('masks an ambiguous authority through its end unless a host follows its last @', () => { + cases('SECRETS_B4_AUTHORITY_REST', [ + ['https://svc?opaque=P@SECRETTAIL1 failed', 'https://[REDACTED] failed'], + ['https://svc#frag@SECRETTAIL1 x', 'https://[REDACTED] x'], + ['{"url":"https://svc?code=ab@SECRETTAIL1","n":1}', '{"url":"https://[REDACTED]","n":1}'], + ['https://us?er:TOKEN@h/', 'https://[REDACTED]/'], + ]); + // A raw ?/# in a password does not hide the host it connects to. + for (const [input, expected] of [ + ['https://alice:pa?ss@provider.invalid/v1', 'https://[REDACTED]@provider.invalid/v1'], + ['https://u:p?w@[::1]:8443/x', 'https://[REDACTED]@[::1]:8443/x'], + ['https://u:p?w@localhost:3000/x', 'https://[REDACTED]@localhost:3000/x'], + ] as const) { + expect(scrubSecrets(input), 'SECRETS_B4_AUTHORITY_REST').toBe(expected); + } + }); + + it( + 'bounds the host test after an ambiguous authority at admitted size', + () => { + const labels = 'a.'.repeat(Math.floor((8 * 1024 * 1024 - 256 - 64) / 2)); + const input = `upstream said https://svc?x=1@${labels}a failed`; + let output = ''; + expect(() => { + output = scrubSecrets(input); + }, 'SECRETS_B4_REST_LENGTH').not.toThrow(); + expect(output === 'upstream said https://[REDACTED] failed', 'SECRETS_B4_REST_LENGTH').toBe( + true, + ); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it('reads a word glued before sftp as part of an sftp scheme', () => { + // Every whitespace-joined probe glues prose to a scheme the same way, so `to sftp://` must stay. + kept('SECRETS_B4_GLUED_SFTP', [ + 'statusftp://deploy@h.example', + 'upload to sftp://deploy@files.example failed', + ]); + expect(scrubSecrets('statusftp://deploy:pw@h.example'), 'SECRETS_B4_GLUED_SFTP').toBe( + 'statusftp://[REDACTED]@h.example', + ); + }); + + it('reaches a fixed point', () => { + for (const [input, expected] of [ + ['api-key: https://"@host.example', 'api-key: [REDACTED]'], + ['{"x-api-key":"https://\\"@h.example"}', '{"x-api-key":"[REDACTED]"}'], + ['x-api-key: Bearer abc"=Bearer xyz', 'x-api-key: Bearer [REDACTED] [REDACTED]'], + ['x-api-key: Bearer abc"(Bearer xyz12345', 'x-api-key: Bearer [REDACTED] [REDACTED]'], + ['api-key:Bearer S":BEAReR', 'api-key:Bearer [REDACTED]'], + ['x:sk-proj-abcdefghijklmnopqrst api-key\\":\\" :', '[REDACTED]'], + ['&key=H X-GOOG-API-KEY:basic |BasiC', '[REDACTED]'], + ['http://i.b#@=\n@', '[REDACTED]'], + ['[REDACTED]', '[REDACTED]'], + ] as const) { + const output = scrubSecrets(input); + expect(output, 'SECRETS_B4_FIXED_POINT').toBe(expected); + expect(scrubSecrets(output), 'SECRETS_B4_FIXED_POINT').toBe(output); + } + }); + + it( + 'keeps quote and escape runs after a label or header token linear at admitted size', + () => { + // Pinned by this case deadline only: a quadratic scan here still finishes, just too late. + const size = 8 * 1024 * 1024 - 256 - 64; + const quotes = '"'.repeat(size); + const inputs = [ + `Bearer ${quotes}`, + `Basic ${quotes}`, + `api-key: ${token}${'\\'.repeat(size)}x`, + ]; + let results: string[] = []; + expect(() => { + results = inputs.map((input) => scrubSecrets(input)); + }, 'SECRETS_B4_ADMITTED_RUNS').not.toThrow(); + expect(results[0] === inputs[0], 'SECRETS_B4_ADMITTED_RUNS').toBe(true); + expect(results[1] === inputs[1], 'SECRETS_B4_ADMITTED_RUNS').toBe(true); + expect(results[2] === 'api-key: [REDACTED]', 'SECRETS_B4_ADMITTED_RUNS').toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); +}); + +describe('PR95 review round 5: label pieces, a split ftp credential and the repeated residual', () => { + const token = ['abcdef', 'ghijkl', '123456'].join(''); + const cases = (marker: string, pairs: readonly (readonly [string, string])[]) => { + for (const [input, expected] of pairs) { + const output = scrubSecrets(input); + expect(output, marker).toBe(expected); + expect(output.includes(token) || output.includes('SECRET'), marker).toBe(false); + expect(scrubSecrets(output), marker).toBe(output); + } + }; + + it('ends a standalone token before a trailing delimiter, a quote-closed label or a concatenated run', () => { + cases('SECRETS_B5_LABEL_PIECES', [ + [`Authorization: Bearer ${token}; retry`, 'Authorization: Bearer [REDACTED]; retry'], + [`Authorization: Bearer ${token}, retry`, 'Authorization: Bearer [REDACTED], retry'], + [`Authorization: Basic ${token}; retry`, 'Authorization: Basic [REDACTED]; retry'], + [`Authorization: Basic ${token}, retry`, 'Authorization: Basic [REDACTED], retry'], + [ + `Authorization: Bearer x("Bearer" ${token}`, + 'Authorization: Bearer [REDACTED]"Bearer" [REDACTED]', + ], + [ + `Authorization: Bearer x(${'Bearer'.repeat(8)} ${token}`, + `Authorization: Bearer [REDACTED]${'Bearer'.repeat(8)} [REDACTED]`, + ], + ]); + }); + + it('blanks an LF/TAB-split ftp credential after a line that ends in s', () => { + cases('SECRETS_B5_SPLIT_SFTP', [ + ['Uploading files\nftp://deploySECRET\n1234@host.example/x', '[REDACTED]'], + ['Uploading files\n\tftp://deploySECRET\n1234@host.example/x', '[REDACTED]'], + ['files\nf\ntp://TOKENSECRET@host', '[REDACTED]'], + ['Uploading files\tftp://deploySECRET\t1234@host.example/x', '[REDACTED]'], + ]); + // A real sftp scheme keeps its colonless user, also when the user is split. + for (const input of [ + 'files\nsftp://deploy\nuser@host', + 'connect to sftp://deploy@files.example', + ]) { + expect(scrubSecrets(input), 'SECRETS_B5_SPLIT_SFTP').toBe(input); + } + }); + + it('blanks a diagnostic whose key prefix reaches its length across spaces after a mask', () => { + // The repeat reads another rule's mask as a segment start for the whitespace-stripped residual. + for (const input of [ + 'Authorization: Bearer xyz sk-ant api key invalid for this org', + 'x-api-key: abc AIza prefix keys are restricted here', + ]) { + expect(scrubSecrets(input), 'SECRETS_B5_RESIDUAL_AFTER_MASK').toBe('[REDACTED]'); + } + }); +}); + +describe('PR95 review round 6: the label bound behind a terminal sequence, fragments and failing closed', () => { + it('fails closed on nine labels behind a terminal sequence', () => { + const bound = (value: unknown) => expect(value, 'SECRETS_B4_LABEL_RUN_BOUND'); + // Normalization reads `ESC [` plus the run's first letter as one terminal sequence. + for (const label of ['Bearer', 'Basic']) { + for (const lead of ['Authorization: Bearer ', 'x-api-key: ', '']) { + bound(scrubSecrets(`${lead}\u001b[${label.repeat(9)} opaqueTOKEN123`)).toBe('[REDACTED]'); + } + } + }); + + it('fails closed on nine labels that only the normalized view joins', () => { + const bound = (value: unknown) => expect(value, 'SECRETS_B4_LABEL_RUN_BOUND'); + // A terminal sequence normalization removes splits the raw run into four and five labels. + for (const label of ['Bearer', 'Basic']) { + for (const lead of ['Authorization: Bearer ', 'x-api-key: ', '']) { + bound( + scrubSecrets(`${lead}${label.repeat(4)}\u001b[0m${label.repeat(5)} opaqueTOKEN123`), + ).toBe('[REDACTED]'); + } + } + }); + + it('masks a literal secret query name in a fragment', () => { + const fragment = (value: unknown) => expect(value, 'SECRETS_B6_QUERY_FRAGMENT'); + fragment(scrubSecrets('https://[::1]#id_token=Zk81Pq0aLm3nB7xY@127.0.0.1/cb')).toBe( + 'https://[::1]#id_token=[REDACTED]', + ); + fragment( + scrubSecrets( + 'redirect https://app.example/cb#access_token=Zk81Pq0aLm3nB7xYt2Rw&token_type=bearer', + ), + ).toBe('redirect https://app.example/cb#access_token=[REDACTED]&token_type=bearer'); + // A name that only begins like one, and an empty value, stay. + for (const kept of ['see https://docs.example/page#tokenizer=fast', 'page#token= next']) { + fragment(scrubSecrets(kept)).toBe(kept); + } + }); + + it('fails closed with its own mask when the scrub does not settle', () => { + vi.mocked(toFixedPoint).mockImplementationOnce((_pass, _text, failed) => failed); + expect(scrubSecrets('public text'), 'SECRETS_B6_FAIL_CLOSED').toBe('[REDACTED]'); + }); +}); + +describe('PR95 review round 8: the split probe behind a terminal introducer and the host after a query', () => { + it('blanks a split credential whose first letter a bare terminal introducer would take', () => { + // Normalization reads ESC '[' plus the credential's first letter as a terminal sequence. + for (const input of [ + '\u001b[sk-Abc4Def5\nGhi6Jkl7Mno8Pqr9', + 'auth failed: \u001b[api-k\ney: Abc4Def5Ghi6Jkl7Mno8', + '\u001b[https://user:Abc4Def5\tGhi6Jkl7Mno8@host.example/x', + // A two-byte escape takes the letter too. + '\u001bAIzaSyAbc12345\n678901234567890xyz', + ]) { + expect(scrubSecrets(input), 'SECRETS_B8_SPLIT_RAW').toBe('[REDACTED]'); + } + const benign = 'plain \u001b[1mbold\u001b[0m text\nnext line'; + expect(scrubSecrets(benign), 'SECRETS_B8_SPLIT_RAW').toBe(benign); + }); + + it('reads the host after an ambiguous authority up to its query or fragment', () => { + for (const [input, expected] of [ + ['https://gw?a=1@host.example?q=1', 'https://[REDACTED]@host.example?q=1'], + ['https://gw?a=1@host.example#frag rest', 'https://[REDACTED]@host.example#frag rest'], + ] as const) { + expect(scrubSecrets(input), 'SECRETS_B8_HOST_QUERY').toBe(expected); + } + }); + + it('masks a header token that a JSON escape begins', () => { + // The escape's letter starts the token, so the value glued to it is masked with it. + for (const [input, expected] of [ + [ + '400 invalid header value: {"x-api-key":"\\nsk-ant-api03-Q9zXw7TokenValue12345abcdef"}', + '400 invalid header value: {"x-api-key":"\\[REDACTED]"}', + ], + [ + '{"x-goog-api-key":"\\tAIzaSyQ9zXw7TokenValue12345abcdef01"}', + '{"x-goog-api-key":"\\[REDACTED]"}', + ], + ['{"api-key":"\\r\\nQ9zXw7TokenValue12345abcdef"}', '{"api-key":"\\[REDACTED]"}'], + ['x-goog-api-key:\\n\\n\\nQ9zXw7TokenValue12345abcdef', 'x-goog-api-key:\\[REDACTED]'], + ['x-api-key:\\r\\n\\r\\nQ9zXw7TokenValue12345abcdef', 'x-api-key:\\[REDACTED]'], + [ + '{\\"api-key\\": \\"\\nQ9zXw7TokenValue12345abcdef\\"}', + '{\\"api-key\\": \\"\\[REDACTED]\\"}', + ], + ['api-key:"\\ nQ9zXw7TokenValue12345abcdef', '[REDACTED]'], + ] as const) { + expect(scrubSecrets(input), 'SECRETS_B8_HEADER_ESCAPE_TOKEN').toBe(expected); + } + }); + + it('blanks a path-less hyperlink target that the raw split probe joins with a later @', () => { + // A documented fail-closed trade-off: the raw LF/TAB-joined view reads a terminal hyperlink's bare + // origin and the next cell's address as one userinfo; a target with a path stops it. + const row = (target: string) => + `ID\tOWNER\n\u001b]8;;${target}\u001b\\build-42\u001b]8;;\u001b\\\tbob@example.com\n`; + expect(scrubSecrets(row('https://ci.example.com')), 'SECRETS_B8_HYPERLINK_TRADEOFF').toBe( + '[REDACTED]', + ); + const withPath = row('https://ci.example.com/builds/42'); + expect(scrubSecrets(withPath), 'SECRETS_B8_HYPERLINK_TRADEOFF').toBe(withPath); + }); + + it('masks a credential before or inside a terminal sequence in place', () => { + for (const [input, expected] of [ + [ + 'request failed: Authorization: Bearer Q9zXw7TokenValue\u001b[0m (status 401)', + 'request failed: Authorization: Bearer [REDACTED] (status 401)', + ], + [ + 'x-api-key: \u001b[33mabcdef123456\u001b[0m rejected by upstream', + 'x-api-key: [REDACTED] rejected by upstream', + ], + ['see https://user:pa\u001b[0mss@host.example/x', 'see https://[REDACTED]@host.example/x'], + ['Bearer \u001b[1mtoken123abc\u001b[0m done', 'Bearer [REDACTED] done'], + ] as const) { + expect(scrubSecrets(input), 'SECRETS_B8_TERMINAL_IN_PLACE').toBe(expected); + } + }); +}); diff --git a/packages/llm/src/scrub-secrets.ts b/packages/llm/src/scrub-secrets.ts index 9d9921a3..db8bf7b4 100644 --- a/packages/llm/src/scrub-secrets.ts +++ b/packages/llm/src/scrub-secrets.ts @@ -1,88 +1,249 @@ import { stripTerminalControls } from '@relavium/shared/terminal-safe'; +import { toFixedPoint } from './fixed-point.js'; +import { splitProbe, type SplitProbe, type SplitProbeAnchor } from './split-credential-probe.js'; + +/** The marker every masked span becomes; a diagnostic that cannot be masked in place becomes it whole. */ +const MASKED = '[REDACTED]'; + /** * Defense-in-depth secret scrub applied at the one `makeLlmError` choke point. The CALLER (adapter) is * still responsible for passing an already-redacted message; this is the **structural backstop** — like * the engine's `deInlineMedia` pass, leak-freedom here is an active transform at one boundary, not * per-adapter discipline. It masks finite common secret **shapes**: prefixed keys, literal secret query - * names, URL userinfo and finite API-key/Bearer/Basic headers. Each non-query rule probes the original diagnostic - * independently, including terminal-normalized and LF/TAB-joined views, before any - * other rule can erase its identifying context. A match revealed by normalization masks the whole - * diagnostic. The finite patterns check boundaries in the original text and preserve already-masked spans; - * this is not universal fragment discovery. Query names are excluded from the residual probe. - * Pure and total for string input: regex replacements and a - * finite residual check, with no parsing or external calls on the error path. + * names, URL userinfo of a finite scheme list and finite API-key/Bearer/Basic headers. Each non-query rule probes + * the original diagnostic independently, including the terminal-normalized view, and the key prefixes, URL + * userinfo and exact header names the LF/TAB-joined view too, before any other rule can erase its identifying + * context; an opaque Bearer/Basic value split by an LF or TAB keeps its tail (a documented limit). A match revealed by normalization masks the whole + * diagnostic. The finite patterns check boundaries in the original text (the URL rule has none, like the + * pre-PR matcher, so a scheme glued to a word character is masked in place) and preserve already-masked spans; + * this is not universal fragment discovery. Query names are excluded from the residual probe, but their + * values are masked in the same merged pass as every other rule. One pass can glue a marker to the + * text after another rule's token, so the scrub repeats until it reaches a fixed point within three + * passes, failing closed otherwise: its output is stable under another scrub. + * Pure and total for string input: regex matching, merged masking passes and a finite residual + * check, with no parsing or external calls on the error path. * Asserted by `llm-error.test.ts` (the backstop) and each adapter's * secret-safety test (a planted secret → a secret-free surfaced `LlmError`); see security-review.md. + * The rule is error-handling.md's diagnostic secret backstop; the finite shape list and its limits are + * specified in docs/reference/shared-core/diagnostic-secret-backstop.md. */ export function scrubSecrets(text: string): string { - return scrubQuerySecrets(scrubNonQuerySecrets(text)); + return toFixedPoint(scrubOnce, text, MASKED); +} + +/** A range of the original diagnostic that one rule masks with `[REDACTED]`. */ +interface MaskRange { + readonly start: number; + readonly end: number; } interface SecretRule { readonly pattern: RegExp; - readonly replacement: string | ((match: string) => string); + /** The masked range of one match, relative to the match; `undefined` keeps the match. */ + readonly mask: (match: RegExpExecArray) => MaskRange | undefined; + /** Whether a joined-view candidate is a credential; absent means every candidate is. */ + readonly credential?: (matched: string) => boolean; readonly joinedPattern?: RegExp; readonly boundary?: RegExp; + /** Split-scanner prefilter: a match must reach this literal within its window (see `splitProbe`). */ + readonly anchor?: SplitProbeAnchor; +} + +/** Mask the whole match. */ +const WHOLE_MATCH = (match: RegExpExecArray): MaskRange => ({ start: 0, end: match[0].length }); + +/** Mask only the named `token` group of a `d`-flagged pattern, keeping its label or header around it. */ +function tokenGroup(match: RegExpExecArray): MaskRange | undefined { + const span = match.indices?.groups?.['token']; + return span === undefined + ? undefined + : { start: span[0] - match.index, end: span[1] - match.index }; } /** - * Deliberately NOT word-bounded: the pre-PR matcher masked `xhttps://user:pass@host`, and a `\b` here - * would let any scheme glued to a preceding word character through unmasked. The rule has no `\b` - * prefix, so it also supplies its own joined form rather than letting the scanner strip one. + * Deliberately NOT word-bounded: the pre-PR http(s) matcher masked `xhttps://user:pass@host`, and a `\b` + * here would let any scheme glued to a preceding word character through unmasked. PR #95 added + * ws/wss/ftp/ftps under the same rule, and the PR95 increment added the connection-string schemes + * (postgres(ql), mysql, mariadb, redis(s), mongodb(+srv), amqp(s), ssh, sftp, ldap(s), smtp(s)), whose + * colonless userinfo names a user and stays visible (see `urlUserinfoRange`). Only `sftp` ends in another + * listed scheme, the colonless `ftp`, and a word glued before it (`statusftp://`) still reads as `sftp`: + * every whitespace-stripped probe (the residual below, the CLI's display residual) glues ordinary prose + * to a scheme the same way, so reading it as `ftp` would blank `to sftp://user@host`. The rule has no + * `\b` prefix, so it supplies its own joined form (the split-credential scanner would otherwise strip two + * characters that are not there) and its `boundary` blocks no start. Consequences, pinned by tests and + * fail-closed: an unlisted longer scheme name ending in a listed one (`gftp://u@h`) matches by suffix, + * and the whitespace-stripped residual probe turns a path-less URL, prose and a later `@` into a + * whole-diagnostic `[REDACTED]` (the pre-PR matcher did so for http(s); dropping `\b` extends it to + * prose that stripping glues to the scheme). A credential split by a plain SPACE that also holds a raw `@` remains the documented + * opaque-fragments-across-spaces limit, as it was for the pre-PR matcher. */ -const URL_USERINFO_PATTERN = /(?:https?|wss?|ftps?):\/\/[^/\s]*@/gi; +const URL_USERINFO_PATTERN = + /(?:https?|wss?|ftps?|postgres(?:ql)?|mysql|mariadb|rediss?|mongodb(?:\+srv)?|amqps?|ssh|sftp|ldaps?|smtps?):\/\/[^/\s]*@/gi; -/** One private authority per existing credential family, shared by probes and output masking. */ +/** The schemes whose colonless userinfo is masked too; every other listed scheme needs a `:` in it. */ +const COLONLESS_USERINFO_SCHEME = /^(?:https?|wss?|ftps?)$/i; + +/** + * Every listed scheme ends within this many joined characters before its `://`, so the split scanner + * tries no URL start farther away. It is generous on purpose: the longest scheme (`mongodb+srv`) has 11. + */ +const URL_SCHEME_WINDOW = 32; + +/** A start boundary that never matches: no preceding character makes a start ineligible. */ +const NO_BOUNDARY = /(?!)/; + +/** + * One private authority per existing credential family, shared by probes and output masking. Every + * rule finds its matches in the original diagnostic and the masked ranges of all rules are merged + * before anything is replaced, so no rule can erase the context another rule needs, whatever their + * order. A token rule masks only its token, so its label or header name stays visible, and masking a + * token that is already exactly `[REDACTED]` changes nothing: every masked token is a fixed point. + */ const NON_QUERY_SECRET_RULES: readonly SecretRule[] = [ // OpenAI sk-/sk-proj-/sk-svcacct- and Anthropic sk-ant- key prefixes. { pattern: /\bsk-(?=[\w-]{16})[\w-]+/g, - replacement: '[REDACTED]', + mask: WHOLE_MATCH, boundary: /\w/, }, { pattern: /\bAIza(?=[\w-]{20})[\w-]+/g, - replacement: '[REDACTED]', + mask: WHOLE_MATCH, boundary: /\w/, }, // Finite authority schemes, not URL validation. Raw ?/# before the first @ may belong - // to a password; after an existing @ they begin public query/fragment text. + // to a username or password; after an existing @ they begin public query/fragment text. { pattern: URL_USERINFO_PATTERN, joinedPattern: URL_USERINFO_PATTERN, - replacement: maskUrlUserinfo, - boundary: /\w/, + mask: (match) => urlUserinfoRange(match[0], () => authorityRest(match)), + credential: (matched) => urlUserinfoRange(matched, () => '') !== undefined, + boundary: NO_BOUNDARY, + anchor: { text: '://', window: URL_SCHEME_WINDOW }, }, - // Exact literal header names, also in quoted diagnostic objects. No arbitrary header discovery. + // Exact literal header names, also in quoted or JSON-escaped diagnostic objects. No arbitrary + // header discovery. A value may begin with up to eight Bearer/Basic labels (possibly quoted, + // escaped, glued behind punctuation, LF/TAB-split or a concatenated run of up to eight, as in + // `BearerBearer`), as a doubled `Bearer Bearer` misconfiguration does; the token after them is + // masked and the labels stay. The token may begin with any run of (escaped) quotes, is never a + // label or a run of labels that whitespace or the end follows, nor another header name, and ends + // at a delimiter, at escapes before one or before a header name glued to it, which then masks its + // own value. A JSON escape after the `:` starts the token, so it takes a value glued to the + // escape (`"x-api-key":"\n"` keeps only the backslash) but not a quoted value behind one + // (`x-api-key:\t""`), a documented limit (diagnostic-secret-backstop.md). Every label repetition is + // bounded so no label run can exhaust the RegExp stack: past eight separate labels the standalone + // label rule masks the token after the last one, and nine concatenated labels blank the diagnostic + // (`LABEL_RUN_PAST_BOUND`). { - pattern: /(^|[^A-Za-z0-9_-])((?:x-)?api-key|x-goog-api-key)(["']?\s*:\s*["']?)[^\s"',;}]+/gi, - replacement: '$1$2$3[REDACTED]', - joinedPattern: /(?:(?:x-)?api-key|x-goog-api-key)["']?\s*:/gi, + pattern: + /(?[^\s"',;}\\][^\s"',;}]*?)(?=(?<=[^A-Za-z0-9_-])(?:(?:x-)?api-key|x-goog-api-key)\\*["']?\s*:|\\{0,15}[\s"',;}]|\\{0,15}$)/dgi, + mask: tokenGroup, + joinedPattern: /(?:(?:x-)?api-key|x-goog-api-key)\\*["']?\s*:/gi, boundary: /[A-Za-z0-9_-]/, }, - // LF/TAB may split the finite scheme LABEL. Preserve the required label/value separator and - // ordinary following prose; opaque value fragments cannot be distinguished from that prose. + // One standalone rule for both labels, so neither can drift from the other. LF/TAB may split the + // finite scheme LABEL, a label may be closed by its own quote before the whitespace + // (`"Bearer" `), and a run of up to eight concatenated labels (`BearerBearer`) counts as one + // label. Preserve the required label/value separator and ordinary following prose; opaque value + // fragments cannot be distinguished from that prose. A label never takes as its token another + // label that whitespace or the end follows, even one glued behind punctuation, quoted or + // concatenated, so any number of stacked labels leaves the last one to mask the token, and a + // token ends before a label (or a concatenated run) glued to it behind a non-word character, + // which then masks its own token. Quotes and `,;}` around a token stay outside it unless it + // continues past them, so `[REDACTED];tail` is masked whole. { - pattern: /\bB[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r\s+\S+/gi, - replacement: 'Bearer [REDACTED]', + pattern: + /\b(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)(?:[\\"']*(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)){0,7}[\\"']*\s+(?![^\w\s]*(?:(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)[\\"']*){1,8}(?:\s|$))[\\"']*(?(?=[^\s\\"'])\S*?[^\s\\"',;}])[\\"',;}]*(?=(?<=[^\w])(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)(?:[\\"']*(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)){0,7}[\\"']*(?:\s|$)|\s|$)/dgi, + mask: tokenGroup, }, - { pattern: /\bB[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c\s+\S+/gi, replacement: 'Basic [REDACTED]' }, ]; +/** + * Exact literal query names with exact boundaries, so benign suffixes stay; no URL-name decoding. + * A fragment carries them too (an OAuth implicit-flow `#access_token=`). Their values join the merged + * masking pass, but no probe joins query names or ordinary query prose. + */ +const QUERY_SECRET_RULE: SecretRule = { + pattern: + /[?&#](?:(?:x[-_])?api[-_]?key|api[-_]?secret|(?:access|secret)[-_]?key|(?:access|refresh|id|auth|session)[-_]?token|client[-_]?secret|key|token|auth|password|secret)=(?[^&\s#]+)/dgi, + mask: tokenGroup, +}; + +/** The merged pass masks the ranges of every rule; the probes run only over the non-query rules. */ +const SECRET_RULES: readonly SecretRule[] = [...NON_QUERY_SECRET_RULES, QUERY_SECRET_RULE]; + +/** + * Nine concatenated labels (`BearerBearer…`, possibly with quotes or escapes between them) exceed + * every bounded label run above, so the token after them cannot be told from them: fail closed. The + * raw text is checked as well as its normalized view, which can drop a label letter as the final + * byte of a terminal sequence (`ESC [` then `BearerBearer…`). + */ +const LABEL_RUN_PAST_BOUND = + /(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)(?:[\\"']*(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)){8}/i; + +/** + * Each boundary-bearing rule compiled once for the split scanner. `splitProbe` checks the scanner's + * two invariants here, at module load, so a misconfigured rule fails fast with a typed error instead of + * a `SyntaxError` inside `makeLlmError` on some later LF/TAB diagnostic. + */ +const SPLIT_PROBES: ReadonlyMap = new Map( + NON_QUERY_SECRET_RULES.flatMap((rule) => + rule.boundary === undefined + ? [] + : [[rule, splitProbe({ ...rule, boundary: rule.boundary })] as const], + ), +); + /** Reuse the display boundary's platform-free normalization without changing benign output. */ function normalizeSecretProbe(text: string): string { return stripTerminalControls(text); } -function isPublicHostPort(text: string): boolean { - const colon = text.lastIndexOf(':'); - if (colon < 0 || !/^\d{1,5}$/.test(text.slice(colon + 1))) return false; - const hostname = text.slice(0, colon); - return hostname.includes('.') || hostname === 'localhost' || /^\[[\da-f:]+\]$/i.test(hostname); +/** A host-shaped prefix: a dotted name (or IPv4), `localhost` or a bracketed IP literal, with an optional port. */ +const HOST_SHAPED_AUTHORITY = + /^(?:[a-z\d-]+(?:\.[a-z\d-]+)+\.?|localhost|\[[\da-f:.]+\])(?::\d{1,5})?$/i; + +/** A DNS name has at most 253 characters (254 with a trailing dot), and a `:port` adds at most six. */ +const HOST_SHAPED_MAX_LENGTH = 260; + +/** + * A raw `?`/`#` before the authority `@` begins public query or fragment text only when the text + * before it is host-shaped **and** an `=` assignment follows it, as in + * `https://host.example?contact=a@b.example`. Otherwise the text is userinfo: `https://us?er:TOKEN@h/`, + * `http://#TOKEN@h`, `https://my.user:1234?TOKEN@h/` and a host fragment without an assignment + * (`https://host.example#sec@x`, failing closed) are masked under the rest of `urlUserinfoRange`'s policy. + * Core's journal projection applies the same predicate (`packages/core/src/tools/bounding.ts`). + */ +function isPublicQueryContact(userinfo: string, query: number): boolean { + // A longer prefix cannot be a hostname, and an unbounded dotted run would exhaust V8's RegExp + // backtrack stack at admitted size; such a prefix stays userinfo, which fails closed. + if (query > HOST_SHAPED_MAX_LENGTH) return false; + return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query)) && userinfo.includes('=', query + 1); +} + +/** An authority's text after its last `@`, up to a path, whitespace, a quote, `,` or `}`, as in core. */ +const AUTHORITY_REST = /[^/\s"',}]*/y; + +function authorityRest(match: RegExpExecArray): string { + AUTHORITY_REST.lastIndex = match.index + match[0].length; + return AUTHORITY_REST.exec(match.input)?.[0] ?? ''; +} + +/** Whether an authority's text after its last `@` begins with a host, up to any query or fragment. */ +function hostFollows(rest: string): boolean { + const end = rest.search(/[?#]/); + const host = end < 0 ? rest : rest.slice(0, end); + // Bounded like the contact prefix, so an admitted-size run cannot exhaust the RegExp stack. + return host.length <= HOST_SHAPED_MAX_LENGTH && HOST_SHAPED_AUTHORITY.test(host); } -function maskUrlUserinfo(match: string): string { +/** + * The userinfo range of one finite-scheme URL match, or `undefined` when the policy keeps it: a + * public query contact, a connection string's colonless user or already-masked userinfo. `rest` + * reads the authority after the match. + */ +function urlUserinfoRange(match: string, rest: () => string): MaskRange | undefined { const start = match.indexOf('://') + 3; const authority = match.slice(start); const firstQuery = authority.search(/[?#]/); @@ -90,40 +251,93 @@ function maskUrlUserinfo(match: string): string { const at = authorityAt >= 0 ? authorityAt : authority.length - 1; const userinfo = authority.slice(0, at); const query = userinfo.search(/[?#]/); + if (query >= 0 && isPublicQueryContact(userinfo, query)) return undefined; + const colonless = COLONLESS_USERINFO_SCHEME.test(match.slice(0, start - 3)); const colon = userinfo.indexOf(':'); - // Raw ?/# may occur inside a malformed password. Preserve host query/fragment assignments, - // including hosts with ports, rather than interpreting their contact values as credentials. - if (query >= 0 && (colon < 0 || colon > query || isPublicHostPort(userinfo.slice(0, query)))) - return match; - return `${match.slice(0, start)}[REDACTED]${authority.slice(at)}`; + // A connection string's colonless userinfo names a user (`ssh://git@host`), not a credential. + if (colon < 0 && !colonless) return undefined; + // Already-masked userinfo stays as it is: an exactly masked password under a connection-string + // scheme, whose usernames are public, or a masked or empty username beside it under any scheme. + // The authored-content detector's projection `postgres://app:[REDACTED]@db` is a fixed point. + const username = userinfo.slice(0, Math.max(colon, 0)); + if ( + colon >= 0 && + userinfo.slice(colon + 1) === MASKED && + (!colonless || username === '' || username === MASKED) + ) + return undefined; + if (query < 0) return { start, end: start + at }; + // A raw ?/# leaves the authority ambiguous: unless a host follows its last `@`, the text there may + // be the rest of a query value (`https://svc?password=P@ss`), so the mask runs through it. + const tail = rest(); + return { start, end: start + at + (hostFollows(tail) ? 0 : 1 + tail.length) }; +} + +/** Every masked range of one rule's matches in `text`. */ +function ruleRanges(text: string, rule: SecretRule): MaskRange[] { + const ranges: MaskRange[] = []; + for (const match of text.matchAll(rule.pattern)) { + const range = rule.mask(match); + if (range !== undefined) { + ranges.push({ start: match.index + range.start, end: match.index + range.end }); + } + } + return ranges; +} + +/** Replace each run of overlapping ranges with one `[REDACTED]`; the rest of `text` is kept. */ +function maskRanges(text: string, ranges: readonly MaskRange[]): string { + const ordered = [...ranges].sort( + (left, right) => left.start - right.start || left.end - right.end, + ); + let output = ''; + let cursor = 0; + let open: { start: number; end: number } | undefined; + for (const range of ordered) { + if (open !== undefined && range.start < open.end) { + open.end = Math.max(open.end, range.end); + continue; + } + if (open !== undefined) { + output += text.slice(cursor, open.start) + MASKED; + cursor = open.end; + } + open = { start: range.start, end: range.end }; + } + if (open === undefined) return text; + return output + text.slice(cursor, open.start) + MASKED + text.slice(open.end); } function applyRule(text: string, rule: SecretRule): string { - const replacement = rule.replacement; - return typeof replacement === 'string' - ? text.replace(rule.pattern, replacement) - : text.replace(rule.pattern, (match) => replacement(match)); + return maskRanges(text, ruleRanges(text, rule)); } /** * Probe LF/TAB joins at eligible ORIGINAL starts before a greedy match can hide a later boundary. * All boundary-bearing rules use this scanner and their canonical matcher (or finite header-label - * override). Prefixes start with ASCII letters; that prefilter skips only impossible starts. + * override), compiled once into `SPLIT_PROBES` (`splitProbe` rejects a rule with no joined form). + * Prefixes start with ASCII letters; + * that prefilter skips only impossible starts. * Keys have one token run after a bounded prefix/minimum. Header labels are bounded and their * trailing whitespace cannot contain another label. URL authority excludes slash, so a later URL - * prefix delimits a failed authority scan. Successful matches advance both cursors over their span; + * prefix delimits a failed authority scan. Successful matches advance both cursors over their span, + * except a kept candidate whose first letter ends a line, which advances one start, once per start; * no failed candidate rescans an admitted-size run at every LF/TAB boundary. * Opaque Bearer/Basic values and literal query names keep their established delimiter policy. */ function hasSplitCredential(text: string, joined: string, rule: SecretRule): boolean { - if (rule.boundary === undefined) return false; - const canonical = rule.joinedPattern ?? rule.pattern; - const source = rule.joinedPattern === undefined ? canonical.source.slice(2) : canonical.source; + const probe = SPLIT_PROBES.get(rule); + if (rule.boundary === undefined || probe === undefined) return false; // Absence under the same boundary-free canonical matcher proves there is no joined // candidate. Presence still requires the original-boundary and split checks below. // Native scanning avoids a JS/coverage branch for every byte of unrelated large text. - if (!new RegExp(source, canonical.flags).test(joined)) return false; - const pattern = new RegExp(source, canonical.flags.replace('g', 'y')); + if (!probe.preflight.test(joined)) return false; + const pattern = probe.sticky; + // Only a rule whose boundary rejects every ASCII word character may skip word-preceded starts; + // `splitProbe` refused any boundary that treats some word characters differently from others. + const wordBlocksStart = probe.wordBlocksStart; + const anchor = probe.anchor; + let anchorAt = -1; let originalOffset = 0; let joinedOffset = 0; while (originalOffset < text.length) { @@ -132,21 +346,44 @@ function hasSplitCredential(text: string, joined: string, rule: SecretRule): boo originalOffset += 1; continue; } - // Both finite boundary classes reject ASCII word characters. Skip those impossible + // An anchored rule (the URL rule's `://`) can match only within its window before the next + // anchor. Past the last anchor no match can start, and letter-heavy text far from any anchor is + // skipped without a RegExp attempt per character. + if (anchor !== undefined && anchorAt < joinedOffset) { + anchorAt = joined.indexOf(anchor.text, joinedOffset); + if (anchorAt < 0) return false; + } + // The key and header boundary classes reject ASCII word characters. Skip those impossible // starts without allocating/testing RegExp objects for every admitted-size character; - // the rule's own boundary and canonical matcher still decide every eligible start. + // the rule's own boundary and canonical matcher still decide every eligible start. The URL + // rule blocks no start, so a glued scheme (`xhttps://`) is examined like any other. const code = text.charCodeAt(originalOffset); const previous = text.charCodeAt(originalOffset - 1); - const letter = (code >= 65 && code <= 90) || (code >= 97 && code <= 122); + const letter = + ((code >= 65 && code <= 90) || (code >= 97 && code <= 122)) && + (anchor === undefined || anchorAt - joinedOffset <= anchor.window); const previousWord = - (previous >= 48 && previous <= 57) || - (previous >= 65 && previous <= 90) || - (previous >= 97 && previous <= 122) || - previous === 95; + wordBlocksStart && + ((previous >= 48 && previous <= 57) || + (previous >= 65 && previous <= 90) || + (previous >= 97 && previous <= 122) || + previous === 95); if (letter && !previousWord && !rule.boundary.test(text[originalOffset - 1] ?? '')) { pattern.lastIndex = joinedOffset; const match = pattern.exec(joined); if (match !== null) { + // A candidate the policy keeps (a connection string's user, a public contact) whose first + // letter ends a line may be a word glued across the break to a shorter listed scheme + // (`files\nftp://` reads as `sftp://`), so the next start is tried instead of its span. + if ( + /[\n\t]/.test(text[originalOffset + 1] ?? '') && + rule.credential !== undefined && + !rule.credential(match[0]) + ) { + originalOffset += 1; + joinedOffset += 1; + continue; + } const start = originalOffset; const joinedStart = joinedOffset; while (joinedOffset < pattern.lastIndex) { @@ -156,7 +393,7 @@ function hasSplitCredential(text: string, joined: string, rule: SecretRule): boo if ( originalOffset - start > joinedOffset - joinedStart && // A malformed URL candidate may actually be a public contact query. - (typeof rule.replacement !== 'function' || rule.replacement(match[0]) !== match[0]) + (rule.credential === undefined || rule.credential(match[0])) ) return true; continue; @@ -168,9 +405,20 @@ function hasSplitCredential(text: string, joined: string, rule: SecretRule): boo return false; } -function scrubNonQuerySecrets(text: string): string { +/** + * One merged masking pass. Every non-query rule probes the original diagnostic first; then the ranges + * of every rule, the literal query names' included, come from the original text and are merged, so no + * rule erases the context another rule needs. + */ +function scrubOnce(text: string): string { const normalized = normalizeSecretProbe(text); + if (LABEL_RUN_PAST_BOUND.test(normalized) || LABEL_RUN_PAST_BOUND.test(text)) return MASKED; const joined = /[\n\t]/.test(normalized) ? normalized.replace(/[\n\t]+/g, '') : undefined; + // Normalization can take a split credential's first letter into a terminal sequence (ESC 'A' + // before `AIza`, ESC '[' before `sk-`), so the raw text's joined view is probed as well when the + // two views differ. + const rawJoined = + normalized !== text && /[\n\t]/.test(text) ? text.replace(/[\n\t]+/g, '') : undefined; for (const rule of NON_QUERY_SECRET_RULES) { const rawMasked = applyRule(text, rule); // Identical inputs have identical pure projections; unchanged raw text already has @@ -179,6 +427,7 @@ function scrubNonQuerySecrets(text: string): string { if ((rawMasked === text ? normalized : normalizeSecretProbe(rawMasked)) !== normalizedMasked) return '[REDACTED]'; if (joined !== undefined && hasSplitCredential(normalized, joined, rule)) return '[REDACTED]'; + if (rawJoined !== undefined && hasSplitCredential(text, rawJoined, rule)) return '[REDACTED]'; // Retain the existing broader whitespace probe only within this rule's still-unmasked spans. // It detects incomplete prefixes separated by spaces without consuming prose after a full mask. for (const remaining of normalizedMasked.split('[REDACTED]')) { @@ -186,13 +435,8 @@ function scrubNonQuerySecrets(text: string): string { if (applyRule(residual, rule) !== residual) return '[REDACTED]'; } } - return NON_QUERY_SECRET_RULES.reduce((masked, rule) => applyRule(masked, rule), text); -} - -/** Exact literal query names, applied last so a value cannot erase another family's evidence. */ -function scrubQuerySecrets(text: string): string { - return text.replace( - /([?&](?:(?:x[-_])?api[-_]?key|api[-_]?secret|(?:access|secret)[-_]?key|(?:access|refresh|id|auth|session)[-_]?token|client[-_]?secret|key|token|auth|password|secret)=)[^&\s#]+/gi, - '$1[REDACTED]', - ); // Finite literal names; exact boundaries preserve benign suffixes. No URL-name decoding. + return maskRanges( + text, + SECRET_RULES.flatMap((rule) => ruleRanges(text, rule)), + ); } diff --git a/packages/llm/src/split-credential-probe.test.ts b/packages/llm/src/split-credential-probe.test.ts new file mode 100644 index 00000000..5333ce4a --- /dev/null +++ b/packages/llm/src/split-credential-probe.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest'; + +import { SplitProbeInvariantError, splitProbe } from './split-credential-probe.js'; + +function invariantCode(rule: Parameters[0]): string | undefined { + try { + splitProbe(rule); + return undefined; + } catch (error) { + if (error instanceof SplitProbeInvariantError) return error.code; + throw error; + } +} + +describe('split-credential scanner invariants', () => { + it('refuses a boundary-bearing rule with neither a joined pattern nor a \\b-prefixed source', () => { + expect(invariantCode({ pattern: /token-[a-z]+/g, boundary: /\w/ })).toBe( + 'joined_form_required', + ); + expect(invariantCode({ pattern: /\btoken-[a-z]+/g, boundary: /\w/ })).toBeUndefined(); + expect( + invariantCode({ pattern: /token-[a-z]+/g, joinedPattern: /token-/g, boundary: /\w/ }), + ).toBeUndefined(); + }); + + it('refuses a boundary that treats some ASCII word characters differently from others', () => { + for (const boundary of [/[a-z]/, /[A-Za-z]/, /\d/, /_/]) { + expect(invariantCode({ pattern: /\bx/g, boundary })).toBe('mixed_word_boundary'); + } + for (const boundary of [/\w/, /[A-Za-z0-9_-]/, /(?!)/]) { + expect(invariantCode({ pattern: /\bx/g, boundary })).toBeUndefined(); + } + }); + + it('reports the refusal as a typed error with a fixed message', () => { + expect(() => splitProbe({ pattern: /x/g, boundary: /\w/ })).toThrow( + new SplitProbeInvariantError('joined_form_required'), + ); + const error = new SplitProbeInvariantError('mixed_word_boundary'); + expect(error.name).toBe('SplitProbeInvariantError'); + expect(error.message).toBe( + 'scrub rule violates the split-credential scanner invariant: mixed_word_boundary', + ); + }); + + it('strips only the \\b of a source without a joined pattern', () => { + const probe = splitProbe({ pattern: /\bsk-[\w-]+/g, boundary: /\w/ }); + expect(probe.preflight.source).toBe('sk-[\\w-]+'); + expect(probe.sticky.source).toBe('sk-[\\w-]+'); + const joined = splitProbe({ pattern: /x/gi, joinedPattern: /\bkey:/gi, boundary: /\w/ }); + expect(joined.preflight.source).toBe('\\bkey:'); + }); + + it('compiles a stateless preflight and a sticky scanner that keep the other flags', () => { + const probe = splitProbe({ pattern: /x/gi, joinedPattern: /api-key:/gi, boundary: /\w/ }); + expect(probe.preflight.flags).toBe('i'); + expect(probe.sticky.flags).toBe('iy'); + // A global preflight would carry lastIndex between diagnostics and miss the second match. + expect(probe.preflight.test('API-KEY:')).toBe(true); + expect(probe.preflight.test('API-KEY:')).toBe(true); + }); + + it('derives the word-start skip from the boundary and carries the anchor through', () => { + expect(splitProbe({ pattern: /\bx/g, boundary: /\w/ }).wordBlocksStart).toBe(true); + const anchor = { text: '://', window: 32 }; + const url = splitProbe({ pattern: /x/g, joinedPattern: /x/g, boundary: /(?!)/, anchor }); + expect(url.wordBlocksStart).toBe(false); + expect(url.anchor).toBe(anchor); + }); +}); diff --git a/packages/llm/src/split-credential-probe.ts b/packages/llm/src/split-credential-probe.ts new file mode 100644 index 00000000..35c44f2b --- /dev/null +++ b/packages/llm/src/split-credential-probe.ts @@ -0,0 +1,78 @@ +/** + * Package-internal support for `scrub-secrets.ts`'s LF/TAB split-credential scanner. It is not a + * package export: the scanner's static rules are compiled once, at module load, into the probes this + * file describes, and a rule that breaks the scanner's two hidden assumptions fails there — with a + * typed error — instead of throwing a `SyntaxError` from inside `makeLlmError` on a later diagnostic. + * It imports nothing, so the pure `@relavium/llm/scrub-secrets` leaf still reaches only `terminal-safe` + * beyond its own package. + */ + +export type SplitProbeInvariantCode = 'joined_form_required' | 'mixed_word_boundary'; + +/** + * A static scrub rule violates a split-credential scanner invariant. A programming error, never + * input-driven, and not an exported error family (like `UsageCoverageInvariantError`). + */ +export class SplitProbeInvariantError extends Error { + readonly code: SplitProbeInvariantCode; + + constructor(code: SplitProbeInvariantCode) { + super(`scrub rule violates the split-credential scanner invariant: ${code}`); + this.name = 'SplitProbeInvariantError'; + this.code = code; + } +} + +/** A literal that every scanner match must reach within `window` joined characters of its start. */ +export interface SplitProbeAnchor { + readonly text: string; + readonly window: number; +} + +export interface SplitProbeRule { + readonly pattern: RegExp; + readonly joinedPattern?: RegExp; + readonly boundary: RegExp; + readonly anchor?: SplitProbeAnchor; +} + +export interface SplitProbe { + /** Non-global, so `test` carries no `lastIndex` state from one diagnostic to the next. */ + readonly preflight: RegExp; + /** Sticky; every use sets `lastIndex` first. */ + readonly sticky: RegExp; + /** The rule's boundary rejects every ASCII word character, so word-preceded starts can be skipped. */ + readonly wordBlocksStart: boolean; + readonly anchor: SplitProbeAnchor | undefined; +} + +const ASCII_WORD_CHARACTERS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_'; + +/** + * Compile one boundary-bearing rule. Two invariants are checked rather than assumed: + * + * - **A joined form exists.** The scanner probes a boundary-free form of the rule. A rule supplies + * it as `joinedPattern`, or its `pattern` begins with `\b`, which is stripped. Any other pattern + * would lose two real characters and become an invalid expression. + * - **The boundary treats every ASCII word character alike.** The scanner skips word-preceded starts + * as a block when the boundary rejects word characters; a boundary that rejected only some of them + * would let the skip hide an eligible start. + */ +export function splitProbe(rule: SplitProbeRule): SplitProbe { + const canonical = rule.joinedPattern ?? rule.pattern; + if (rule.joinedPattern === undefined && !canonical.source.startsWith(String.raw`\b`)) { + throw new SplitProbeInvariantError('joined_form_required'); + } + const verdicts = new Set( + Array.from(ASCII_WORD_CHARACTERS, (character) => rule.boundary.test(character)), + ); + if (verdicts.size !== 1) throw new SplitProbeInvariantError('mixed_word_boundary'); + const source = rule.joinedPattern === undefined ? canonical.source.slice(2) : canonical.source; + const flags = canonical.flags.replace('g', ''); + return { + preflight: new RegExp(source, flags), + sticky: new RegExp(source, `${flags}y`), + wordBlocksStart: verdicts.has(true), + anchor: rule.anchor, + }; +} diff --git a/tools/budget-replay-compat/README.md b/tools/budget-replay-compat/README.md index 364ab4c3..3c1a4178 100644 --- a/tools/budget-replay-compat/README.md +++ b/tools/budget-replay-compat/README.md @@ -144,11 +144,13 @@ they never modify the extracted predecessor, whose archived files can be read-on ten concurrent completions, future timestamps, active/interrupted/foreign/redirected records, current-owner refusal and direct-child ownership. Seven additional finalization cases, grouped into three results, preserve Error/primitive primary failures and a failed diagnostic sink while surfacing -finalization-only failures. Fourteen additional production-caller controls cover six stage write +finalization-only failures. Fifteen additional production-caller controls cover six stage write faults, ten genuinely failed retention children, a native failed spawn, three real closed-stderr -pipes retaining `Error`/`undefined`/`null`, and three post-scan owner/physical replacement claims. +pipes retaining `Error`/`undefined`/`null`, and four post-scan owner/physical replacement claims. The completed replacement deliberately reuses the same owner/completion bytes and proves that -directory identity is required. These controls execute the actual helpers/callers; they do not +directory identity is required. Its collision variant (NI) substitutes every lstat inode with +2^60 + k, distinct as a bigint but one Number, so only an exact bigint directory identity refuses +the lookalike. These controls execute the actual helpers/callers; they do not mirror their implementations. The ESM control uses the actual production loader with an inert, pinned TypeScript stand-in; it loads no TS source and is separate from the real predecessor/source/dependency acceptance workers. diff --git a/tools/budget-replay-compat/evidence-retention.mjs b/tools/budget-replay-compat/evidence-retention.mjs index 8f3495b3..4e76d1e6 100644 --- a/tools/budget-replay-compat/evidence-retention.mjs +++ b/tools/budget-replay-compat/evidence-retention.mjs @@ -16,9 +16,11 @@ import { basename, dirname, join } from 'node:path'; const kind = 'relavium-budget-replay-evidence-v1'; function directory(path) { - const stat = lstatSync(path); + // Exact identities: a 64-bit file ID (NTFS, overlayfs xino) exceeds a Number's precision, and this + // identity alone tells a lookalike replacement from the completed directory before its deletion. + const stat = lstatSync(path, { bigint: true }); assert.ok(stat.isDirectory() && !stat.isSymbolicLink(), 'replay evidence directory redirected'); - if (process.getuid !== undefined) assert.equal(stat.uid, process.getuid()); + if (process.getuid !== undefined) assert.equal(stat.uid, BigInt(process.getuid())); return { dev: stat.dev, ino: stat.ino }; } function readRecord(path) { diff --git a/tools/budget-replay-compat/lifecycle-smoke.mjs b/tools/budget-replay-compat/lifecycle-smoke.mjs index fa861244..6c6bc39d 100644 --- a/tools/budget-replay-compat/lifecycle-smoke.mjs +++ b/tools/budget-replay-compat/lifecycle-smoke.mjs @@ -279,6 +279,28 @@ async function closedDiagnosticControl(root, environment, mode) { return { mode, pid: child.pid, closedPipe: true, actualCheckPrimaryRetained: true, ...ended }; } +/** + * Substitute every lstat identity with 2^60 + k: distinct as bigints but one Number (the spacing there + * is 256), so a lookalike differs from the completed directory only beyond a Number's precision. + */ +function exactIdentityLstat() { + const lstat = fs.lstatSync; + const fakes = new Map(); + return (path, ...args) => { + const info = lstat(path, ...args); + const exact = lstat(path, { bigint: true, throwIfNoEntry: false }); + if (info !== undefined && exact !== undefined) { + const key = `${exact.dev}:${exact.ino}`; + const value = fakes.get(key) ?? (1n << 60n) + BigInt(fakes.size); + fakes.set(key, value); + Object.defineProperty(info, 'ino', { + value: typeof info.ino === 'bigint' ? value : Number(value), + }); + } + return info; + }; +} + async function retirementIntervention(root, mode) { const fixture = join(root, `retirement-${mode}`); const repository = join(fixture, 'repository'); @@ -299,7 +321,8 @@ async function retirementIntervention(root, mode) { const current = allocateEvidence(repository, temporary); const owner = fs.readFileSync(join(candidate.owned, 'OWNER.json')); const completion = fs.readFileSync(join(candidate.owned, 'completion.json')); - const inode = fs.lstatSync(candidate.owned).ino; + const collision = mode === 'completed-replacement-collision'; + let inode = fs.lstatSync(candidate.owned, { bigint: true }).ino; const read = fs.readFileSync; const readdir = fs.readdirSync; let validated = false; @@ -307,6 +330,7 @@ async function retirementIntervention(root, mode) { let caught; await overrideBuiltins( [ + ...(collision ? [[fs, 'lstatSync', exactIdentityLstat()]] : []), [ fs, 'readdirSync', @@ -336,12 +360,15 @@ async function retirementIntervention(root, mode) { JSON.stringify({ ...JSON.parse(owner.toString('utf8')), repository: fixture }), ); } else { + if (collision) inode = fs.lstatSync(candidate.owned, { bigint: true }).ino; fs.renameSync(candidate.owned, join(fixture, 'saved-completed')); fs.mkdirSync(candidate.owned); fs.writeFileSync(join(candidate.owned, 'OWNER.json'), owner); - if (mode === 'completed-replacement') + if (mode !== 'active-replacement') fs.writeFileSync(join(candidate.owned, 'completion.json'), completion); - assert.notEqual(fs.lstatSync(candidate.owned).ino, inode); + const replaced = fs.lstatSync(candidate.owned, { bigint: true }).ino; + assert.notEqual(replaced, inode); + if (collision) assert.equal(Number(replaced), Number(inode)); } fs.writeFileSync(join(candidate.owned, 'sentinel'), 'preserve unexpected claim'); intervened = true; @@ -382,7 +409,12 @@ export async function checkLifecycleGuards(owned, environment) { }; for (const mode of ['Error', 'undefined', 'null']) results.diagnosticPipes.push(await closedDiagnosticControl(root, environment, mode)); - for (const mode of ['foreign-owner', 'active-replacement', 'completed-replacement']) + for (const mode of [ + 'foreign-owner', + 'active-replacement', + 'completed-replacement', + 'completed-replacement-collision', + ]) results.retirementClaims.push(await retirementIntervention(root, mode)); fs.writeFileSync(join(root, 'results.json'), `${JSON.stringify(results, null, 2)}\n`); return results; diff --git a/tools/bundle-closure/check.mjs b/tools/bundle-closure/check.mjs index 2aa9bb81..85887c96 100644 --- a/tools/bundle-closure/check.mjs +++ b/tools/bundle-closure/check.mjs @@ -1,45 +1,24 @@ -/** Thin filesystem/exit adapter for the ADR-0111 emitted ESM graph guard. */ -import { lstatSync, readFileSync, readdirSync } from 'node:fs'; -import { join, relative, sep } from 'node:path'; +/** Thin filesystem/exit adapter for the ADR-0111 emitted ESM graph guard of an ordinary build. */ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { BundleClosureError, checkBundleClosure } from './closure.mjs'; +import { MODE_EVIDENCE, emittedInventory } from './inventory.mjs'; -const modeEvidence = process.platform === 'win32' ? 'not-observable-windows' : 'posix'; const packageDirectory = fileURLToPath(new URL('../../apps/cli/', import.meta.url)); -function inventory(directory, files = {}) { - if (!lstatSync(directory).isDirectory()) { - throw new BundleClosureError('OUTPUT_KIND', 'dist inventory must use ordinary directories'); - } - for (const entry of readdirSync(directory, { withFileTypes: true })) { - const path = join(directory, entry.name); - if (entry.isDirectory()) inventory(path, files); - else if (/\.(?:js|mjs|cjs)$/iu.test(entry.name)) { - const key = relative(packageDirectory, path).split(sep).join('/'); - const stat = lstatSync(path); - files[key] = { - contents: entry.isFile() ? readFileSync(path, 'utf8') : '', - ...(modeEvidence === 'posix' ? { mode: stat.mode & 0o7777 } : {}), - kind: entry.isFile() ? 'file' : 'other', - }; - } else if (entry.isSymbolicLink()) { - // Never follow a directory symlink that could hide runtime chunks. - throw new BundleClosureError('OUTPUT_KIND', 'dist contains a symbolic link'); - } - } - return files; -} - try { + // An ordinary build is map-free: its metafile is the only unpublished file it may leave in dist. const report = checkBundleClosure({ metafile: JSON.parse(readFileSync(join(packageDirectory, 'dist/metafile-esm.json'), 'utf8')), manifest: JSON.parse(readFileSync(join(packageDirectory, 'package.json'), 'utf8')), - files: inventory(join(packageDirectory, 'dist')), - modeEvidence, + files: emittedInventory(packageDirectory, { privateFiles: ['metafile-esm.json'] }), + modeEvidence: MODE_EVIDENCE, + maps: 'none', }); console.log( `✓ CLI finite ESM closure: ${report.outputs.length} output(s), ${report.inputs.length} input(s), ` + - `${report.dependencies.length} declared external dependencies; engine inlined; ` + + `${report.dependencies.length} declared external dependencies; engine inlined; map-free; ` + (report.modeEvidence === 'posix' ? 'POSIX modes verified.' : 'POSIX modes not observable on Windows (graph evidence only).'), diff --git a/tools/bundle-closure/closure.mjs b/tools/bundle-closure/closure.mjs index 95e1ccf6..dfd25708 100644 --- a/tools/bundle-closure/closure.mjs +++ b/tools/bundle-closure/closure.mjs @@ -1,13 +1,21 @@ /** * Finite emitted ESM graph guard (ADR-0111). No filesystem reads or process state: * the CLI supplies an inventory, and controls can remove/change any independent - * witness. This checks emitted files, not a tarball or Step 9 map custody. + * witness. This checks emitted files, not a tarball. A finalized candidate's maps are + * checked here only for pairing; the candidate finalizer owns their content and custody. */ import { Buffer } from 'node:buffer'; import { isBuiltin } from 'node:module'; import { posix } from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; import ts from 'typescript'; +/** Code-unit order: deterministic and locale-independent, as `Array#sort` without a comparator. */ +function byCodeUnit(left, right) { + if (left < right) return -1; + return left > right ? 1 : 0; +} + export class BundleClosureError extends Error { constructor(code, detail) { super(`${code}: ${detail}`); @@ -203,11 +211,60 @@ function actualImports(contents, path) { return edges; } +/** + * Separate external maps from runtime outputs. An ordinary build is map-free. A finalized + * candidate pairs every runtime output with exactly one map, staged outside the package, whose + * metadata entry carries no graph information of its own. + */ +function runtimeOutputs(outputs, directory, maps) { + const runtime = {}; + const staged = new Set(); + for (const [path, output] of Object.entries(outputs)) { + if (!path.endsWith('.map')) { + runtime[path] = output; + continue; + } + if (maps !== 'staged') refuse('MAP_OUTPUT', `ordinary build emitted a source map: ${path}`); + outputPath(path.slice(0, -'.map'.length), directory); + const entry = record(output, `map output ${path}`); + bytes(entry.bytes, `map output ${path}.bytes`); + if ( + !isDeepStrictEqual(Object.keys(entry).sort(byCodeUnit), [ + 'bytes', + 'exports', + 'imports', + 'inputs', + ]) || + Object.keys(record(entry.inputs, `map output ${path}.inputs`)).length > 0 || + strings(entry.exports, `map output ${path}.exports`).length > 0 || + imports(entry.imports, `map output ${path}`).length > 0 + ) { + refuse('MAP_OUTPUT', `staged map metadata carries graph information: ${path}`); + } + staged.add(path); + } + if (maps === 'staged') { + for (const path of Object.keys(runtime)) { + if (!staged.has(`${path}.map`)) { + refuse('MAP_PAIRING', `runtime output has no staged map: ${path}`); + } + } + for (const path of staged) { + if (!Object.hasOwn(runtime, path.slice(0, -'.map'.length))) { + refuse('MAP_PAIRING', `staged map has no runtime output: ${path}`); + } + } + } + return runtime; +} + /** * @param {{metafile: unknown, manifest: unknown, * files: Record, * outputDirectory?: string, entryPoint?: string, - * modeEvidence?: "posix" | "not-observable-windows"}} options + * modeEvidence?: "posix" | "not-observable-windows", + * maps?: "none" | "staged"}} options `staged` admits a finalized candidate's paired map + * metadata; the map files themselves must already be outside the inventoried package. * @returns {{outputs: string[], dependencies: string[], inputs: string[], modeEvidence: string}} * @throws {BundleClosureError} when an emitted graph invariant fails. */ @@ -218,6 +275,7 @@ export function checkBundleClosure({ outputDirectory = 'dist', entryPoint = 'src/index.ts', modeEvidence = 'posix', + maps = 'none', }) { if (!/^[a-zA-Z0-9_-]+$/u.test(outputDirectory)) { refuse('OUTPUT_PATH', 'outputDirectory must be one plain directory name'); @@ -225,10 +283,13 @@ export function checkBundleClosure({ if (!['posix', 'not-observable-windows'].includes(modeEvidence)) { refuse('MODE_EVIDENCE', 'mode evidence must name its observable platform scope'); } + if (!['none', 'staged'].includes(maps)) { + refuse('MAP_MODE', 'map mode must be none or staged'); + } sourcePath(entryPoint, 'entryPoint'); const meta = record(metafile, 'metafile'); const inputs = record(meta.inputs, 'metafile.inputs'); - const outputs = record(meta.outputs, 'metafile.outputs'); + const outputs = runtimeOutputs(record(meta.outputs, 'metafile.outputs'), outputDirectory, maps); const dependencies = record(record(manifest, 'manifest').dependencies, 'manifest.dependencies'); record(files, 'files'); const declared = new Set(Object.keys(dependencies)); @@ -310,6 +371,10 @@ export function checkBundleClosure({ } else if (file.mode !== undefined) { refuse('MODE_EVIDENCE', 'Windows graph evidence cannot assert unobservable POSIX modes'); } + // Neither an ordinary build nor a finalized candidate may link a map from runtime code. + if (/sourceMappingURL\s*=/u.test(file.contents)) { + refuse('MAP_LINK', `runtime output links a source map: ${path}`); + } if (Buffer.byteLength(file.contents, 'utf8') !== output.bytes) { refuse('OUTPUT_BYTES', `output byte inventory disagrees: ${path}`); } diff --git a/tools/bundle-closure/closure.test.mjs b/tools/bundle-closure/closure.test.mjs index 404bb0a4..f8245562 100644 --- a/tools/bundle-closure/closure.test.mjs +++ b/tools/bundle-closure/closure.test.mjs @@ -17,6 +17,7 @@ import { dirname, join, relative, sep } from 'node:path'; import { after, test } from 'node:test'; import { fileURLToPath } from 'node:url'; import { BundleClosureError, checkBundleClosure } from './closure.mjs'; +import { emittedInventory } from './inventory.mjs'; // Use the exact esbuild installed behind the existing CLI tsup tool, not a new // dependency, a copied metadata generator, or the product build. Everything is @@ -77,6 +78,47 @@ async function fixture(splitting = true) { }; } const original = await fixture(); + +// Every fixture is built before the first test is registered: a top-level await after a registration +// lets the runner finish the earlier tests and run this file's `after` hook, which removes `scratch`. +// A finalized candidate (Wave 3 Step 9a.1): esbuild emitted external maps that the finalizer moved +// out of the package; their metafile entries remain and must pair exactly with runtime outputs. +async function stagedFixture() { + const built = await build({ + absWorkingDir: scratch, + entryPoints: { index: 'src/index.ts' }, + outdir: 'dist', + format: 'esm', + platform: 'node', + target: 'node22', + bundle: true, + splitting: true, + minify: true, + metafile: true, + write: false, + sourcemap: 'external', + sourcesContent: false, + external: ['commander', '@modelcontextprotocol/sdk', '@modelcontextprotocol/sdk/*'], + }); + const files = {}; + for (const output of built.outputFiles) { + const path = relative(scratch, output.path).split(sep).join('/'); + if (path.endsWith('.map')) continue; + files[path] = { + contents: output.text, + mode: path === 'dist/index.js' ? 0o755 : 0o644, + kind: 'file', + }; + } + return { + metafile: built.metafile, + manifest: { dependencies: { commander: '1', '@modelcontextprotocol/sdk': '1' } }, + files, + maps: 'staged', + }; +} +const stagedOriginal = await stagedFixture(); +const staged = () => JSON.parse(JSON.stringify(stagedOriginal)); function candidate() { return JSON.parse(JSON.stringify(original)); } @@ -383,6 +425,10 @@ test('JS byte drift, unexpected permissions and hashbang drift each refuse', () const path = entry ? 'dist/index.js' : lazyFile(value); value.files[path].mode = entry ? 0o644 : 0o755; refuses(value, 'OUTPUT_MODE'); + // A set-id or sticky bit is part of the mode, not ignored beside it. + const special = candidate(); + special.files[path].mode = entry ? 0o4755 : 0o2644; + refuses(special, 'OUTPUT_MODE'); const other = candidate(); setContents( other, @@ -442,7 +488,7 @@ test('thin CLI inventories actual files from its package root and refuses missin join(root, 'node_modules'), process.platform === 'win32' ? 'junction' : 'dir', ); - for (const name of ['check.mjs', 'closure.mjs']) { + for (const name of ['check.mjs', 'closure.mjs', 'inventory.mjs']) { copyFileSync(fileURLToPath(new URL(name, import.meta.url)), join(tooling, name)); } writeFileSync(join(pkg, 'package.json'), JSON.stringify(original.manifest)); @@ -461,6 +507,14 @@ test('thin CLI inventories actual files from its package root and refuses missin return result; } assert.equal(run().status, 0); + // An ordinary build is map-free: a planted map or unreviewed file in dist refuses. + writeFileSync(join(pkg, 'dist/index.js.map'), '{}'); + assert.match(run().stderr, /MAP_OUTPUT/u); + rmSync(join(pkg, 'dist/index.js.map')); + writeFileSync(join(pkg, 'dist/notes.txt'), 'x'); + assert.match(run().stderr, /EXTRA_OUTPUT/u); + rmSync(join(pkg, 'dist/notes.txt')); + assert.equal(run().status, 0); writeFileSync(join(pkg, 'dist/extra.js'), 'export {};'); assert.match(run().stderr, /EXTRA_OUTPUT/u); rmSync(join(pkg, 'dist/extra.js')); @@ -490,3 +544,181 @@ test('POSIX modes require actual evidence; Windows graph evidence is explicitly missing.modeEvidence = 'unverified'; refuses(missing, 'MODE_EVIDENCE'); }); + +test('a finalized candidate passes with paired staged maps; an ordinary build may not carry maps', () => { + const value = staged(); + const maps = Object.keys(value.metafile.outputs).filter((path) => path.endsWith('.map')); + assert.ok(maps.length >= 4); + assert.equal(checkBundleClosure(value).outputs.length, maps.length); + value.maps = 'none'; + refuses(value, 'MAP_OUTPUT'); + const implicit = staged(); + delete implicit.maps; + refuses(implicit, 'MAP_OUTPUT'); +}); + +for (const [label, mutate, code] of [ + [ + 'an unpaired runtime output', + (value) => delete value.metafile.outputs['dist/index.js.map'], + 'MAP_PAIRING', + ], + [ + 'an orphan map', + (value) => + (value.metafile.outputs['dist/orphan.js.map'] = { + imports: [], + exports: [], + inputs: {}, + bytes: 1, + }), + 'MAP_PAIRING', + ], + [ + 'map metadata with inputs', + (value) => + (value.metafile.outputs['dist/index.js.map'].inputs = { + 'src/index.ts': { bytesInOutput: 1 }, + }), + 'MAP_OUTPUT', + ], + [ + 'map metadata with imports', + (value) => + value.metafile.outputs['dist/index.js.map'].imports.push({ + path: 'commander', + kind: 'import-statement', + external: true, + }), + 'MAP_OUTPUT', + ], + [ + 'map metadata with an entry point', + (value) => (value.metafile.outputs['dist/index.js.map'].entryPoint = 'src/index.ts'), + 'MAP_OUTPUT', + ], + [ + 'map metadata with exports', + (value) => (value.metafile.outputs['dist/index.js.map'].exports = ['run']), + 'MAP_OUTPUT', + ], + [ + 'a nested map', + (value) => + (value.metafile.outputs['dist/nested/index.js.map'] = { + imports: [], + exports: [], + inputs: {}, + bytes: 1, + }), + 'OUTPUT_PATH', + ], + ...[-1, 1.5, '7'].map((bytes) => [ + `map metadata with malformed bytes ${JSON.stringify(bytes)}`, + (value) => (value.metafile.outputs['dist/index.js.map'].bytes = bytes), + 'METADATA_SHAPE', + ]), + ['an unknown map mode', (value) => (value.maps = 'external'), 'MAP_MODE'], +]) { + test(`staged closure refuses ${label}`, () => { + const value = staged(); + mutate(value); + refuses(value, code); + }); +} + +test('runtime code may never link a map, in an ordinary build or a finalized candidate', () => { + // Node honours both comment forms; V8 and DevTools also honour the legacy `@` marker. + for (const link of [ + '//# sourceMappingURL=index.js.map', + '//@ sourceMappingURL=index.js.map', + '/*# sourceMappingURL=index.js.map */', + ]) { + for (const value of [candidate(), staged()]) { + setContents(value, 'dist/index.js', `${value.files['dist/index.js'].contents}${link}`); + refuses(value, 'MAP_LINK'); + } + } +}); + +test('the emitted inventory refuses maps and unreviewed files and admits only named private files', () => { + const pkg = mkdtempSync(join(scratch, 'inventory-')); + mkdirSync(join(pkg, 'dist')); + writeFileSync(join(pkg, 'dist/index.js'), '#!/usr/bin/env node\n'); + writeFileSync(join(pkg, 'dist/metafile-esm.json'), '{}'); + assert.deepEqual(Object.keys(emittedInventory(pkg, { privateFiles: ['metafile-esm.json'] })), [ + 'dist/index.js', + ]); + // An unnamed build file is not private: a finalized candidate's dist holds no metafile at all. + assert.throws( + () => emittedInventory(pkg), + (error) => error instanceof BundleClosureError && error.code === 'EXTRA_OUTPUT', + ); + for (const [path, code] of [ + ['dist/index.js.map', 'MAP_OUTPUT'], + ['dist/nested/chunk.js.map', 'MAP_OUTPUT'], + ['dist/notes.txt', 'EXTRA_OUTPUT'], + ['dist/nested/metafile-esm.json', 'EXTRA_OUTPUT'], + ]) { + mkdirSync(dirname(join(pkg, path)), { recursive: true }); + writeFileSync(join(pkg, path), 'x'); + assert.throws( + () => emittedInventory(pkg, { privateFiles: ['metafile-esm.json'] }), + (error) => error instanceof BundleClosureError && error.code === code, + path, + ); + rmSync(join(pkg, path)); + } +}); + +test('the emitted inventory never lists dist through a link', () => { + const pkg = mkdtempSync(join(scratch, 'linked-dist-')); + mkdirSync(join(pkg, 'real-dist')); + writeFileSync(join(pkg, 'real-dist', 'index.js'), '#!/usr/bin/env node\n'); + symlinkSync( + join(pkg, 'real-dist'), + join(pkg, 'dist'), + process.platform === 'win32' ? 'junction' : 'dir', + ); + assert.throws( + () => emittedInventory(pkg), + (error) => error instanceof BundleClosureError && error.code === 'OUTPUT_KIND', + ); +}); + +test( + 'the emitted inventory records a special file named like a chunk without reading it', + { skip: process.platform === 'win32' }, + () => { + const pkg = mkdtempSync(join(scratch, 'fifo-')); + mkdirSync(join(pkg, 'dist')); + assert.equal(spawnSync('mkfifo', [join(pkg, 'dist', 'chunk.js')]).status, 0); + // Reading a FIFO would block until a writer appears, so the inventory runs in a bounded child. + const run = spawnSync( + process.execPath, + [ + '--input-type=module', + '--eval', + `import { emittedInventory } from ${JSON.stringify(new URL('inventory.mjs', import.meta.url).href)}; +process.stdout.write(JSON.stringify(emittedInventory(${JSON.stringify(pkg)})));`, + ], + { encoding: 'utf8', timeout: 10_000, killSignal: 'SIGKILL' }, + ); + assert.equal(run.signal, null, 'the inventory blocked on a FIFO'); + assert.equal(run.status, 0, run.stderr); + const { kind, contents } = JSON.parse(run.stdout)['dist/chunk.js']; + assert.deepEqual({ kind, contents }, { kind: 'other', contents: '' }); + }, +); + +test( + 'the emitted inventory reports set-id and sticky bits in the mode', + { skip: process.platform === 'win32' }, + () => { + const pkg = mkdtempSync(join(scratch, 'special-mode-')); + mkdirSync(join(pkg, 'dist')); + writeFileSync(join(pkg, 'dist', 'index.js'), '#!/usr/bin/env node\n'); + chmodSync(join(pkg, 'dist', 'index.js'), 0o4755); + assert.equal(emittedInventory(pkg)['dist/index.js'].mode, 0o4755); + }, +); diff --git a/tools/bundle-closure/inventory.mjs b/tools/bundle-closure/inventory.mjs new file mode 100644 index 00000000..2bfc1ade --- /dev/null +++ b/tools/bundle-closure/inventory.mjs @@ -0,0 +1,47 @@ +/** Filesystem inventory for the ADR-0111 emitted graph guard. Never follows a link. */ +import { lstatSync, readFileSync, readdirSync } from 'node:fs'; +import { join, relative, sep } from 'node:path'; +import { BundleClosureError } from './closure.mjs'; + +/** POSIX modes are actual evidence only where the platform exposes them. */ +export const MODE_EVIDENCE = process.platform === 'win32' ? 'not-observable-windows' : 'posix'; + +/** + * Inventory every emitted JS file below `/dist`. A map, an unknown file kind or a + * private build file other than the named ones refuses: an ordinary build is map-free, and a + * finalized candidate has already moved its maps and build metadata out of the package. + * @param {string} packageDirectory + * @param {{privateFiles?: readonly string[]}} [options] dist-root files that may remain unpublished + * @returns {Record} + */ +export function emittedInventory(packageDirectory, { privateFiles = [] } = {}) { + const files = {}; + const root = join(packageDirectory, 'dist'); + function visit(directory) { + if (!lstatSync(directory).isDirectory()) { + throw new BundleClosureError('OUTPUT_KIND', 'dist inventory must use ordinary directories'); + } + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + const key = relative(packageDirectory, path).split(sep).join('/'); + if (entry.isDirectory()) visit(path); + else if (entry.isSymbolicLink()) { + // Never follow a link that could hide runtime chunks. + throw new BundleClosureError('OUTPUT_KIND', 'dist contains a symbolic link'); + } else if (/\.(?:js|mjs|cjs)$/iu.test(entry.name)) { + const stat = lstatSync(path); + files[key] = { + contents: entry.isFile() ? readFileSync(path, 'utf8') : '', + ...(MODE_EVIDENCE === 'posix' ? { mode: stat.mode & 0o7777 } : {}), + kind: entry.isFile() ? 'file' : 'other', + }; + } else if (/\.map$/iu.test(entry.name)) { + throw new BundleClosureError('MAP_OUTPUT', `dist contains a source map: ${key}`); + } else if (!(directory === root && privateFiles.includes(entry.name))) { + throw new BundleClosureError('EXTRA_OUTPUT', `dist contains an unreviewed file: ${key}`); + } + } + } + visit(root); + return files; +} diff --git a/tools/causal-controls/README.md b/tools/causal-controls/README.md index c46dbe90..cedf34ef 100644 --- a/tools/causal-controls/README.md +++ b/tools/causal-controls/README.md @@ -128,6 +128,13 @@ scratch tree through a symlink. The bound includes the final export record: at m failure. Selected runs never claim complete-matrix acceptance. Source HEAD and working-tree capture attribution describe the execution actually performed. +`--shard /` deals the declared controls round-robin in declaration +order, so the `count` shards partition the family; each shard is a selection with +`complete: false` and its own export, and records `shard: { index, count, declared, +members }`, so the shards' summaries can be joined against the declaration. Only every +shard of one count passing, each within the export bounds, covers the family. The +secrets family runs as two shards because one export of all its controls exceeds the +64-MiB bound; a full unsharded run still executes every control, but it cannot export. The permanent coordinator integration guard runs three bounded actual captures. Each family proves synthetic fixed/red/restored execution before a real assertion diff --git a/tools/causal-controls/provenance.mjs b/tools/causal-controls/provenance.mjs index a90c5876..0bdcb3e6 100644 --- a/tools/causal-controls/provenance.mjs +++ b/tools/causal-controls/provenance.mjs @@ -13,8 +13,9 @@ import { digest } from '../usage-coverage-controls/source-graph.mjs'; export function selectControls(mutations, requested) { assert.ok( - requested.length === 0 || (requested.length === 2 && requested[0] === '--mechanism'), - 'Usage: check.mjs [--mechanism declared-id]', + requested.length === 0 || + (requested.length === 2 && ['--mechanism', '--shard'].includes(requested[0])), + 'Usage: check.mjs [--mechanism declared-id | --shard index/count]', ); assert.ok( mutations.length > 0 && new Set(mutations.map((entry) => entry.id)).size === mutations.length, @@ -28,6 +29,12 @@ export function selectControls(mutations, requested) { 'Empty declared inventory', ); } + if (requested[0] === '--shard') { + const { index, count } = parseShard(requested[1]); + assert.ok(count <= mutations.length, 'Shard out of range'); + // Declaration order dealt round-robin: every control belongs to exactly one of the shards. + return mutations.filter((_, position) => position % count === index - 1); + } const selected = requested.length ? mutations.filter((entry) => entry.id === requested[1]) : mutations; @@ -35,6 +42,16 @@ export function selectControls(mutations, requested) { return selected; } +/** `index/count`, 1-based, as `--shard` takes it. */ +export function parseShard(value) { + const match = /^([1-9]\d{0,2})\/([1-9]\d{0,2})$/.exec(value ?? ''); + assert.ok(match, 'Usage: --shard index/count'); + const index = Number(match[1]); + const count = Number(match[2]); + assert.ok(index <= count, 'Shard out of range'); + return { index, count }; +} + export function ownedTools(repository, roots) { const files = []; function walk(path) { diff --git a/tools/causal-controls/provenance.test.mjs b/tools/causal-controls/provenance.test.mjs index d1f19400..f07879d7 100644 --- a/tools/causal-controls/provenance.test.mjs +++ b/tools/causal-controls/provenance.test.mjs @@ -154,6 +154,21 @@ test('native child and target need same pinned child PID, parent link and depend assert.throws(() => assertEvaluations(candidate, { files }, target.path, graph, options)); }); }); +test('shards deal the declaration round-robin into disjoint, complete parts', () => { + const entries = ['a', 'b', 'c', 'd', 'e'].map((id) => ({ + id, + before: 'old', + after: 'new', + expectedTests: 1, + })); + const ids = (args) => selectControls(entries, args).map((entry) => entry.id); + assert.deepEqual(ids(['--shard', '1/2']), ['a', 'c', 'e']); + assert.deepEqual(ids(['--shard', '2/2']), ['b', 'd']); + assert.deepEqual(ids(['--shard', '1/1']), ['a', 'b', 'c', 'd', 'e']); + for (const shard of ['0/2', '3/2', '1/6', '1/0', '01/2', '1/2/3', 'x/2', '', '-1/2']) + assert.throws(() => selectControls(entries, ['--shard', shard])); + assert.throws(() => selectControls(entries, ['--shard'])); +}); test('unknown, ambiguous, empty and ineffective selection refuse before work', () => { const entry = { id: 'declared', before: 'old', after: 'new', expectedTests: 1 }; assert.deepEqual(selectControls([entry], ['--mechanism', 'declared']), [entry]); @@ -267,10 +282,10 @@ test('every declared filesystem and secrets anchor matches current production so }); test('secrets workload corrections retain independent complete-count and reader-witness pins', () => { - assert.equal(secretsControls.length, 57); + assert.equal(secretsControls.length, 130); assert.equal( secretsControls.reduce((count, control) => count + 1 + (control.witnesses ?? []).length, 0), - 336, + 513, ); for (const [id, file, expectedTests] of [ ['run-default-secret-reader', 'apps/cli/src/commands/run.test.ts', 84], @@ -309,7 +324,42 @@ test('secrets workload corrections retain independent complete-count and reader- 'core-raw-password-userinfo', 'core-joined-original-boundary-recovery', 'core-header-original-boundary-recovery', - ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 142]), + 'core-userinfo-contact-assignment', + 'core-userinfo-contact-host', + 'core-userinfo-host-length', + 'core-userinfo-host-anchors', + 'core-userinfo-split-scheme', + 'core-userinfo-split-edges', + 'core-fixed-point', + // Round 6: core's merged ranges and the mechanisms they read through. + 'core-merged-ranges', + 'core-userinfo-query-zone', + 'core-key-value-nested', + 'core-key-value-through-masks', + 'core-scheme-nested-label', + 'core-mask-boundaries', + // Round 7: userinfo ownership, the quoted-value scan, the summary and the journal. + 'core-userinfo-username-zone', + 'core-split-zone-cap', + 'core-quoted-value-scan', + 'core-summary-fixed-point', + 'core-journal-settled', + // Round 8: structured summaries, escaped and unclosed quotes, continuation labels. + 'core-summary-walk', + 'core-escaped-quote', + 'core-unclosed-run', + 'core-unclosed-closing-view', + 'core-continuation-label', + // Pre-review: the summary's closer and whole-JSON reading, colliding names, values one JSON level down. + 'core-summary-through-closer', + 'core-summary-whole-json', + 'core-journal-key-collision', + 'core-escaped-value-start', + // Sixth pre-review and round 9: a summary's escaped separators and its unwalked reading. + 'core-summary-escape-view', + 'core-summary-pem-markers', + ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 234]), + ['core-unsettled-refusal', ['packages/core/src/tools/registry.test.ts'], 116], ...[ 'diagnostic-joined-before-masking', 'diagnostic-joined-original-boundary', @@ -317,7 +367,49 @@ test('secrets workload corrections retain independent complete-count and reader- 'diagnostic-raw-password-userinfo', 'diagnostic-joined-original-boundary-recovery', 'diagnostic-header-original-boundary-recovery', - ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 103]), + 'diagnostic-header-scheme-label', + 'diagnostic-userinfo-contact-assignment', + 'diagnostic-userinfo-contact-host', + 'diagnostic-connection-scheme-userinfo', + 'diagnostic-joined-url-userinfo', + 'diagnostic-url-glued-scheme', + 'diagnostic-url-scheme-window', + 'diagnostic-header-stacked-labels', + 'diagnostic-header-label-whitespace', + 'diagnostic-userinfo-host-length', + 'diagnostic-userinfo-host-anchors', + 'diagnostic-header-label-bound', + 'diagnostic-header-concat-bound', + 'diagnostic-header-token-not-label', + 'diagnostic-header-not-nested', + 'diagnostic-header-nested-stop', + 'diagnostic-header-lookbehind', + 'diagnostic-header-escaped-separator', + 'diagnostic-header-quote-run', + 'diagnostic-header-escaped-stop', + 'diagnostic-label-token-not-label', + 'diagnostic-label-quoted-label', + 'diagnostic-label-glued-label', + 'diagnostic-label-embedded-stop', + 'diagnostic-label-masked-tail', + 'diagnostic-merged-masking', + 'diagnostic-mask-merge', + 'diagnostic-token-only-mask', + // Round 4: one label rule for both labels, and each new leak or totality mechanism. + ...['concat-run', 'concat-stop', 'start-bound', 'lead', 'run-bound'].map( + (mechanism) => `diagnostic-label-${mechanism}`, + ), + 'diagnostic-header-concat-run', + 'diagnostic-query-merge', + 'diagnostic-mask-merge-extent', + 'diagnostic-fixed-point', + 'diagnostic-userinfo-authority-rest', + 'diagnostic-userinfo-rest-length', + 'diagnostic-url-split-retry', + 'diagnostic-split-probe-raw', + 'diagnostic-label-run-normalized', + 'diagnostic-label-run-raw', + ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 224]), ...['diagnostic-inline-projection-order', 'diagnostic-original-control-boundary'].map((id) => [ id, ['apps/cli/src/render/sanitize.test.ts', 'apps/cli/src/process/render-error.test.ts'], diff --git a/tools/causal-controls/run.mjs b/tools/causal-controls/run.mjs index d03ae37c..0dd6d693 100644 --- a/tools/causal-controls/run.mjs +++ b/tools/causal-controls/run.mjs @@ -36,6 +36,7 @@ import { mergeFiles, assertEvaluations, selectControls, + parseShard, phaseEnvironment, exactReplacement, assertSelectedAnchors, @@ -53,12 +54,23 @@ export async function runControls({ repository, evidenceKind, mutations, toolRoo const vitest = join(dirname(require.resolve('vitest/package.json')), 'vitest.mjs'); const requested = process.argv.slice(2); const selected = selectControls(mutations, requested); + // A shard is a selection like any other, so it never claims the complete family; it records which + // controls it was dealt, so the shards' summaries can be joined against the declaration. + const shard = + requested[0] === '--shard' + ? { + ...parseShard(requested[1]), + declared: mutations.length, + members: selected.map((entry) => entry.id), + } + : undefined; const scratch = realpathSync(mkdtempSync(join(tmpdir(), `relavium-${evidenceKind}-controls-`))); const graph = join(scratch, 'graph'); const summary = { version: 1, complete: false, node: process.version, + ...(shard === undefined ? {} : { shard }), mechanisms: [], restorations: [], }; diff --git a/tools/ci/require-packed-cli.test.mjs b/tools/ci/require-packed-cli.test.mjs index 94272154..3a143984 100644 --- a/tools/ci/require-packed-cli.test.mjs +++ b/tools/ci/require-packed-cli.test.mjs @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { readFileSync } from 'node:fs'; +import { readFileSync, readdirSync } from 'node:fs'; import { createRequire } from 'node:module'; import { test } from 'node:test'; @@ -126,7 +126,7 @@ function checkAggregate(job, packId, consumerId) { } function checkUnprivilegedPackageJobs(workflow) { - for (const id of ['pack-cli', 'packed-cli-consumer', 'packed-cli']) { + for (const id of ['pack-cli', 'packed-cli-consumer', 'packed-cli', 'map-controls']) { const job = workflow.jobs[id]; for (const key of ['TURBO_TOKEN', 'TURBO_TEAM']) { assert.equal(job.env?.[key], '', `${id} must clear inherited ${key}`); @@ -141,7 +141,7 @@ test('all PR-controlled installed-package jobs clear inherited remote-cache cred checkUnprivilegedPackageJobs(readWorkflow('ci')); }); -for (const id of ['pack-cli', 'packed-cli-consumer', 'packed-cli']) +for (const id of ['pack-cli', 'packed-cli-consumer', 'packed-cli', 'map-controls']) for (const key of ['TURBO_TOKEN', 'TURBO_TEAM']) for (const value of [undefined, '${{ secrets.REMOTE_CACHE_CREDENTIAL }}']) test(`package privilege guard refuses ${id} ${key}: ${String(value)}`, () => { @@ -227,6 +227,165 @@ for (const [name, packId, consumerId] of [ }); } +const mapControlsCommand = 'pnpm check:map-controls'; +const mapControlsGuard = 'Require complete product map controls'; + +/** The product map controls job: uncached, unprivileged, no upload, joined to required CI only. */ +function checkMapControls(workflow) { + const job = workflow.jobs['map-controls']; + requiredJob(job); + // Every step runs in the default shell with only the job's own environment. + assert.equal(workflow.defaults, undefined); + assert.equal(job.defaults, undefined); + assert.equal(workflow.jobs.ci.defaults, undefined); + assert.deepEqual(job.env, { TURBO_TOKEN: '', TURBO_TEAM: '' }); + assert.equal(job.needs, undefined); + assert.equal(job.strategy, undefined); + assert.equal(job['runs-on'], 'ubuntu-latest'); + // Checkout, pnpm and Node only: no cache, upload or download action. + assert.deepEqual( + job.steps.filter((step) => step.uses).map((step) => step.uses), + [actions.checkout, actions.pnpm, actions.node], + ); + assert.deepEqual(job.steps.find((step) => step.uses === actions.checkout).with, { + 'persist-credentials': false, + }); + assert.equal(job.steps.find((step) => step.uses === actions.pnpm).with, undefined); + assert.deepEqual(job.steps.find((step) => step.uses === actions.node).with, { + 'node-version-file': '.nvmrc', + }); + assert.deepEqual( + job.steps.filter((step) => step.run).map((step) => step.run), + ['pnpm install --frozen-lockfile', mapControlsCommand], + ); + const controls = job.steps.find((step) => step.run === mapControlsCommand); + assert.deepEqual(Object.keys(controls).sort(), ['env', 'name', 'run']); + assert.deepEqual(controls.env, { RELAVIUM_PACKAGE_SOURCE_SHA: '${{ github.sha }}' }); + // Required CI waits for it and refuses every result except success; the package aggregate does not. + const ci = workflow.jobs.ci; + assert.ok(ci.needs.includes('map-controls')); + assert.ok(![workflow.jobs['packed-cli'].needs].flat().includes('map-controls')); + const guard = ci.steps.find((step) => step.name === mapControlsGuard); + assert.ok(guard, mapControlsGuard); + assert.deepEqual(Object.keys(guard).sort(), ['env', 'name', 'run']); + assert.deepEqual(guard.env, { MAP_CONTROLS_RESULT: '${{ needs.map-controls.result }}' }); + assert.equal( + guard.run, + "node -e \"require('node:assert/strict').equal(process.env.MAP_CONTROLS_RESULT, 'success')\"", + ); + assert.equal(guard.if, undefined); + assert.equal(guard['continue-on-error'], undefined); +} + +test('required CI runs the product map controls uncached and without uploads; release does not', () => { + checkMapControls(readWorkflow('ci')); + // The release workflow's graph belongs to Step 9d (the 2026-10-24 ruling). + assert.equal(readWorkflow('release').jobs['map-controls'], undefined); +}); + +for (const [name, mutate] of [ + [ + 'a pnpm cache', + (job) => (job.steps.find((step) => step.uses === actions.node).with.cache = 'pnpm'), + ], + [ + 'a cache action', + (job) => + job.steps.splice(3, 0, { + uses: 'actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830', + with: { path: '.turbo', key: 'turbo' }, + }), + ], + [ + 'an upload', + (job) => job.steps.push({ uses: actions.upload, with: { name: 'maps', path: 'x' } }), + ], + ['credentials kept in the checkout', (job) => (job.steps[0].with['persist-credentials'] = true)], + ['a build before the controls', (job) => job.steps.splice(4, 0, { run: 'pnpm turbo run build' })], + ['a tolerated failure', (job) => (job['continue-on-error'] = true)], + ['a disabled job', (job) => (job.if = '${{ false }}')], + ['a longer deadline', (job) => (job['timeout-minutes'] = 30)], + [ + 'no source identity', + (job) => delete job.steps.find((step) => step.run === mapControlsCommand).env, + ], + [ + 'a tolerated controls step', + (job) => + (job.steps.find((step) => step.run === mapControlsCommand)['continue-on-error'] = true), + ], + [ + 'a custom shell on the controls step', + (job) => (job.steps.find((step) => step.run === mapControlsCommand).shell = 'true {0}'), + ], + ['a default shell for the job', (job) => (job.defaults = { run: { shell: 'true {0}' } })], + ['a preload in the job environment', (job) => (job.env.NODE_OPTIONS = '--import ./x.mjs')], + ['write permissions', (job) => (job.permissions = { contents: 'write' })], + ['another platform', (job) => (job['runs-on'] = 'macos-latest')], + ['a dependency', (job) => (job.needs = 'pack-cli')], + ['a matrix', (job) => (job.strategy = { matrix: { os: ['ubuntu-latest'] } })], + [ + 'pnpm setup inputs', + (job) => (job.steps.find((step) => step.uses === actions.pnpm).with = { version: '9.12.3' }), + ], +]) + test(`map controls job refuses ${name}`, () => { + const workflow = readWorkflow('ci'); + mutate(workflow.jobs['map-controls']); + assert.throws(() => checkMapControls(workflow), assert.AssertionError); + }); + +for (const [name, mutate] of [ + [ + 'removal from required CI', + (jobs) => (jobs.ci.needs = jobs.ci.needs.filter((id) => id !== 'map-controls')), + ], + [ + 'joining the package aggregate', + (jobs) => (jobs['packed-cli'].needs = [...jobs['packed-cli'].needs, 'map-controls']), + ], + [ + 'a removed guard', + (jobs) => (jobs.ci.steps = jobs.ci.steps.filter((step) => step.name !== mapControlsGuard)), + ], + [ + 'a disabled guard', + (jobs) => (jobs.ci.steps.find((step) => step.name === mapControlsGuard).if = '${{ false }}'), + ], + [ + 'a guard that admits other results', + (jobs) => (jobs.ci.steps.find((step) => step.name === mapControlsGuard).run = 'true'), + ], + [ + 'a guard reading another result', + (jobs) => + (jobs.ci.steps.find((step) => step.name === mapControlsGuard).env = { + MAP_CONTROLS_RESULT: '${{ needs.packed-cli.result }}', + }), + ], + [ + 'a tolerated guard', + (jobs) => + (jobs.ci.steps.find((step) => step.name === mapControlsGuard)['continue-on-error'] = true), + ], + [ + 'a custom shell on the guard', + (jobs) => (jobs.ci.steps.find((step) => step.name === mapControlsGuard).shell = 'true {0}'), + ], + ['a default shell for ci', (jobs) => (jobs.ci.defaults = { run: { shell: 'true {0}' } })], +]) + test(`required CI refuses ${name} of the map controls`, () => { + const workflow = readWorkflow('ci'); + mutate(workflow.jobs); + assert.throws(() => checkMapControls(workflow), assert.AssertionError); + }); + +test('a workflow-wide default shell refuses the map controls', () => { + const workflow = readWorkflow('ci'); + workflow.defaults = { run: { shell: 'true {0}' } }; + assert.throws(() => checkMapControls(workflow), assert.AssertionError); +}); + test('required CI joins packed acceptance and explicitly refuses every result except success', () => { const job = readWorkflow('ci').jobs.ci; assert.ok(job.needs.includes('packed-cli')); @@ -268,8 +427,51 @@ test('release keeps main ancestry, version, real-push publication and npm proven ); assert.deepEqual( jobs.publish.steps.filter((step) => step.run).map((step) => step.run), - ['npm publish ./artifact/*.tgz --provenance --access public'], + [ + 'node tools/cli-package/release-guard.mjs ./artifact', + 'npm publish ./artifact/*.tgz --provenance --access public', + ], + ); +}); + +test('release publication is refused fail-closed until a verified release candidate exists', () => { + const workflow = readWorkflow('release'); + const steps = workflow.jobs.publish.steps; + const checkout = steps.find((step) => step.uses === actions.checkout); + assert.deepEqual(checkout.with, { + 'persist-credentials': false, + 'sparse-checkout': + '.nvmrc\ntools/cli-package/release-guard.mjs\ntools/cli-package/release-guard-core.mjs\n', + 'sparse-checkout-cone-mode': false, + }); + const guard = steps.find( + (step) => step.name === 'Refuse anything but a verified release candidate', ); + const publish = steps.find((step) => step.name === 'Publish'); + assert.equal(guard.run, 'node tools/cli-package/release-guard.mjs ./artifact'); + // An unconditional, non-advisory guard runs after the download and before any publish, under the + // runner's default shell and directory: no condition, shell, environment or defaults at the step, + // job or workflow level, and no earlier `run` step that could write the job's environment. + assert.deepEqual(Object.keys(guard).sort(), ['name', 'run']); + // The deployment environment scopes the npm token and its reviewer; it writes no job environment. + assert.equal(workflow.jobs.publish.environment, 'release'); + assert.deepEqual(Object.keys(workflow.jobs.publish).sort(), [ + 'environment', + 'if', + 'name', + 'needs', + 'permissions', + 'runs-on', + 'steps', + 'timeout-minutes', + ]); + assert.deepEqual(Object.keys(workflow).sort(), ['concurrency', 'jobs', 'name', 'on']); + assert.ok(steps.slice(0, steps.indexOf(guard)).every((step) => step.run === undefined)); + assert.ok( + steps.indexOf(steps.find((step) => step.uses === actions.download)) < steps.indexOf(guard), + ); + assert.equal(steps.indexOf(guard) + 1, steps.indexOf(publish)); + assert.equal(publish.if, undefined); }); for (const [label, alter] of [ @@ -400,3 +602,20 @@ for (const command of ['pnpm smoke:cli-bootstrap', 'pnpm smoke:budget-replay']) }); } } + +test('every CLI package, closure, bootstrap and CI tooling test runs in test:acceptance-tools', () => { + // The list is the only place CI runs these files, so an unlisted test would silently never run. + const manifest = JSON.parse(readFileSync(new URL('package.json', root), 'utf8')); + const listed = new Set(manifest.scripts['test:acceptance-tools'].split(' ')); + for (const directory of [ + 'tools/cli-package', + 'tools/cli-package/smoke', + 'tools/bundle-closure', + 'tools/cli-bootstrap', + 'tools/ci', + ]) { + for (const name of readdirSync(new URL(directory, root))) { + if (name.endsWith('.test.mjs')) assert.ok(listed.has(`${directory}/${name}`), name); + } + } +}); diff --git a/tools/ci/require-w8-acceptance.test.mjs b/tools/ci/require-w8-acceptance.test.mjs index e2bbd452..670a4a4c 100644 --- a/tools/ci/require-w8-acceptance.test.mjs +++ b/tools/ci/require-w8-acceptance.test.mjs @@ -42,6 +42,7 @@ test('the actual required workflow joins both full offline controls and checks t 'fs-mutations', 'secrets-mutations', 'packed-cli', + 'map-controls', ]); const packedGuard = ci.steps.find((step) => step.name === 'Require complete packed CLI controls'); assert.equal(packedGuard.env.PACKED_CLI_RESULT, '${{ needs.packed-cli.result }}'); @@ -89,37 +90,36 @@ test('the actual required workflow joins both full offline controls and checks t ); assert.equal(secretsGuard.if, undefined); assert.equal(secretsGuard['continue-on-error'], undefined); - for (const kind of ['secrets']) { + // Both families run as two legs that each run and export a disjoint half under the unchanged + // 64-MiB retained-evidence bound, which one export of either whole cohort no longer fits. + for (const [kind, directory, minutes] of [ + ['fs', 'filesystem', 60], + ['secrets', 'secrets', 75], + ]) { const job = workflow.jobs[`${kind}-mutations`]; assert.equal(job['continue-on-error'], undefined); assert.equal(job.if, undefined); - assert.equal(job['timeout-minutes'], 30); + assert.equal(job['timeout-minutes'], minutes); + // Two legs, each running and exporting a disjoint half; neither may stop the other early. + assert.deepEqual(job.strategy, { 'fail-fast': false, matrix: { shard: [1, 2] } }); assert.ok(job.steps.some((step) => step.run === 'pnpm install --frozen-lockfile')); assert.ok(job.steps.some((step) => step.run === 'pnpm test:acceptance-tools')); assert.ok(job.steps.some((step) => step.with?.['node-version-file'] === '.nvmrc')); - const command = job.steps.find((step) => step.run === `pnpm check:${kind}-mutations`); + const command = job.steps.find( + (step) => step.run === `pnpm check:${kind}-mutations --shard \${{ matrix.shard }}/2`, + ); assert.equal(command['continue-on-error'], undefined); assert.equal(command.if, undefined); assert.equal(command.env.RELAVIUM_ACCEPTANCE_EVIDENCE, '${{ runner.temp }}/wave2-acceptance'); const artifact = job.steps.find((step) => step.uses?.startsWith('actions/upload-artifact@')); assert.equal(artifact.if, '${{ always() }}'); - assert.equal(artifact.with.path, '${{ runner.temp }}/wave2-acceptance/secrets'); + assert.equal(artifact.with.path, `\${{ runner.temp }}/wave2-acceptance/${directory}`); + assert.ok( + artifact.with.name.startsWith(`wave2-${directory}-\${{ matrix.shard }}-of-2-`), + artifact.with.name, + ); assert.equal(artifact.with['retention-days'], 7); } - const filesystem = workflow.jobs['fs-mutations']; - assert.equal(filesystem['continue-on-error'], undefined); - assert.equal(filesystem.if, undefined); - assert.equal(filesystem['timeout-minutes'], 60); - assert.ok(filesystem.steps.some((step) => step.run === 'pnpm install --frozen-lockfile')); - const fsCommand = filesystem.steps.find((step) => step.run === 'pnpm check:fs-mutations'); - assert.equal(fsCommand['continue-on-error'], undefined); - assert.equal(fsCommand.env.RELAVIUM_ACCEPTANCE_EVIDENCE, '${{ runner.temp }}/wave2-acceptance'); - const fsArtifact = filesystem.steps.find((step) => - step.uses?.startsWith('actions/upload-artifact@'), - ); - assert.equal(fsArtifact.if, '${{ always() }}'); - assert.equal(fsArtifact.with.path, '${{ runner.temp }}/wave2-acceptance/filesystem'); - assert.equal(fsArtifact.with['retention-days'], 7); const packageJson = JSON.parse( readFileSync(new URL('../../package.json', import.meta.url), 'utf8'), ); @@ -133,4 +133,16 @@ test('the actual required workflow joins both full offline controls and checks t ); for (const kind of ['wire', 'usage', 'fs', 'secrets']) assert.ok(packageJson.scripts.ci.split(' && ').includes(`pnpm check:${kind}-mutations`)); + // The product map controls run locally right after the producer and consumer smoke. + assert.equal( + packageJson.scripts['check:map-controls'], + 'node tools/cli-package/map-controls.mjs', + ); + const chain = packageJson.scripts.ci.split(' && '); + assert.ok(chain.indexOf('pnpm smoke:cli-package') >= 0); + assert.equal( + chain.indexOf('pnpm check:map-controls'), + chain.indexOf('pnpm smoke:cli-package') + 1, + ); + assert.equal(chain.filter((step) => step === 'pnpm check:map-controls').length, 1); }); diff --git a/tools/ci/workspace-scheduling.test.mjs b/tools/ci/workspace-scheduling.test.mjs index 071ab632..94ce27a2 100644 --- a/tools/ci/workspace-scheduling.test.mjs +++ b/tools/ci/workspace-scheduling.test.mjs @@ -186,7 +186,7 @@ test('required and local CI share all workspace tasks and uncached per-invocatio } const floor = workflow.jobs['floor-check']; assert.equal(floor['timeout-minutes'], 25); - assert.equal(workflow.jobs.coverage['timeout-minutes'], 30); + assert.equal(workflow.jobs.coverage['timeout-minutes'], 45); assert.equal( workflow.jobs.coverage.steps.find((step) => step.run === 'pnpm coverage:enforced')?.name, 'Engine coverage floor (>=90% line+branch — llm, mcp enforced; core measured)', diff --git a/tools/cli-bootstrap/child.mjs b/tools/cli-bootstrap/child.mjs index 66cb2703..ce044258 100644 --- a/tools/cli-bootstrap/child.mjs +++ b/tools/cli-bootstrap/child.mjs @@ -13,7 +13,7 @@ import { tmpdir } from 'node:os'; import { join, relative } from 'node:path'; import { pathToFileURL } from 'node:url'; import { createHash } from 'node:crypto'; -import { spawnBounded } from './process.mjs'; +import { spawnBounded, stopBoundedChildren } from './process.mjs'; import { maxWitnessBytes } from './policy.mjs'; function snapshot(root) { @@ -34,9 +34,33 @@ function snapshot(root) { return entries; } +/** Every fixture root a running control still owns. */ +const liveFixtures = new Set(); + +/** + * For a caller that an interrupt ends before `withFixture` can remove its own root: synchronously kill + * every bounded child, then remove every fixture root still in use. One root that cannot be removed + * neither keeps the children alive nor the other roots in place; the first failure is rethrown after. + */ +export function abandonFixtures() { + stopBoundedChildren(); + let failure; + for (const root of liveFixtures) { + try { + // Retries absorb a killed child that was still finishing a write. + rmSync(root, { recursive: true, force: true, maxRetries: 3 }); + } catch (error) { + failure ??= error; + } + } + liveFixtures.clear(); + if (failure !== undefined) throw failure; +} + /** Each child receives a synthetic HOME/cwd, no ambient credentials or Node preload. */ export async function withFixture(run) { const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-bootstrap-'))); + liveFixtures.add(root); try { const home = join(root, 'home'), workspace = join(root, 'workspace'), @@ -118,6 +142,7 @@ export async function withFixture(run) { return result; } finally { // This exact mkdtemp-owned directory is the sole cleanup target, including failing controls. + liveFixtures.delete(root); rmSync(root, { recursive: true, force: true }); } } diff --git a/tools/cli-bootstrap/process.mjs b/tools/cli-bootstrap/process.mjs index 4e6e23fa..23c7d0cf 100644 --- a/tools/cli-bootstrap/process.mjs +++ b/tools/cli-bootstrap/process.mjs @@ -3,6 +3,28 @@ import { Buffer } from 'node:buffer'; import { setTimeout, clearTimeout } from 'node:timers'; export const maxOutputBytes = 256 * 1024; + +/** Every bounded child still running, so that an interrupted caller can stop them before it exits. */ +const liveChildren = new Set(); + +/** + * Synchronously kill every bounded child that has not closed yet: its process group, or the child + * itself where it has none. For a signal handler that exits without waiting for `spawnBounded`. + */ +export function stopBoundedChildren() { + for (const { child, grouped } of liveChildren) { + if (grouped) { + try { + process.kill(-child.pid, 'SIGKILL'); + continue; + } catch { + // The group already exited. + } + } + child.kill('SIGKILL'); + } + liveChildren.clear(); +} /** Bound bytes and wall time, kill only this owned process group, and wait for close. */ export function spawnBounded( args, @@ -16,6 +38,8 @@ export function spawnBounded( detached: grouped, stdio: ['ignore', 'pipe', 'pipe'], }); + const live = { child, grouped }; + if (child.pid !== undefined) liveChildren.add(live); const output = { stdout: [], stderr: [] }; let bytes = 0, timedOut = false, @@ -48,6 +72,7 @@ export function spawnBounded( spawnError = error.message; }); child.once('close', (code, signal) => { + liveChildren.delete(live); clearTimeout(timer); kill(); resolve({ diff --git a/tools/cli-bootstrap/witness.test.mjs b/tools/cli-bootstrap/witness.test.mjs index e859489e..2de0eeb1 100644 --- a/tools/cli-bootstrap/witness.test.mjs +++ b/tools/cli-bootstrap/witness.test.mjs @@ -1,12 +1,26 @@ import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; import { test } from 'node:test'; -import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { setTimeout as delay } from 'node:timers/promises'; +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { join, relative } from 'node:path'; import { pathToFileURL } from 'node:url'; +import { exitWithin } from '../cli-package/child-exit.mjs'; import { withFixture, toEntryUrl } from './child.mjs'; import { assertWitness, importFailureCanary, servicePackages } from './policy.mjs'; -import { spawnBounded } from './process.mjs'; +import { spawnBounded, stopBoundedChildren } from './process.mjs'; import { BOOTSTRAP_CONTROL_COUNT, controlName, grammarCases, metaCases } from './inventory.mjs'; import ts from 'typescript'; @@ -400,3 +414,182 @@ test('output cap counts UTF-8 bytes and joins the killed child', async () => { assert.equal(result.joined, true); assert.equal(result.signal, 'SIGKILL'); }); +test( + 'an interrupted caller abandons a running control: its bounded process group is killed, its fixture removed', + { skip: process.platform === 'win32' }, + async (t) => { + const temporary = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-bootstrap-abandon-'))); + t.after(() => rmSync(temporary, { recursive: true, force: true })); + const pidFiles = ['child.pid', 'grandchild.pid'].map((name) => join(temporary, name)); + const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); + // The bounded child and a grandchild in its process group keep writing into the fixture, as a + // running control's processes do, and outlive the fixture's removal unless they are killed. + const writer = (pidFile) => + `const { writeFileSync } = require('node:fs');` + + `const { join } = require('node:path');` + + `writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + `let n = 0; setInterval(() => { try { writeFileSync(join(process.env.TMPDIR, 'f' + n++), 'x'); } catch {} }, 5);`; + const bounded = + `require('node:child_process').spawn(process.execPath, ['-e', ${JSON.stringify( + writer(pidFiles[1]), + )}], { stdio: 'ignore' });` + writer(pidFiles[0]); + const script = ` +import { abandonFixtures, withFixture } from ${url('./child.mjs')}; +import { spawnBounded } from ${url('./process.mjs')}; +process.on('SIGINT', () => { + abandonFixtures(); + process.exit(130); +}); +await withFixture((fixture) => + spawnBounded(['-e', ${JSON.stringify(bounded)}], { env: { TMPDIR: fixture.temporary } }, { + timeoutMs: 60_000, + }), +); +`; + const caller = spawn(process.execPath, ['--input-type=module', '-e', script], { + env: { PATH: process.env.PATH, TMPDIR: temporary }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const alive = (pid) => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }; + const tree = []; + try { + for (const file of pidFiles) { + for (let waited = 0; !existsSync(file) || readFileSync(file, 'utf8') === ''; waited += 25) { + assert.ok(waited < 20_000, 'the bounded control never started'); + await delay(25); + } + tree.push(Number(readFileSync(file, 'utf8'))); + } + assert.equal( + readdirSync(temporary).filter((name) => name.startsWith('relavium-bootstrap-')).length, + 1, + ); + const exited = exitWithin(caller); + caller.kill('SIGINT'); + assert.deepEqual(await exited, [130, null]); + // The whole process group goes, not only the process spawnBounded started. + for (let waited = 0; tree.some(alive); waited += 25) { + assert.ok(waited < 5_000, 'part of the bounded control outlived its abandonment'); + await delay(25); + } + assert.deepEqual( + readdirSync(temporary).filter((name) => name.startsWith('relavium-bootstrap-')), + [], + ); + } finally { + if (caller.exitCode === null && caller.signalCode === null) caller.kill('SIGKILL'); + for (const pid of tree) if (alive(pid)) process.kill(pid, 'SIGKILL'); + } + }, +); +test( + 'abandoning kills a group that ignores SIGTERM and removes the other fixtures when one removal fails', + { skip: process.platform === 'win32' || process.getuid?.() === 0 }, + async (t) => { + const temporary = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-bootstrap-locked-'))); + const pidFiles = ['locked.pid', 'grandchild.pid', 'plain.pid'].map((name) => + join(temporary, name), + ); + const roots = () => + readdirSync(temporary).filter((name) => name.startsWith('relavium-bootstrap-')); + const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); + const idle = (pidFile) => + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + `process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);`; + // The first control's child makes its fixture unremovable (a file in a read-only directory) + // and, like its grandchild, ignores SIGTERM; the second control's fixture is ordinary. + const locked = + `const fs = require('node:fs'); const path = require('node:path');` + + `const directory = path.join(process.env.TMPDIR, 'locked'); fs.mkdirSync(directory);` + + `fs.writeFileSync(path.join(directory, 'file'), 'x'); fs.chmodSync(directory, 0o500);` + + `require('node:child_process').spawn(process.execPath, ['-e', ${JSON.stringify( + idle(pidFiles[1]), + )}], { stdio: 'ignore' });` + + idle(pidFiles[0]); + const script = ` +import { writeSync } from 'node:fs'; +import { abandonFixtures, withFixture } from ${url('./child.mjs')}; +import { spawnBounded } from ${url('./process.mjs')}; +process.on('SIGINT', () => { + try { + abandonFixtures(); + } catch { + writeSync(2, 'ABANDON_FAILED\\n'); + } + process.exit(130); +}); +const control = (code) => (fixture) => + spawnBounded(['-e', code], { env: { TMPDIR: fixture.temporary } }, { timeoutMs: 60_000 }); +await Promise.all([ + withFixture(control(${JSON.stringify(locked)})), + withFixture(control(${JSON.stringify(idle(pidFiles[2]))})), +]); +`; + const caller = spawn(process.execPath, ['--input-type=module', '-e', script], { + env: { PATH: process.env.PATH, TMPDIR: temporary }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stderr = ''; + caller.stderr.on('data', (chunk) => (stderr += chunk)); + const alive = (pid) => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }; + const tree = []; + t.after(() => { + if (caller.exitCode === null && caller.signalCode === null) caller.kill('SIGKILL'); + for (const pid of tree) if (alive(pid)) process.kill(pid, 'SIGKILL'); + for (const name of roots()) { + const directory = join(temporary, name, 'temporary', 'locked'); + if (existsSync(directory)) chmodSync(directory, 0o700); + } + rmSync(temporary, { recursive: true, force: true }); + }); + for (const file of pidFiles) { + for (let waited = 0; !existsSync(file) || readFileSync(file, 'utf8') === ''; waited += 25) { + assert.ok(waited < 20_000, 'a bounded control never started'); + await delay(25); + } + tree.push(Number(readFileSync(file, 'utf8'))); + } + assert.equal(roots().length, 2); + const exited = exitWithin(caller); + caller.kill('SIGINT'); + assert.deepEqual(await exited, [130, null]); + for (let waited = 0; tree.some(alive); waited += 25) { + assert.ok(waited < 5_000, 'a bounded control outlived an abandonment whose removal failed'); + await delay(25); + } + // Only the unremovable fixture is left, and the failure was reported, not swallowed. + const left = roots(); + assert.equal(left.length, 1); + assert.equal(existsSync(join(temporary, left[0], 'temporary', 'locked')), true); + assert.equal(stderr, 'ABANDON_FAILED\n'); + }, +); +test( + 'a closed bounded child leaves nothing for an interrupted caller to kill', + { skip: process.platform === 'win32' }, + async (t) => { + // A closed child's process group may be recycled; abandoning must never signal it. + assert.equal((await spawnBounded(['-e', ''], {})).code, 0); + assert.equal( + (await spawnBounded(['-e', 'setInterval(() => {}, 1000)'], {}, { timeoutMs: 100 })).timedOut, + true, + ); + const kill = t.mock.method(process, 'kill', () => true); + stopBoundedChildren(); + assert.equal(kill.mock.callCount(), 0); + }, +); diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index b6520200..db2695b4 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -6,17 +6,30 @@ The canonical package and release contract lives in [packaging.md](../../docs/reference/cli/packaging.md). Step 8b adds required creation-owned native process controls to the same installed artifact cohort. Step 8d adds installed global fatal/background/output-hook -controls; complete Step 8d acceptance remains separate. +controls; complete Step 8d acceptance remains separate. Step 9a.1 makes the producer +build an owned map-bearing candidate and finalize it before packing. ```sh pnpm package:cli:pack /absolute/empty/artifact-directory pnpm package:cli:consume /absolute/artifact-directory /absolute/fresh/report-directory pnpm smoke:cli-package +pnpm check:map-controls ``` -The producer emits the five inlined dependency packages in order with the installed -compiler and no build cache. Prepack performs the sole final CLI build. The runner -observes one actual prepack build-start record, checks the emitted graph, and +The producer (`produce.mjs`, run by `pack.mjs`) never builds in the developer's workspace. +It copies the files Git would ship, by the repository's ignore rules and the developer's +global excludes file (its Git reads no other home, global or system configuration or attributes), into a +fresh owner-only candidate tree (owner-only where POSIX modes are observable; on Windows it +inherits the temporary directory's ACL, which is not verified), refusing if the original checkout's commit, status or set of +files Git is told not to check changes during the copy (such a skip-worktree or assume-unchanged +file makes the source `dirty`), and installs the locked closure there offline, frozen and +without lifecycle scripts from the existing store. In that tree it emits the five +inlined dependency packages in order with the installed compiler, upstream maps and no +build cache. Prepack (`prepack.mjs`) performs the sole final CLI build in candidate +mode and runs the finalizer (`finalize.mjs`), which validates every consumed upstream map and every +final map, removes only the exact map-link trailers and stages maps, metafile and its hash +record in a private directory outside the artifact. The runner observes one actual +prepack build-start record and one finalization, checks the emitted graph, and independently inspects the produced gzip/USTAR bytes. The inspector never extracts the archive. Every JS and migration byte must match the finalized build and canonical migrations; package metadata must match pnpm's publication transform. @@ -27,10 +40,61 @@ An exact public allowlist, checksums, modes, hashbangs, byte ceilings and negati controls exclude links, traversal, duplicates, extra files, maps and private build inventories. This deliberately supports the pinned packer's dialect, not general tar. -The resulting public `package-test-receipt.json` records `compiled_package_test`. -It does **not** certify original-TypeScript maps, symbolication, encrypted custody -or a production release candidate. Those are Step 9 obligations. No private map, -metafile or per-source inventory enters the public artifact. +The resulting public `package-test-receipt.json` records `package_test` (Step 9a.2b; it recorded +`compiled_package_test` before the product map controls existed). +The candidate's maps are built, validated and then removed with the rest of the +private staging, also when the producer is interrupted (SIGINT, SIGTERM, SIGHUP), which +stops the build first and empties the artifact directory, even after the last build child +exited. An interrupted consumer stops its children, the bootstrap witness's included, and +removes its installation, the witness's fixtures and the installed smoke's working directory; the +installed CLI's detached native fixture targets end once the smoke worker that owns them is gone. +On POSIX `pnpm smoke:cli-package` forwards an interrupt to the producer or consumer it started +instead of ending before them, and it interrupts a stage past its deadline before it kills the +stage's process group, so the stage stops its own children; on Windows the stages receive Ctrl-C +from the shared console and a stage past its deadline is killed with its tree at once (no CI job +exercises these Windows paths). It starts no later stage after an interrupt. SIGHUP aborts and +cleans up even under `nohup`; use `setsid` or a terminal multiplexer to detach a long run. A +producer killed outright can leave its owner-only root in the temporary directory. The local +consumer installs under the temporary directory, so a shared `/tmp` is a named limit +(packaging.md). A failed +production prints only prepack's own fixed `✗ CLI …` lines and one fixed code +(`production-failure.mjs`), never a build child's captured output, which can quote a map. A +corepack-managed pnpm always runs from the developer's own corepack cache, never from the shared +temporary directory that holds the isolated homes (`corepackHome` in `process.mjs`). The finalizer +decodes every private map strictly (`source-map.mjs`). Before the receipt is written, the seal seals +the private manifest (`manifest.mjs`) and symbolicates actual packed CLI and core frames, captured +by a tooling-only preload while the CLI imports a canary into a private project +(`frame-capture.mjs`, which refuses any module that resolves outside the candidate, the keyring and +native addons in its own process, and needs Node 22.15 or later; an older producer Node refuses with +`PACKED_CLI_PRODUCER_NODE` before building), through the strictly decoded maps of the chunks those frames land in against +the matching source revision (`symbolicate.mjs`). The manifest is discarded with the staging, and +the capture's home, project and frames with the private root. The +receipt does **not** certify encrypted custody or a production release candidate (Steps 9b–9d), +and its `package_test` mode is a claim, never evidence (packaging.md says when a `ci` run backs it). An +artifact directory with a receipt, or a staging with a manifest, is never sealed again +(`PACKED_CLI_ALREADY_SEALED`), and `dist` and the tarball must still be the manifest's when the receipt +is written (`verifySealedDist`). `pnpm check:map-controls` (`map-controls.mjs` over +`map-controls-core.mjs`) runs every named negative control on product copies of one real production, +prints one JSON summary of codes and fixed lines (no absolute path, frame or map byte) and exits 1 +unless every expected control ran once with its expected outcome; the required `map-controls` CI job +runs it uncached and uploads nothing. Step 9b.1's custody library (nothing outside Step 9b.2's release +modules and the tests calls it) builds one owner-only USTAR of the sealed staging (`container.mjs`, over the one strict +dialect in `ustar.mjs`) and encrypts it once to one pinned recipient with GnuPG ≥ 2.2.8 in fresh +isolated homes (`custody.mjs` over `gnupg.mjs`; POSIX only, since no private root qualifies on +Windows): a colon pre-check refuses missing, expired, revoked, unusable or unexpected recipients and the +secret material it can detect, the ciphertext must be SEIPDv1 with MDC and +AES-256 naming exactly the pinned subkey, an existing ciphertext is never overwritten, the plaintext +work directory is removed on every path and every refusal is a fixed `PACKED_CLI_CUSTODY_*` code. +`custody-fixtures/` holds the committed synthetic public keys; no secret key is committed. Step 9b.2 adds, again +with nothing calling it yet, the release binding receipt v1 with its validator and verified reuse +(`release-binding.mjs`, whose import closure starts no process), the scope gate and the sealer that writes the +receipt once after re-reading the ciphertext (`release-custody.mjs`), and the production recipient loader +(`release-recipient.mjs`), which refuses in Wave 3 because no recipient is committed. No private map, +metafile, manifest or per-source inventory enters the public artifact. The release workflow's +`release-guard.mjs` refuses every artifact until verified release custody exists: it +sets its exit code to 1 before it evaluates its rule (`release-guard-core.mjs`), so a rule +that returns, throws or cannot load still refuses, and no way of invoking it can skip the +refusal. The consumer verifies the chosen bytes before installation, installs a copy outside the workspace, and compares every installed public file with the archive receipt. diff --git a/tools/cli-package/archive.mjs b/tools/cli-package/archive.mjs index 6b2afa0c..b2e89ca5 100644 --- a/tools/cli-package/archive.mjs +++ b/tools/cli-package/archive.mjs @@ -2,6 +2,7 @@ import { Buffer } from 'node:buffer'; import { createHash } from 'node:crypto'; import { isDeepStrictEqual, TextDecoder } from 'node:util'; import { gunzipSync, inflateRawSync } from 'node:zlib'; +import { readUstar } from './ustar.mjs'; // This inspects the pinned pnpm pack dialect, not arbitrary tar archives. There // is no extraction and no fallback to a more permissive platform tar executable. @@ -23,9 +24,9 @@ export class PackageArchiveError extends Error { } } -/** @param {string} code @returns {never} */ -function refuse(code) { - throw new PackageArchiveError(code); +/** @param {string} code @param {unknown} [cause] @returns {never} */ +function refuse(code, cause) { + throw new PackageArchiveError(code, cause === undefined ? undefined : { cause }); } /** @param {unknown} value @returns {value is Record} */ @@ -78,23 +79,33 @@ function unpack(bytes) { } } -/** @param {Buffer} field */ -function octal(field) { - const text = field.toString('latin1'); - if (!/^[0-7]+[ \0]*$/u.test(text)) refuse('TAR_NUMBER'); - const value = Number.parseInt(text, 8); - if (!Number.isSafeInteger(value)) refuse('TAR_NUMBER'); - return value; -} +/** @param {string} path */ +/** + * A runtime dependency requirement the public manifest may carry: an exact, caret or tilde registry + * version, the form pnpm publishes for a `catalog:` entry. An allowlist rather than a list of refused + * protocols, so `npm:` aliases, `github:`, git and tarball URLs, `file:`/`link:`/`workspace:`/`catalog:` + * specifiers, `*` and open ranges never reach a consumer's install (a PR #96 review finding). + */ +const REGISTRY_REQUIREMENT = /^[~^]?\d{1,9}\.\d{1,9}\.\d{1,9}(?:-[0-9A-Za-z.-]{1,64})?$/u; -/** @param {Buffer} field */ -function nameField(field) { - const end = field.indexOf(0); - if (end < 1 || field.subarray(end).some((byte) => byte !== 0)) refuse('TAR_NAME'); - return utf8(field.subarray(0, end)); +/** + * Whether one public runtime dependency entry is admitted: an npm package name outside the private + * `@relavium/` scope, required at a registry version. Shared by the archive inspector and the package + * receipt binding so the two cannot drift. + * @param {string} name + * @param {unknown} requirement + * @returns {boolean} + */ +export function isPublicRuntimeDependency(name, requirement) { + return ( + /^(?:@[a-z0-9_.-]+\/)?[a-z0-9_.-]+$/u.test(name) && + name.length <= 200 && + !name.startsWith('@relavium/') && + typeof requirement === 'string' && + REGISTRY_REQUIREMENT.test(requirement) + ); } -/** @param {string} path */ export function assertPublicPackagePath(path) { if ( !/^package\/(?:dist\/[A-Za-z0-9_-]+\.js|drizzle\/[0-9]{4}_[a-z0-9_]+\.sql|drizzle\/meta\/(?:[0-9]{4}_snapshot|_journal)\.json|package\.json|README\.md|CHANGELOG\.md|LICENSE)$/u.test( @@ -106,45 +117,6 @@ export function assertPublicPackagePath(path) { if (/\/(?:con|prn|aux|nul|com[1-9]|lpt[1-9])\.js$/iu.test(path)) refuse('PUBLIC_PATH'); } -/** @param {Buffer} header */ -function readHeader(header) { - const checksum = octal(header.subarray(148, 156)); - let actual = 0; - for (let index = 0; index < 512; index += 1) { - actual += index >= 148 && index < 156 ? 32 : (header[index] ?? 0); - } - if (checksum !== actual) refuse('TAR_CHECKSUM'); - if (!header.subarray(257, 265).equals(Buffer.from('ustar\0' + '00'))) refuse('TAR_FORMAT'); - if (header[156] !== 48) refuse('TAR_TYPE'); - // No linkname, owner strings, prefix, extension records or reserved bytes. - // A regular USTAR file is the only dialect produced by our current packer. - for (const [start, end] of [ - [157, 257], - [265, 329], - [345, 512], - ]) { - if (start === undefined || end === undefined) refuse('TAR_FORMAT'); - if (header.subarray(start, end).some((byte) => byte !== 0)) refuse('TAR_METADATA'); - } - for (const [start, end] of [ - [108, 116], - [116, 124], - [329, 337], - [337, 345], - ]) { - if (start === undefined || end === undefined) refuse('TAR_FORMAT'); - if (octal(header.subarray(start, end)) !== 0) refuse('TAR_METADATA'); - } - octal(header.subarray(136, 148)); // Validate mtime syntax without requiring a clock value. - const path = nameField(header.subarray(0, 100)); - assertPublicPackagePath(path); - const mode = octal(header.subarray(100, 108)); - if (mode !== (path === 'package/dist/index.js' ? 0o755 : 0o644)) refuse('TAR_MODE'); - const size = octal(header.subarray(124, 136)); - if (size > ARCHIVE_LIMITS.fileBytes) refuse('FILE_SIZE'); - return { path, mode, size }; -} - /** @param {Buffer} bytes @param {Record} expected */ function manifest(bytes, expected) { if (bytes.length > ARCHIVE_LIMITS.manifestBytes) refuse('MANIFEST_SIZE'); @@ -176,13 +148,7 @@ function manifest(bytes, expected) { if (dependencies === undefined) continue; if (!isRecord(dependencies)) refuse('MANIFEST_RUNTIME'); for (const [name, version] of Object.entries(dependencies)) { - if ( - name.startsWith('@relavium/') || - typeof version !== 'string' || - /^(?:workspace|catalog|file|link):/u.test(version) - ) { - refuse('MANIFEST_RUNTIME'); - } + if (!isPublicRuntimeDependency(name, version)) refuse('MANIFEST_RUNTIME'); } } return parsed; @@ -225,59 +191,45 @@ export function inspectPackageArchive(bytes, { expectedFiles, expectedPackage }) if (bytes.byteLength > ARCHIVE_LIMITS.compressedBytes) refuse('GZIP_SIZE'); const compressed = Buffer.from(bytes); const tar = unpack(compressed); - const seen = new Set(); - const portableNames = new Set(); const files = []; - let offset = 0; - let ended = false; /** @type {Record | undefined} */ let packageManifest; - while (offset < tar.length) { - if (tar.length - offset < 512) refuse('TAR_TRUNCATED'); - const header = tar.subarray(offset, offset + 512); - if (header.every((byte) => byte === 0)) { - if (tar.length - offset !== 1024 || tar.subarray(offset).some((byte) => byte !== 0)) { - refuse('TAR_END'); - } - ended = true; - break; - } - if (files.length >= ARCHIVE_LIMITS.files) refuse('FILE_COUNT'); - const { path, mode, size } = readHeader(header); - if (seen.has(path)) refuse('DUPLICATE_FILE'); - seen.add(path); - if (portableNames.has(path.toLowerCase())) refuse('PATH_ALIAS'); - portableNames.add(path.toLowerCase()); - const expected = expectedFiles.get(path); - if (expected === undefined) refuse('EXTRA_FILE'); - const start = offset + 512; - const end = start + size; - const next = start + Math.ceil(size / 512) * 512; - if (next > tar.length) refuse('TAR_TRUNCATED'); - if (tar.subarray(end, next).some((byte) => byte !== 0)) refuse('TAR_PADDING'); - const contents = tar.subarray(start, end); - if (path === 'package/package.json') packageManifest = manifest(contents, expectedPackage); - if (expected instanceof Uint8Array) { - if (path !== 'package/package.json' && !contents.equals(expected)) refuse('FILE_DRIFT'); - } else if (contents.length !== expected.bytes || digest(contents) !== expected.sha256) { - refuse('FILE_DRIFT'); - } - if (path.endsWith('.js')) { - const source = utf8(contents); - if ( - path === 'package/dist/index.js' - ? !source.startsWith('#!/usr/bin/env node\n') - : source.startsWith('#!') + readUstar(tar, { + admitPath: assertPublicPackagePath, + modeFor: (path) => (path === 'package/dist/index.js' ? 0o755 : 0o644), + maxFiles: ARCHIVE_LIMITS.files, + maxFileBytes: ARCHIVE_LIMITS.fileBytes, + refuse, + admitEntry: (path) => { + if (!expectedFiles.has(path)) refuse('EXTRA_FILE'); + }, + onFile: ({ path, mode, size, contents }) => { + const expected = expectedFiles.get(path); + if (path === 'package/package.json') packageManifest = manifest(contents, expectedPackage); + if (expected instanceof Uint8Array) { + if (path !== 'package/package.json' && !contents.equals(expected)) refuse('FILE_DRIFT'); + } else if ( + expected === undefined || + contents.length !== expected.bytes || + digest(contents) !== expected.sha256 ) { - refuse('HASHBANG'); + refuse('FILE_DRIFT'); } - if (/sourceMappingURL\s*=/u.test(source)) refuse('MAP_LINK'); - } - files.push({ path, bytes: size, sha256: digest(contents), mode }); - offset = next; - } - if (!ended) refuse('TAR_END'); - if (seen.size !== expectedFiles.size || !packageManifest) refuse('MISSING_FILE'); + if (path.endsWith('.js')) { + const source = utf8(contents); + if ( + path === 'package/dist/index.js' + ? !source.startsWith('#!/usr/bin/env node\n') + : source.startsWith('#!') + ) { + refuse('HASHBANG'); + } + if (/sourceMappingURL\s*=/u.test(source)) refuse('MAP_LINK'); + } + files.push({ path, bytes: size, sha256: digest(contents), mode }); + }, + }); + if (files.length !== expectedFiles.size || !packageManifest) refuse('MISSING_FILE'); return { sha256: digest(compressed), bytes: compressed.length, diff --git a/tools/cli-package/archive.test.mjs b/tools/cli-package/archive.test.mjs index fd381714..ce9bf97b 100644 --- a/tools/cli-package/archive.test.mjs +++ b/tools/cli-package/archive.test.mjs @@ -4,6 +4,7 @@ import { createHash } from 'node:crypto'; import { test } from 'node:test'; import { gzipSync } from 'node:zlib'; import { ARCHIVE_LIMITS, inspectPackageArchive, PackageArchiveError } from './archive.mjs'; +import { checksum, entry, ustar } from './ustar-fixture.mjs'; const expectedPackage = { name: 'relavium', @@ -30,40 +31,9 @@ function inventory() { ]); } -/** @param {Buffer} header */ -function checksum(header) { - header.fill(32, 148, 156); - header.write( - header - .reduce((sum, byte) => sum + byte, 0) - .toString(8) - .padStart(6, '0'), - 148, - ); - header[154] = 0; - header[155] = 32; -} - -/** @param {string} path @param {Uint8Array} bytes */ -function entry(path, bytes) { - const header = Buffer.alloc(512); - header.write(path, 0, 100); - for (const offset of [100, 108, 116, 329, 337]) header.write('000000 \0', offset); - header.write(path === 'package/dist/index.js' ? '000755' : '000644', 100); - header.write(bytes.length.toString(8).padStart(11, '0') + ' ', 124); - header.write('00000000000 ', 136); - header[156] = 48; - header.write('ustar\0' + '00', 257); - checksum(header); - return Buffer.concat([header, bytes, Buffer.alloc((512 - (bytes.length % 512)) % 512)]); -} - /** @param {Map} [files] */ function tar(files = inventory()) { - return Buffer.concat([ - ...[...files].map(([path, bytes]) => entry(path, bytes)), - Buffer.alloc(1024), - ]); + return ustar(files); } /** @param {Buffer} input @param {string} code @param {Map} [files] */ @@ -326,6 +296,16 @@ test('manifest rejects runtime drift and unresolved/private runtime dependencies { commander: 'catalog:' }, { commander: 'workspace:*' }, { '@relavium/core': '0.0.0' }, + // A PR #96 review: a denylist admitted every other non-registry specifier. + { commander: 'npm:evil@1.0.0' }, + { commander: 'github:attacker/commander' }, + { commander: 'git+https://example.invalid/commander.git' }, + { commander: 'https://example.invalid/commander-12.1.0.tgz' }, + { commander: '*' }, + { commander: '>=12.0.0' }, + { commander: 'latest' }, + { commander: '' }, + { commander: 12 }, ]) { const badPackage = { ...expectedPackage, dependencies }; const files = inventory(); diff --git a/tools/cli-package/binding.mjs b/tools/cli-package/binding.mjs index d2236b4c..928dc82b 100644 --- a/tools/cli-package/binding.mjs +++ b/tools/cli-package/binding.mjs @@ -1,12 +1,19 @@ import { isDeepStrictEqual } from 'node:util'; import { BOOTSTRAP_CONTROL_COUNT } from '../cli-bootstrap/inventory.mjs'; -import { ARCHIVE_LIMITS, assertPublicPackagePath } from './archive.mjs'; +import { ARCHIVE_LIMITS, assertPublicPackagePath, isPublicRuntimeDependency } from './archive.mjs'; import { REQUIRED_SMOKE_CASES } from './smoke/cases.mjs'; /** @typedef {{kind: 'local'} | {kind: 'github', repository: string, workflow: string, runId: string, runAttempt: string, event: string, source: string}} Origin */ /** @typedef {{path: string, bytes: number, sha256: string, mode: number}} PublicFile */ /** @typedef {{source: string, origin: Origin, nodeMajor: number, allowLocal?: boolean}} Expected */ +/** + * The package-test receipt's mode (Step 9a.2b): the producer verified the maps and symbolicated actual + * packed frames before writing it, and the product map controls are required by the same commit's + * `ci`. The mode is a claim, not evidence of those checks; the earlier `compiled_package_test` is refused. + */ +export const PACKAGE_TEST_MODE = 'package_test'; + export const REQUIRED_CONSUMER_LEGS = Object.freeze([ 'current-linux', 'current-macos', @@ -65,9 +72,19 @@ function sha(value) { function sourceSha(value) { return string(value, /^[0-9a-f]{40}$/u); } +const PACKAGE_VERSION = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/u; +/** + * Whether `value` is a package version a receipt admits: a SemVer core with an optional prerelease and + * build, at most 100 characters. The private manifest admits exactly the same versions. + * @param {unknown} value @returns {value is string} + */ +export function isPackageVersion(value) { + return typeof value === 'string' && value.length <= 100 && PACKAGE_VERSION.test(value); +} /** @param {unknown} value */ function version(value) { - return string(value, /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/u, 100); + if (!isPackageVersion(value)) refuse('VALUE'); + return value; } /** @param {unknown} value */ function nodeVersion(value) { @@ -201,13 +218,7 @@ function publicPackage(value) { refuse('PACKAGE'); for (const [name, requirement] of Object.entries(dependencies)) { string(name, /^(?:@[a-z0-9_.-]+\/)?[a-z0-9_.-]+$/u, 200); - if ( - name.startsWith('@relavium/') || - typeof requirement !== 'string' || - requirement.length > 100 || - /(?:workspace|catalog|file|link):/u.test(requirement) - ) - refuse('PACKAGE'); + if (!isPublicRuntimeDependency(name, requirement)) refuse('PACKAGE'); } // Full package metadata is public npm material. Keep it bounded; private map // manifests and build/source inventories have no admitted top-level field. @@ -235,7 +246,7 @@ export function validatePackageReceipt(value, expected) { ]); if ( receipt.version !== 1 || - receipt.mode !== 'compiled_package_test' || + receipt.mode !== PACKAGE_TEST_MODE || receipt.source !== expected.source || typeof receipt.dirty !== 'boolean' ) @@ -258,7 +269,7 @@ export function validatePackageReceipt(value, expected) { if (manifest.version !== packageVersion) refuse('PACKAGE'); return { version: 1, - mode: 'compiled_package_test', + mode: PACKAGE_TEST_MODE, source: expected.source, dirty: receipt.dirty, packageVersion, diff --git a/tools/cli-package/binding.test.mjs b/tools/cli-package/binding.test.mjs index 804f615c..06660761 100644 --- a/tools/cli-package/binding.test.mjs +++ b/tools/cli-package/binding.test.mjs @@ -14,6 +14,7 @@ import { REQUIRED_CONSUMER_LEGS, requirePackageJobResults, validateConsumerRecord, + isPackageVersion, validatePackageReceipt, } from './binding.mjs'; import { REQUIRED_SMOKE_CASES } from './smoke/cases.mjs'; @@ -25,7 +26,7 @@ const expected = { source, origin: { kind: 'local' }, nodeMajor: 24 }; function receipt() { return { version: 1, - mode: 'compiled_package_test', + mode: 'package_test', source, dirty: false, packageVersion: '0.1.1', @@ -108,6 +109,13 @@ test('one actual artifact binds every required OS/minimum-runtime consumer', () assert.equal(result.tarballSha256, digest); }); +test('a package version is a SemVer core from its first character', () => { + assert.equal(isPackageVersion('0.1.1'), true); + for (const version of ['v0.1.1', 'x0.1.1', '../0.1.1', 'relavium-0.1.1', ' 0.1.1', '0.1.1 ']) { + assert.equal(isPackageVersion(version), false, version); + } +}); + test('strict receipt refuses omitted or added private/custody claims', () => { for (const key of Object.keys(receipt())) refuses(() => validatePackageReceipt(omit(receipt(), key), expected)); @@ -122,6 +130,8 @@ test('strict receipt refuses omitted or added private/custody claims', () => { } for (const changed of [ { mode: 'release' }, + // The mode before the product map controls existed. + { mode: 'compiled_package_test' }, { source: '0'.repeat(40) }, { dirty: 'false' }, { version: 2 }, @@ -176,6 +186,9 @@ test('package metadata is public-only and keeps private packages out of runtime' { dependencies: { '@relavium/core': '0.0.0' } }, { dependencies: { commander: 'workspace:*' } }, { dependencies: { commander: 'catalog:' } }, + ...['npm:evil@1.0.0', 'github:attacker/commander', 'https://example.invalid/c.tgz', '*'].map( + (requirement) => ({ dependencies: { commander: requirement } }), + ), ]) refuses(() => validatePackageReceipt({ ...pack, package: { ...pack.package, ...changed } }, expected), diff --git a/tools/cli-package/candidate-fixture.mjs b/tools/cli-package/candidate-fixture.mjs new file mode 100644 index 00000000..f95deaec --- /dev/null +++ b/tools/cli-package/candidate-fixture.mjs @@ -0,0 +1,278 @@ +/** + * Test-only finalized candidates for the private manifest and symbolication tests. The TypeScript sits + * at the symbolication anchors' real repository paths with the same function names, the CLI imports + * its canary into the project's `.relavium/` found by the same upward search as the real CLI, and the + * exact esbuild behind the installed tsup bundles it with splitting and minify, as the CLI build does. + * The maps are staged with repository-relative sources and recorded as the finalizer records them. + */ +import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; +import { createHash } from 'node:crypto'; +import { + chmodSync, + cpSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { dirname, join, posix } from 'node:path'; +import { after } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { mapTrailer } from './finalize.mjs'; +import { MANIFEST_INPUTS, sealPrivateManifest } from './manifest.mjs'; + +export const cliRequire = createRequire(new URL('../../apps/cli/package.json', import.meta.url)); +const { build } = createRequire(cliRequire.resolve('tsup/package.json'))('esbuild'); +export const ts = cliRequire('typescript'); +/** The directory of an installed package the CLI resolves. */ +const installed = (name) => dirname(cliRequire.resolve(`${name}/package.json`)); +export const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-candidate-fixture-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); +export const digest = (bytes) => createHash('sha256').update(bytes).digest('hex'); +export const posixModes = process.platform !== 'win32'; +export const PRELOAD = fileURLToPath(new URL('./frame-capture.mjs', import.meta.url)); +let count = 0; +/** A fresh scratch directory name. */ +export const fresh = (name) => { + count += 1; + return join(scratch, `${name}-${count}`); +}; + +export const SOURCES = { + 'packages/core/src/yaml-decode.ts': [ + "import { parse as parseYaml } from 'yaml';", + '', + 'export function decodeHardenedYaml(text: string): unknown {', + " return parseYaml(text, { version: '1.2' });", + '}', + '', + ], + 'packages/core/src/parser.ts': [ + "import { decodeHardenedYaml } from './yaml-decode.js';", + '', + 'type Raw = { schema_version?: unknown; workflow?: { id?: unknown } };', + '', + 'export function parseWorkflow(text: string): { id: string } {', + ' const raw = decodeHardenedYaml(text) as Raw;', + " if (raw.schema_version !== '1.0' || typeof raw.workflow?.id !== 'string') {", + " throw new Error('not a workflow');", + ' }', + ' return { id: raw.workflow.id };', + '}', + '', + ], + 'apps/cli/src/authoring/authoring.ts': [ + "import { parseWorkflow } from '../../../../packages/core/src/parser.js';", + '', + 'export function detectAndParse(text: string, fileName: string): { id: string } {', + " if (fileName.endsWith('.relavium.yaml')) {", + ' return parseAsWorkflow(text);', + ' }', + " throw new Error('unsupported');", + '}', + '', + 'function parseAsWorkflow(text: string): { id: string } {', + ' return parseWorkflow(text);', + '}', + '', + ], + 'apps/cli/src/commands/import.ts': [ + "import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';", + "import { basename, dirname, join, resolve } from 'node:path';", + "import { detectAndParse } from '../authoring/authoring.js';", + '', + '// The real CLI walks up from its working directory to the first `.relavium/` (config/paths.ts).', + 'function projectConfigDir(cwd: string): string {', + ' let current = resolve(cwd);', + ' for (;;) {', + " const found = join(current, '.relavium');", + ' if (existsSync(found) && statSync(found).isDirectory()) return found;', + ' const parent = dirname(current);', + " if (parent === current) return join(resolve(cwd), '.relavium');", + ' current = parent;', + ' }', + '}', + '', + 'export function importCommand(path: string, cwd: string): string {', + " const text = readFileSync(path, 'utf8');", + ' const id = detectAndParse(text, basename(path)).id;', + " const target = join(projectConfigDir(cwd), 'workflows', `${id}.relavium.yaml`);", + ' mkdirSync(dirname(target), { recursive: true });', + ' writeFileSync(target, text);', + ' return id;', + '}', + '', + ], + 'apps/cli/src/index.ts': [ + '#!/usr/bin/env node', + "import { Entry } from '@napi-rs/keyring';", + 'try {', + " new Entry('relavium', 'probe');", + '} catch {', + ' // The preload replaces the keyring binding with one that refuses.', + '}', + 'const [command, path, flag, cwd] = process.argv.slice(2);', + "if (command === 'import' && path !== undefined) {", + " const { importCommand } = await import('./commands/import.js');", + " importCommand(path, flag === '--cwd' && cwd !== undefined ? cwd : process.cwd());", + '}', + '', + ], +}; + +/** Write `path` (repository-relative) under `root`. */ +export function put(root, path, bytes) { + mkdirSync(dirname(join(root, ...path.split('/'))), { recursive: true }); + writeFileSync(join(root, ...path.split('/')), bytes); +} + +/** + * A finalized synthetic candidate: its finalized dist, its staged maps with repository-relative + * sources, the finalizer's record, the manifest inputs and the CLI installation, with an owner-only + * staging directory and work directory (under `parent` when given). + */ +export async function candidate({ sources = SOURCES, parent = scratch } = {}) { + const root = fresh('candidate'); + const staging = fresh('staging'); + count += 1; + const work = join(parent, `work-${count}`); + for (const directory of [staging, work]) { + mkdirSync(directory, { recursive: true }); + chmodSync(directory, 0o700); + } + mkdirSync(join(staging, 'maps')); + for (const [path, lines] of Object.entries(sources)) put(root, path, lines.join('\n')); + for (const path of MANIFEST_INPUTS) { + put( + root, + path, + path.endsWith('.json') ? `${JSON.stringify({ synthetic: path })}\n` : `# ${path}\n`, + ); + } + put(root, 'apps/cli/tsup.config.ts', "export default {\n target: 'node22',\n};\n"); + put(root, 'apps/cli/package.json', JSON.stringify({ name: 'relavium', type: 'module' })); + // The CLI's installation: the build tools the test process reads link to the repository's, and + // `yaml`, the one runtime package the packed CLI loads, is a copy inside the candidate laid out as + // pnpm installs it (the package below the root's `node_modules/.pnpm`, linked from the CLI's), because + // the preload refuses any module outside the candidate. + const modules = join(root, 'apps', 'cli', 'node_modules'); + mkdirSync(join(modules, '@napi-rs'), { recursive: true }); + for (const name of ['typescript', 'tsup', '@napi-rs/keyring']) { + symlinkSync(installed(name), join(modules, ...name.split('/')), 'junction'); + } + const store = join(root, 'node_modules', '.pnpm', 'yaml', 'node_modules', 'yaml'); + cpSync(installed('yaml'), store, { recursive: true }); + symlinkSync(store, join(modules, 'yaml'), 'junction'); + const cli = join(root, 'apps', 'cli'); + const built = await build({ + absWorkingDir: cli, + entryPoints: { index: 'src/index.ts' }, + outdir: 'dist', + format: 'esm', + platform: 'node', + target: 'node22', + bundle: true, + splitting: true, + minify: true, + sourcemap: 'external', + sourcesContent: false, + external: ['yaml', '@napi-rs/keyring'], + write: false, + logLevel: 'silent', + }); + const outputs = []; + const maps = []; + for (const file of built.outputFiles) { + const name = posix.basename(file.path.replaceAll('\\', '/')); + if (name.endsWith('.map')) { + const map = JSON.parse(file.text); + map.sources = map.sources.map((source) => + posix.normalize(posix.join('apps/cli/dist', source)), + ); + const bytes = Buffer.from(JSON.stringify(map)); + writeFileSync(join(staging, 'maps', name), bytes); + maps.push({ path: `dist/${name}`, bytes: bytes.length, sha256: digest(bytes) }); + } else { + // An external map leaves no link in the code: this is already the finalized JS. + const code = file.text; + assert.equal(code.includes(mapTrailer(name)), false); + put(cli, `dist/${name}`, code); + const bytes = Buffer.from(code); + outputs.push({ path: `dist/${name}`, bytes: bytes.length, sha256: digest(bytes) }); + } + } + writeFileSync( + join(staging, 'finalized.json'), + JSON.stringify({ version: 1, outputs, maps, upstreamMaps: 0 }), + ); + const archive = { + sha256: 'a'.repeat(64), + package: { version: '0.1.1' }, + files: outputs.map((output) => ({ path: `package/${output.path}`, sha256: output.sha256 })), + }; + return { root, cli, staging, work, archive, outputs }; +} + +export const TREE = 'b'.repeat(40); +export const COMMIT = 'c'.repeat(40); +/** Git as the manifest asks it: the commit's tree. */ +export const git = async (command, args) => { + assert.equal(command, 'git'); + assert.deepEqual(args, ['--no-replace-objects', 'rev-parse', '--verify', `${COMMIT}^{tree}`]); + return { stdout: `${TREE}\n`, stderr: '' }; +}; + +/** Seal the manifest of a dirty local candidate, whose source revision is its own copy. */ +export function seal(fixture) { + return sealPrivateManifest({ + candidate: fixture.root, + staging: fixture.staging, + checkout: fixture.root, + source: COMMIT, + dirty: true, + archive: fixture.archive, + run: git, + }); +} + +/** Re-record the finalized digests of the staged maps only, after a deliberate map change. */ +export function rerecordMaps(fixture, mutateRecord = () => {}) { + const record = JSON.parse(readFileSync(join(fixture.staging, 'finalized.json'), 'utf8')); + record.maps = readdirSync(join(fixture.staging, 'maps')).map((name) => { + const bytes = readFileSync(join(fixture.staging, 'maps', name)); + return { path: `dist/${name}`, bytes: bytes.length, sha256: digest(bytes) }; + }); + mutateRecord(record); + writeFileSync(join(fixture.staging, 'finalized.json'), JSON.stringify(record)); +} + +/** Re-record every finalized digest after a deliberate change, then seal a fresh manifest. */ +export function reseal(fixture) { + const outputs = readdirSync(join(fixture.cli, 'dist')).map((name) => { + const bytes = readFileSync(join(fixture.cli, 'dist', name)); + return { path: `dist/${name}`, bytes: bytes.length, sha256: digest(bytes) }; + }); + rerecordMaps(fixture, (record) => (record.outputs = outputs)); + rmSync(join(fixture.staging, 'manifest.json'), { force: true }); + fixture.archive.files = outputs.map((output) => ({ + path: `package/${output.path}`, + sha256: output.sha256, + })); + return seal(fixture); +} + +/** The finalized chunk that holds the core call sites. */ +export function coreChunk(fixture) { + return readdirSync(join(fixture.cli, 'dist')).find((name) => + JSON.parse(readFileSync(join(fixture.staging, 'maps', `${name}.map`), 'utf8')).sources.includes( + 'packages/core/src/parser.ts', + ), + ); +} diff --git a/tools/cli-package/candidate.mjs b/tools/cli-package/candidate.mjs new file mode 100644 index 00000000..c0a70db3 --- /dev/null +++ b/tools/cli-package/candidate.mjs @@ -0,0 +1,493 @@ +/** + * The owned isolated candidate tree (ADR-0111, Wave 3 Step 9a.1). A release-map build never runs in + * the developer's workspace: the producer copies the exact listed source files of the original + * checkout into a fresh private tree, installs the locked dependency closure there from the existing + * content-addressed store without lifecycle scripts or network, and verifies that every workspace + * link resolves inside that tree. No original workspace `dist` can stand in for freshly mapped + * output, and no build cache is read or written. + */ +import assert from 'node:assert/strict'; +import { + chmodSync, + existsSync, + lstatSync, + mkdirSync, + readdirSync, + realpathSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { dirname, isAbsolute, join, resolve } from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; +import { ENGINE_PACKAGES } from './finalize.mjs'; +import { readOrdinaryFile, byCodeUnit } from './files.mjs'; +import { within } from './paths.mjs'; +import { + GIT_NULL_DEVICE, + isolatedEnvironment, + isolatedGitEnvironment, + runCommand, + runPnpm, +} from './process.mjs'; + +export const CANDIDATE_LIMITS = Object.freeze({ + files: 20_000, + fileBytes: 64 * 1024 * 1024, + totalBytes: 512 * 1024 * 1024, +}); + +/** Only the variables pnpm uses to locate its store; never credentials, preloads or registries. */ +const STORE_DISCOVERY_VARIABLES = Object.freeze([ + 'HOME', + 'USERPROFILE', + 'XDG_DATA_HOME', + 'XDG_CONFIG_HOME', + 'LOCALAPPDATA', + 'APPDATA', + 'PNPM_HOME', + 'npm_config_store_dir', + 'NPM_CONFIG_STORE_DIR', +]); + +/** @param {string} output NUL-separated Git path list */ +function gitPaths(output) { + return output.split('\0').filter((path) => path !== ''); +} + +/** Names the developer's global Git excludes file for a producer whose own environment is isolated. */ +export const GIT_EXCLUDES_ENV = 'RELAVIUM_GIT_EXCLUDES_FILE'; + +/** + * Only the variables Git uses to locate its global configuration, plus the command-scope configuration + * the developer's Git takes from the environment (`GIT_CONFIG_COUNT` with its `GIT_CONFIG_KEY_n` and + * `GIT_CONFIG_VALUE_n`, and `GIT_CONFIG_PARAMETERS`); never credentials or hooks. They reach only the + * read-only lookup of `core.excludesFile`, which runs nothing. + */ +const GIT_CONFIG_VARIABLES = Object.freeze([ + 'HOME', + 'USERPROFILE', + 'XDG_CONFIG_HOME', + 'GIT_CONFIG_GLOBAL', + 'GIT_CONFIG_SYSTEM', + 'GIT_CONFIG_NOSYSTEM', + 'GIT_CONFIG_COUNT', + 'GIT_CONFIG_PARAMETERS', + 'SystemRoot', + 'WINDIR', +]); +const GIT_CONFIG_ENTRY = /^GIT_CONFIG_(?:KEY|VALUE)_(?:0|[1-9]\d{0,3})$/u; + +/** + * The developer's global Git excludes file, so that the candidate's isolated Git ignores what the + * developer's own `git status` ignores: the explicit `RELAVIUM_GIT_EXCLUDES_FILE` when a caller + * already isolated this process's environment, otherwise `core.excludesFile` as the developer's Git + * reads it (present but empty: none, as Git then applies no global excludes), else, when the setting + * is absent, Git's default `$XDG_CONFIG_HOME/git/ignore` or `$HOME/.config/git/ignore`; undefined + * when no such file exists. Only that location is read from the developer's configuration: every + * candidate Git command runs without global or system configuration (`isolatedGitEnvironment`), so no + * global hook, fsmonitor or other setting applies, wherever it is stored. + * @param {{repository: string, environment: Record, run?: typeof runCommand}} options + * @returns {Promise} + */ +export async function globalExcludesFile({ repository, environment, run = runCommand }) { + const explicit = environment[GIT_EXCLUDES_ENV]; + if (explicit !== undefined) { + assert.ok(isAbsolute(explicit) && existsSync(explicit), 'PACKED_CLI_GIT_EXCLUDES'); + return explicit; + } + // No PATH means no PATH: an empty one would make the spawn look the command up in the checkout. + const env = environment.PATH === undefined ? {} : { PATH: environment.PATH }; + for (const key of GIT_CONFIG_VARIABLES) { + if (environment[key] !== undefined) env[key] = environment[key]; + } + for (const [key, value] of Object.entries(environment)) { + if (GIT_CONFIG_ENTRY.test(key) && value !== undefined) env[key] = value; + } + let configured; + try { + configured = ( + await run('git', ['config', '--path', '--get', 'core.excludesFile'], { cwd: repository, env }) + ).stdout.trim(); + } catch (error) { + // Git exits 1 when the setting is absent; anything else is a refusal. + if (error?.result?.code !== 1) throw error; + } + const xdg = environment.XDG_CONFIG_HOME; + const home = environment.HOME ?? environment.USERPROFILE; + let path; + // A setting present but empty disables Git's global excludes; only an absent one has a default. + if (configured !== undefined) + path = configured === '' ? undefined : resolve(repository, configured); + else if (xdg !== undefined && xdg !== '') path = join(xdg, 'git', 'ignore'); + else if (home !== undefined && home !== '') path = join(home, '.config', 'git', 'ignore'); + return path !== undefined && existsSync(path) && statSync(path).isFile() ? path : undefined; +} + +/** + * The environment of a producer that a caller has isolated: `base` plus the dependency store and, when + * the caller found one, the developer's global Git excludes file, both found in the caller's own + * environment (`dependencyStore`, `globalExcludesFile`). + * @param {Record} base + * @param {{store: string, excludes?: string}} found + * @returns {Record} + */ +export function producerEnvironment(base, { store, excludes }) { + return { + ...base, + [PACKAGE_STORE_ENV]: store, + ...(excludes === undefined ? {} : { [GIT_EXCLUDES_ENV]: excludes }), + }; +} + +/** + * The attributes and excludes files a checkout-reading Git uses, named explicitly so that no default + * location applies — not even one Git derives from the null-device home on Windows. + * @param {string | undefined} excludes @returns {string[]} + */ +function readArguments(excludes) { + return [ + '-c', + `core.attributesFile=${GIT_NULL_DEVICE}`, + '-c', + `core.excludesFile=${excludes ?? GIT_NULL_DEVICE}`, + ]; +} + +/** + * The tracked paths whose working-tree changes `git status` cannot see: entries marked skip-worktree + * (`S`) or assume-unchanged (a lowercase tag) in `git ls-files -v`. + * @param {string} output NUL-separated `git ls-files -v -z` entries + */ +function uncheckedPaths(output) { + return gitPaths(output) + .filter((entry) => entry.startsWith('S') || /^[a-z]$/u.test(entry[0])) + .map((entry) => entry.slice(2)); +} + +/** + * The source identity of the original checkout: its exact commit, its full porcelain status and the + * tracked files Git is told not to check, so a candidate copied from a modified tree records `dirty` + * and a change to any of the three during the copy refuses. A skip-worktree or assume-unchanged file + * counts as dirty whatever its bytes, since its copy is never compared with the commit. Untracked + * files are judged with the developer's global excludes (`globalExcludesFile`). + * @param {string} repository @param {typeof runCommand} [run] @param {string} [excludes] + */ +export async function sourceIdentity(repository, run = runCommand, excludes = undefined) { + const options = { cwd: repository, env: isolatedGitEnvironment() }; + const source = ( + await run('git', ['--no-replace-objects', 'rev-parse', 'HEAD'], options) + ).stdout.trim(); + assert.match(source, /^[0-9a-f]{40}$/u, 'PACKED_CLI_SOURCE_IDENTITY'); + const status = ( + await run( + 'git', + [...readArguments(excludes), 'status', '--porcelain=v1', '-z', '--untracked-files=all'], + options, + ) + ).stdout; + const unchecked = uncheckedPaths((await run('git', ['ls-files', '-v', '-z'], options)).stdout); + return { source, status, unchecked, dirty: status !== '' || unchecked.length > 0 }; +} + +/** + * Every file Git would ship from the working tree: tracked plus untracked-but-not-ignored, minus + * deletions, by the repository's ignore rules and the developer's global excludes + * (`globalExcludesFile`). Ignored output, caches, dependencies and private notes never enter a + * candidate; an untracked nested repository, which Git lists only as its directory, refuses. + * @param {string} repository @param {typeof runCommand} [run] @param {string} [excludes] + * @param {{files: number}} [limits] the bounds (`CANDIDATE_LIMITS`; smaller ones in tests) + */ +export async function candidateFiles( + repository, + run = runCommand, + excludes = undefined, + limits = CANDIDATE_LIMITS, +) { + const options = { cwd: repository, env: isolatedGitEnvironment() }; + const listed = gitPaths( + ( + await run( + 'git', + [ + ...readArguments(excludes), + 'ls-files', + '-z', + '--cached', + '--others', + '--exclude-standard', + ], + options, + ) + ).stdout, + ); + // Git lists an untracked nested repository as its directory (`dir/`), never its files. + assert.ok( + listed.every((path) => !path.endsWith('/')), + 'PACKED_CLI_CANDIDATE_NESTED_REPOSITORY', + ); + const deleted = new Set( + gitPaths((await run('git', ['ls-files', '-z', '--deleted'], options)).stdout), + ); + const files = [...new Set(listed)].filter((path) => !deleted.has(path)).sort(byCodeUnit); + assert.ok(files.length > 0 && files.length <= limits.files, 'PACKED_CLI_CANDIDATE_FILES'); + return files; +} + +/** + * Copy each listed ordinary file byte-for-byte into the empty `destination`. Links (also as a parent + * directory of a listed path), special files, escaping or noncanonical paths and oversized trees + * refuse; only the owner's executable bit is kept, as Git records it, as an + * exact 0755 or 0644 mode under any umask that lets the owner write (one that removes the owner's + * write bit leaves the created directories unwritable, and the copy refuses before any build). + * @param {{repository: string, destination: string, files: readonly string[], + * limits?: {fileBytes: number, totalBytes: number}}} options `limits` defaults to `CANDIDATE_LIMITS` + */ +export function copyCandidateTree({ repository, destination, files, limits = CANDIDATE_LIMITS }) { + assert.ok( + isAbsolute(destination) && readdirSync(destination).length === 0, + 'PACKED_CLI_CANDIDATE_ROOT', + ); + let total = 0; + const directories = new Set(); + for (const path of files) { + assert.ok( + typeof path === 'string' && + path.length > 0 && + !path.includes('\\') && + path.split('/').every((part) => part !== '' && part !== '.' && part !== '..'), + 'PACKED_CLI_CANDIDATE_PATH', + ); + const parts = path.split('/'); + // Every parent below the checkout must be an ordinary directory: Git lists the tracked children + // of a directory replaced by an (ignored) link, and reading through it would copy outside bytes. + for (let depth = 1; depth < parts.length; depth += 1) { + const parent = parts.slice(0, depth).join('/'); + if (directories.has(parent)) continue; + assert.ok( + lstatSync(join(repository, ...parts.slice(0, depth))).isDirectory(), + 'PACKED_CLI_CANDIDATE_PATH', + ); + directories.add(parent); + } + const source = join(repository, ...parts); + // Git records a file as executable by its owner's execute bit alone. + const executable = (lstatSync(source).mode & 0o100) !== 0; + const bytes = readOrdinaryFile(source, limits.fileBytes); + total += bytes.length; + assert.ok(total <= limits.totalBytes, 'PACKED_CLI_CANDIDATE_BYTES'); + const target = join(destination, ...path.split('/')); + mkdirSync(dirname(target), { recursive: true, mode: 0o700 }); + const mode = executable ? 0o755 : 0o644; + writeFileSync(target, bytes, { flag: 'wx', mode }); + // The creation mode passes through the umask; the candidate keeps the exact mode. + if (process.platform !== 'win32') chmodSync(target, mode); + } + return { files: files.length, bytes: total }; +} + +/** An explicit store directory, set by a caller that discovered it in its own environment. */ +export const PACKAGE_STORE_ENV = 'RELAVIUM_PACKAGE_STORE_DIR'; + +/** + * The developer's existing content-addressed dependency store: the explicit + * `RELAVIUM_PACKAGE_STORE_DIR` when a caller already isolated this process's environment, + * otherwise discovered by the pinned pnpm with only the store-location variables of `environment`. + * Temporary files and pnpm's caches go to the private `home`, never into the original checkout; a + * corepack-managed pnpm runs from the developer's own corepack cache (`COREPACK_HOME`, see + * `corepackHome` in process.mjs), never from the shared temporary directory. + * @param {{repository: string, environment: Record, home: string, + * pnpm?: typeof runPnpm}} options + */ +export async function dependencyStore({ repository, environment, home, pnpm = runPnpm }) { + const explicit = environment[PACKAGE_STORE_ENV]; + if (explicit !== undefined) { + assert.ok(isAbsolute(explicit) && existsSync(explicit), 'PACKED_CLI_DEPENDENCY_STORE'); + return explicit; + } + assert.ok(isAbsolute(home) && !within(repository, home), 'PACKED_CLI_PRIVATE_HOME'); + const env = isolatedEnvironment(home); + for (const key of STORE_DISCOVERY_VARIABLES) { + if (environment[key] === undefined) delete env[key]; + else env[key] = environment[key]; + } + const store = (await pnpm(['store', 'path'], { cwd: repository, env })).stdout.trim(); + assert.ok(isAbsolute(store) && existsSync(store), 'PACKED_CLI_DEPENDENCY_STORE'); + return store; +} + +/** + * Whether every copied file is byte for byte the commit's blob. A clean status does not prove it: + * line-ending conversion (`core.autocrlf`, the Git for Windows default) and Git's stat cache can report + * converted working-tree bytes as unchanged, and the candidate is built from those bytes. Each copy, + * an ordinary file, must have the commit tree's blob id for its path, as Git itself names its raw bytes + * (`hash-object --no-filters`): Git's collision-detecting SHA-1 in the repository's object format, not a + * plain hash of ours. + * @param {{repository: string, candidate: string, source: string, files: readonly string[], + * run?: typeof runCommand}} options + */ +export async function copyMatchesCommit({ + repository, + candidate, + source, + files, + run = runCommand, +}) { + const git = async (args) => + ( + await run('git', ['--no-replace-objects', ...args], { + cwd: repository, + env: isolatedGitEnvironment(), + }) + ).stdout; + const blobs = new Map(); + for (const entry of gitPaths(await git(['ls-tree', '-r', '-z', '--full-tree', source]))) { + const match = /^[0-7]{6} blob ([0-9a-f]{40,64})\t(.+)$/su.exec(entry); + if (match !== null) blobs.set(match[2], match[1]); + } + const copies = files.map((path) => join(candidate, ...path.split('/'))); + // `hash-object` follows a symbolic link, so each copy is first proven an ordinary, bounded file. + for (const copy of copies) readOrdinaryFile(copy, CANDIDATE_LIMITS.fileBytes); + for (const batch of hashObjectBatches(copies)) { + const ids = await git(['hash-object', '--no-filters', '--', ...batch.paths]); + const names = ids.split('\n').filter((line) => line !== ''); + if (names.length !== batch.paths.length) return false; + for (const [offset, id] of names.entries()) { + if (blobs.get(files[batch.start + offset]) !== id) return false; + } + } + return true; +} + +/** Windows caps one command line at 32,767 characters; a batch of paths stays well inside it. */ +const HASH_OBJECT_BATCH_CHARACTERS = 16 * 1024; + +/** + * Consecutive batches of paths for one `git hash-object` each, with the index of each batch's first path. + * @param {readonly string[]} paths + * @returns {{start: number, paths: string[]}[]} + */ +function hashObjectBatches(paths) { + const batches = []; + let current = { start: 0, paths: [] }; + let characters = 0; + for (const [index, path] of paths.entries()) { + if (current.paths.length > 0 && characters + path.length + 1 > HASH_OBJECT_BATCH_CHARACTERS) { + batches.push(current); + current = { start: index, paths: [] }; + characters = 0; + } + current.paths.push(path); + characters += path.length + 1; + } + if (current.paths.length > 0) batches.push(current); + return batches; +} + +/** + * Install the locked closure into the candidate from the store: offline, frozen and without any + * lifecycle script, because building or packing the CLI never loads a native module. + * @param {{candidate: string, store: string, env: Record, pnpm: typeof runPnpm}} options + */ +export async function installCandidateDependencies({ candidate, store, env, pnpm }) { + await pnpm( + ['install', '--offline', '--frozen-lockfile', '--ignore-scripts', '--store-dir', store], + { cwd: candidate, env }, + ); +} + +/** + * Every workspace link must resolve inside the candidate, never to the original checkout whose + * `dist` is map-free and possibly stale. + * @param {string} candidate + */ +export function verifyCandidateLinks(candidate) { + const root = realpathSync(candidate); + for (const name of ENGINE_PACKAGES) { + const link = join(root, 'apps', 'cli', 'node_modules', '@relavium', name); + let info; + try { + info = lstatSync(link); + } catch { + assert.fail('PACKED_CLI_CANDIDATE_LINK'); + } + assert.ok(info.isSymbolicLink(), 'PACKED_CLI_CANDIDATE_LINK'); + assert.equal(realpathSync(link), join(root, 'packages', name), 'PACKED_CLI_CANDIDATE_LINK'); + } + for (const workspace of [ + join('apps', 'cli'), + ...ENGINE_PACKAGES.map((name) => join('packages', name)), + ]) { + const scope = join(root, workspace, 'node_modules', '@relavium'); + if (!existsSync(scope)) continue; + for (const name of readdirSync(scope)) { + assert.ok(ENGINE_PACKAGES.includes(name), 'PACKED_CLI_CANDIDATE_LINK'); + assert.equal( + realpathSync(join(scope, name)), + join(root, 'packages', name), + 'PACKED_CLI_CANDIDATE_LINK', + ); + } + } + assert.ok(within(root, realpathSync(join(root, 'node_modules'))), 'PACKED_CLI_CANDIDATE_LINK'); +} + +/** + * No build cache may exist in the candidate after its build: Turbo state, TypeScript incremental + * files or a tool cache would let a later build read candidate output or a broad capture keep it. + * Dependency directories themselves are installation state and are not walked. + * @param {string} candidate + */ +export function assertNoBuildCache(candidate) { + const found = []; + function visit(directory, prefix) { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const path = prefix === '' ? entry.name : `${prefix}/${entry.name}`; + if (entry.name === '.turbo' || entry.name.endsWith('.tsbuildinfo')) found.push(path); + else if (entry.isDirectory() && entry.name === 'node_modules') { + if (existsSync(join(directory, entry.name, '.cache'))) found.push(`${path}/.cache`); + } else if (entry.isDirectory()) visit(join(directory, entry.name), path); + } + } + visit(candidate, ''); + assert.deepEqual(found, [], 'PACKED_CLI_CANDIDATE_BUILD_CACHE'); +} + +/** + * Prepare the owned candidate: capture the original checkout's identity, copy its listed files, + * refuse if that identity changed during the copy, install the locked closure and verify every + * workspace link. The returned identity is what the candidate is later sealed with. + * @param {{repository: string, candidate: string, environment: Record, + * home: string, env: Record, excludes?: string, run?: typeof runCommand, + * pnpm?: typeof runPnpm}} options + * `env` is the isolated environment of the private `home`; `excludes` is the developer's global + * Git excludes file (`globalExcludesFile`). + */ +export async function prepareCandidate({ + repository, + candidate, + environment, + home, + env, + excludes = undefined, + run = runCommand, + pnpm = runPnpm, +}) { + const identity = await sourceIdentity(repository, run, excludes); + const files = await candidateFiles(repository, run, excludes); + const copied = copyCandidateTree({ repository, destination: candidate, files }); + const after = await sourceIdentity(repository, run, excludes); + assert.ok( + after.source === identity.source && + after.status === identity.status && + isDeepStrictEqual(after.unchecked, identity.unchecked), + 'PACKED_CLI_SOURCE_CHANGED_DURING_COPY', + ); + // A clean status is confirmed byte for byte: the candidate is built from the copied bytes. + const dirty = + identity.dirty || + !(await copyMatchesCommit({ repository, candidate, source: identity.source, files, run })); + const store = await dependencyStore({ repository, environment, home, pnpm }); + await installCandidateDependencies({ candidate, store, env, pnpm }); + verifyCandidateLinks(candidate); + return { source: identity.source, dirty, files: copied.files }; +} diff --git a/tools/cli-package/candidate.test.mjs b/tools/cli-package/candidate.test.mjs new file mode 100644 index 00000000..4f73f0af --- /dev/null +++ b/tools/cli-package/candidate.test.mjs @@ -0,0 +1,1409 @@ +import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; +import { createHash } from 'node:crypto'; +import { + chmodSync, + closeSync, + existsSync, + ftruncateSync, + openSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + renameSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, relative } from 'node:path'; +import { after, test } from 'node:test'; +import { isDeepStrictEqual } from 'node:util'; +import { + assertNoBuildCache, + candidateFiles, + copyCandidateTree, + copyMatchesCommit, + dependencyStore, + globalExcludesFile, + installCandidateDependencies, + prepareCandidate, + producerEnvironment, + sourceIdentity, + verifyCandidateLinks, + CANDIDATE_LIMITS, +} from './candidate.mjs'; +import { isolatedGitEnvironment, runCommand } from './process.mjs'; + +const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-candidate-tree-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); +const posixModes = process.platform !== 'win32'; +const linkKind = process.platform === 'win32' ? 'junction' : 'dir'; +let count = 0; + +function fresh(name) { + count += 1; + const path = join(scratch, `${name}-${count}`); + mkdirSync(path); + return path; +} + +function write(path, contents, mode) { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, contents); + if (mode !== undefined) chmodSync(path, mode); +} + +/** A fake `git` port answering from a table of exact argument lists. */ +function git(answers) { + const calls = []; + const run = async (command, args, options) => { + calls.push({ command, args, cwd: options.cwd, env: options.env }); + const key = args.join(' '); + assert.ok(Object.hasOwn(answers, key), `unexpected git ${key}`); + return { stdout: answers[key], stderr: '' }; + }; + return { run, calls }; +} + +const SHA = 'a'.repeat(40); +/** The explicit attributes and excludes arguments of a checkout-reading status or listing. */ +const READ = (excludes = '/dev/null') => + `-c core.attributesFile=/dev/null -c core.excludesFile=${excludes} `; + +test('source identity records the exact commit and the complete porcelain status', async (t) => { + // A credential in the caller's environment must not reach the isolated Git. + const previous = process.env.NPM_TOKEN; + process.env.NPM_TOKEN = 'synthetic-secret'; + t.after(() => { + if (previous === undefined) delete process.env.NPM_TOKEN; + else process.env.NPM_TOKEN = previous; + }); + const clean = git({ + '--no-replace-objects rev-parse HEAD': `${SHA}\n`, + [`${READ()}status --porcelain=v1 -z --untracked-files=all`]: '', + 'ls-files -v -z': 'H a.ts\0H b.ts\0', + }); + assert.deepEqual(await sourceIdentity('/repository', clean.run), { + source: SHA, + status: '', + unchecked: [], + dirty: false, + }); + assert.ok(clean.calls.every((call) => call.command === 'git' && call.cwd === '/repository')); + // The original checkout is queried with an isolated environment without any home, global or + // system configuration, never the caller's. + assert.ok( + clean.calls.every( + (call) => isDeepStrictEqual(call.env, isolatedGitEnvironment()) && !('NPM_TOKEN' in call.env), + ), + ); + const modified = git({ + '--no-replace-objects rev-parse HEAD': `${SHA}\n`, + [`${READ()}status --porcelain=v1 -z --untracked-files=all`]: '?? notes.txt\0', + 'ls-files -v -z': 'H a.ts\0', + }); + assert.equal((await sourceIdentity('/repository', modified.run)).dirty, true); + // A file Git is told not to check — skip-worktree (S) or assume-unchanged (a lowercase tag) — is + // copied unchecked, so it counts as dirty even when the status is empty. + const unchecked = git({ + '--no-replace-objects rev-parse HEAD': `${SHA}\n`, + [`${READ()}status --porcelain=v1 -z --untracked-files=all`]: '', + 'ls-files -v -z': 'H a.ts\0S sparse.ts\0h assumed.ts\0s both.ts\0M merged.ts\0', + }); + assert.deepEqual(await sourceIdentity('/repository', unchecked.run), { + source: SHA, + status: '', + unchecked: ['sparse.ts', 'assumed.ts', 'both.ts'], + dirty: true, + }); + // The developer's global excludes file judges untracked files. + const excluded = git({ + '--no-replace-objects rev-parse HEAD': `${SHA}\n`, + [`${READ('/home/someone/.config/git/ignore')}status --porcelain=v1 -z --untracked-files=all`]: + '', + 'ls-files -v -z': '', + }); + assert.equal( + (await sourceIdentity('/repository', excluded.run, '/home/someone/.config/git/ignore')).dirty, + false, + ); + for (const head of ['', 'HEAD\n', `${SHA.slice(1)}\n`, `${SHA}${SHA}\n`]) { + const malformed = git({ + '--no-replace-objects rev-parse HEAD': head, + [`${READ()}status --porcelain=v1 -z --untracked-files=all`]: '', + 'ls-files -v -z': '', + }); + await assert.rejects(sourceIdentity('/repository', malformed.run), /SOURCE_IDENTITY/u); + } +}); + +test('candidate files are tracked plus unignored untracked files, minus deletions, sorted once', async () => { + const listing = git({ + [`${READ()}ls-files -z --cached --others --exclude-standard`]: + 'b.ts\0a.ts\0gone.ts\0a.ts\0new.ts\0', + 'ls-files -z --deleted': 'gone.ts\0', + }); + assert.deepEqual(await candidateFiles('/repository', listing.run), ['a.ts', 'b.ts', 'new.ts']); + assert.ok(listing.calls.every((call) => isDeepStrictEqual(call.env, isolatedGitEnvironment()))); + const empty = git({ + [`${READ()}ls-files -z --cached --others --exclude-standard`]: '', + 'ls-files -z --deleted': '', + }); + await assert.rejects(candidateFiles('/repository', empty.run), /CANDIDATE_FILES/u); + const excluded = git({ + [`${READ('/ignore')}ls-files -z --cached --others --exclude-standard`]: 'a.ts\0', + 'ls-files -z --deleted': '', + }); + assert.deepEqual(await candidateFiles('/repository', excluded.run, '/ignore'), ['a.ts']); + // An untracked nested repository is listed only as its directory and refuses by name. + const nested = git({ + [`${READ()}ls-files -z --cached --others --exclude-standard`]: 'a.ts\0vendor/clone/\0', + 'ls-files -z --deleted': '', + }); + await assert.rejects( + candidateFiles('/repository', nested.run), + /PACKED_CLI_CANDIDATE_NESTED_REPOSITORY/u, + ); +}); + +test("the developer's global excludes file is located without the rest of the global configuration", async () => { + const home = fresh('developer-home'); + const ignore = join(home, '.config', 'git', 'ignore'); + write(ignore, 'secret.local\n'); + const calls = []; + const unset = async (command, args, options) => { + calls.push({ args, env: options.env, cwd: options.cwd }); + const error = new Error('PACKED_CLI_CHILD_FAILED: 1'); + error.result = { code: 1, stdout: '', stderr: '' }; + throw error; + }; + // Every variable Git uses to locate its configuration is forwarded, and nothing else. + const locations = { + HOME: home, + USERPROFILE: home, + XDG_CONFIG_HOME: join(home, 'xdg-absent'), + GIT_CONFIG_GLOBAL: join(home, 'global-config'), + GIT_CONFIG_SYSTEM: join(home, 'system-config'), + GIT_CONFIG_NOSYSTEM: '1', + // Command-scope configuration the developer's Git takes from the environment. + GIT_CONFIG_COUNT: '1', + GIT_CONFIG_KEY_0: 'core.excludesFile', + GIT_CONFIG_VALUE_0: '/env/ignore', + GIT_CONFIG_PARAMETERS: "'core.quotepath'='false'", + SystemRoot: 'C:\\Windows', + WINDIR: 'C:\\Windows', + }; + const environment = { + PATH: '/usr/bin', + ...locations, + NPM_TOKEN: 'synthetic-secret', + NODE_OPTIONS: '--require=/tmp/preload.cjs', + GIT_DIR: '/elsewhere', + GIT_CONFIG_KEY_X: 'not an entry', + }; + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { ...environment, XDG_CONFIG_HOME: '' }, + run: unset, + }), + ignore, + ); + assert.deepEqual(calls[0].args, ['config', '--path', '--get', 'core.excludesFile']); + // It reads the checkout's own configuration as the developer's Git there would. + assert.equal(calls[0].cwd, '/repository'); + assert.deepEqual(calls[0].env, { PATH: '/usr/bin', ...locations, XDG_CONFIG_HOME: '' }); + await globalExcludesFile({ repository: '/repository', environment, run: unset }); + assert.deepEqual(calls[1].env, { PATH: '/usr/bin', ...locations }); + // With no PATH the lookup gets no PATH either, never an empty one that names the checkout. + await globalExcludesFile({ repository: '/repository', environment: { HOME: home }, run: unset }); + assert.equal('PATH' in calls.at(-1).env, false); + // Without HOME, Git's default uses USERPROFILE; with both, HOME wins. + const profile = fresh('profile'); + write(join(profile, '.config', 'git', 'ignore'), '*.profile\n'); + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { USERPROFILE: profile }, + run: unset, + }), + join(profile, '.config', 'git', 'ignore'), + ); + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { HOME: home, USERPROFILE: profile }, + run: unset, + }), + ignore, + ); + // An empty HOME names no default, not one relative to this process's working directory. + const cwd = process.cwd(); + const relativeDefault = fresh('relative-default'); + write(join(relativeDefault, '.config', 'git', 'ignore'), '*.relative\n'); + process.chdir(relativeDefault); + try { + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { HOME: '' }, + run: unset, + }), + undefined, + ); + } finally { + process.chdir(cwd); + } + // Git's default prefers XDG_CONFIG_HOME, and a missing file means no global excludes. + const xdg = fresh('xdg'); + write(join(xdg, 'git', 'ignore'), '*.local\n'); + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { HOME: home, XDG_CONFIG_HOME: xdg }, + run: unset, + }), + join(xdg, 'git', 'ignore'), + ); + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { HOME: fresh('empty-home') }, + run: unset, + }), + undefined, + ); + // A configured core.excludesFile wins, resolved against the repository when relative. + const configured = join(fresh('configured'), 'ignore-file'); + write(configured, '*.secret\n'); + const answer = (stdout) => async () => ({ stdout, stderr: '' }); + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { HOME: home }, + run: answer(`${configured}\n`), + }), + configured, + ); + const repository = fresh('relative-repository'); + write(join(repository, 'local-ignore'), '*.secret\n'); + assert.equal( + await globalExcludesFile({ + repository, + environment: { HOME: home }, + run: answer('local-ignore\n'), + }), + join(repository, 'local-ignore'), + ); + // A configured path that is a directory names no excludes file. + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { HOME: home }, + run: answer(`${fresh('directory-excludes')}\n`), + }), + undefined, + ); + // A setting present but empty disables Git's global excludes: the default file no longer applies. + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { HOME: home }, + run: answer('\n'), + }), + undefined, + ); + // Any other Git failure refuses, and an explicit file from an isolating caller must exist. + const broken = async () => { + const error = new Error('PACKED_CLI_CHILD_FAILED: 128'); + error.result = { code: 128, stdout: '', stderr: '' }; + throw error; + }; + await assert.rejects( + globalExcludesFile({ repository: '/repository', environment: { HOME: home }, run: broken }), + /CHILD_FAILED/u, + ); + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { RELAVIUM_GIT_EXCLUDES_FILE: ignore }, + run: broken, + }), + ignore, + ); + // A relative explicit file refuses even when it exists from this process's working directory. + for (const bad of ['relative/ignore', 'package.json', join(home, 'absent-ignore')]) { + await assert.rejects( + globalExcludesFile({ + repository: '/repository', + environment: { RELAVIUM_GIT_EXCLUDES_FILE: bad }, + run: broken, + }), + /GIT_EXCLUDES/u, + ); + } +}); + +test('a real checkout copies nothing its developer ignores globally', async () => { + const repository = fresh('real-checkout'); + // The fixture and the lookup both run without the host's system configuration. + const inRepository = (args) => + runCommand('git', args, { cwd: repository, env: isolatedGitEnvironment(), timeoutMs: 30_000 }); + await inRepository(['init', '--quiet']); + write(join(repository, 'kept.ts'), 'export {};\n'); + write(join(repository, 'secret.local'), 'private\n'); + await inRepository(['add', 'kept.ts']); + await inRepository([ + '-c', + 'user.name=t', + '-c', + 'user.email=t@example.invalid', + 'commit', + '--quiet', + '-m', + 'one', + ]); + const home = fresh('developer-home'); + write(join(home, '.config', 'git', 'ignore'), 'secret.local\n'); + const excludes = await globalExcludesFile({ + repository, + environment: { PATH: process.env.PATH, HOME: home, GIT_CONFIG_NOSYSTEM: '1' }, + }); + assert.equal(excludes, join(home, '.config', 'git', 'ignore')); + assert.deepEqual(await candidateFiles(repository, runCommand, excludes), ['kept.ts']); + assert.equal((await sourceIdentity(repository, runCommand, excludes)).dirty, false); + // Without it the isolated Git sees the globally ignored file as untracked. + assert.deepEqual(await candidateFiles(repository), ['kept.ts', 'secret.local']); + assert.equal((await sourceIdentity(repository)).dirty, true); +}); + +test('a real developer configuration with an empty core.excludesFile applies no global excludes', async () => { + const home = fresh('developer-home'); + write(join(home, '.gitconfig'), '[core]\n\texcludesFile =\n'); + write(join(home, '.config', 'git', 'ignore'), 'secret.local\n'); + assert.equal( + await globalExcludesFile({ + repository: fresh('repository'), + environment: { PATH: process.env.PATH, HOME: home, GIT_CONFIG_NOSYSTEM: '1' }, + }), + undefined, + ); +}); + +for (const [label, place] of [ + [ + 'an XDG configuration directory', + (root, ignore) => { + write(join(root, 'xdg', 'git', 'config'), `[core]\n\texcludesFile = ${ignore}\n`); + return { XDG_CONFIG_HOME: join(root, 'xdg') }; + }, + ], + [ + 'a GIT_CONFIG_GLOBAL file', + (root, ignore) => { + write(join(root, 'dotfiles', 'gitconfig'), `[core]\n\texcludesFile = ${ignore}\n`); + return { GIT_CONFIG_GLOBAL: join(root, 'dotfiles', 'gitconfig') }; + }, + ], +]) + test(`a real core.excludesFile set only in ${label} is found`, async () => { + const root = fresh('developer'); + const ignore = join(root, 'my-ignore'); + write(ignore, 'secret.local\n'); + mkdirSync(join(root, 'empty-home')); + assert.equal( + await globalExcludesFile({ + repository: fresh('repository'), + environment: { + PATH: process.env.PATH, + HOME: join(root, 'empty-home'), + GIT_CONFIG_NOSYSTEM: '1', + ...place(root, ignore), + }, + }), + ignore, + ); + }); + +/** A real repository with one committed file, `kept.ts`, created with Git's own configuration off. */ +async function committedRepository(name) { + const repository = fresh(name); + const inRepository = (args) => + runCommand('git', args, { cwd: repository, env: isolatedGitEnvironment(), timeoutMs: 30_000 }); + await inRepository(['init', '--quiet']); + write(join(repository, 'kept.ts'), 'export {};\n'); + await inRepository(['add', 'kept.ts']); + await inRepository([ + '-c', + 'user.name=t', + '-c', + 'user.email=t@example.invalid', + 'commit', + '--quiet', + '-m', + 'one', + ]); + return { repository, inRepository }; +} + +test("no configuration found inside the checkout acts as Git's global configuration", async () => { + const { repository } = await committedRepository('planted-configuration'); + write(join(repository, 'secret.local'), 'private\n'); + // A home-style .gitconfig and an XDG-style git/ignore at the checkout root, untracked, each of + // which would hide files if Git took the checkout for its home or configuration directory. + write(join(repository, 'hide'), 'secret.local\n.gitconfig\nhide\ngit/\n'); + write(join(repository, '.gitconfig'), `[core]\n\texcludesFile = ${join(repository, 'hide')}\n`); + write(join(repository, 'git', 'ignore'), 'secret.local\ngit/\nhide\n.gitconfig\n'); + assert.deepEqual(await candidateFiles(repository), [ + '.gitconfig', + 'git/ignore', + 'hide', + 'kept.ts', + 'secret.local', + ]); + assert.equal((await sourceIdentity(repository)).dirty, true); +}); + +const COMMIT = ['-c', 'user.name=t', '-c', 'user.email=t@example.invalid']; + +for (const ignoredBy of ['.gitignore', '.git/info/exclude']) + for (const linked of ['a', 'a/b']) + test( + `a tracked directory ${linked} replaced by a link ignored through ${ignoredBy} refuses the copy`, + { skip: !posixModes }, + async () => { + const { repository, inRepository } = await committedRepository('linked-parent'); + write(join(repository, 'a', 'b', 'c'), 'tracked\n'); + await inRepository(['add', 'a/b/c']); + await inRepository([...COMMIT, 'commit', '--quiet', '-m', 'two']); + // The directory moves outside the checkout and comes back as a link that Git ignores; a link + // above the listed file's own directory must refuse as well as its direct parent. + const outside = join(fresh('outside'), 'moved'); + renameSync(join(repository, ...linked.split('/')), outside); + write(join(outside, ...'a/b/c'.slice(linked.length + 1).split('/')), 'outside bytes\n'); + symlinkSync(outside, join(repository, ...linked.split('/')), linkKind); + write(join(repository, ...ignoredBy.split('/')), `/${linked}\n`); + const files = await candidateFiles(repository); + assert.ok(files.includes('a/b/c')); + const destination = fresh('copy'); + assert.throws( + () => copyCandidateTree({ repository, destination, files }), + /CANDIDATE_PATH/u, + ); + assert.equal(existsSync(join(destination, 'a', 'b', 'c')), false); + }, + ); + +test('a replace ref on HEAD cannot hide a substituted tree from the identity', async () => { + const { repository, inRepository } = await committedRepository('replaced'); + const reviewed = (await inRepository(['rev-parse', 'HEAD'])).stdout.trim(); + // An unrelated commit with other bytes, then a replace ref that substitutes it for HEAD and an + // index and working tree that match it, as `git reset --hard` under that ref would leave them. + write(join(repository, 'kept.ts'), 'export const substituted = true;\n'); + await inRepository(['add', 'kept.ts']); + const tree = (await inRepository(['write-tree'])).stdout.trim(); + const other = (await inRepository([...COMMIT, 'commit-tree', tree, '-m', 'other'])).stdout.trim(); + await inRepository(['replace', reviewed, other]); + await inRepository(['read-tree', '--reset', '-u', other]); + // Git honouring the replace ref sees a clean tree; the identity must not. + const honouring = Object.fromEntries( + Object.entries(isolatedGitEnvironment()).filter(([key]) => key !== 'GIT_NO_REPLACE_OBJECTS'), + ); + const status = await runCommand('git', ['status', '--porcelain=v1'], { + cwd: repository, + env: honouring, + }); + assert.equal(status.stdout, ''); + const identity = await sourceIdentity(repository); + assert.equal(identity.source, reviewed); + assert.match(identity.status, /kept\.ts/u); + assert.equal(identity.dirty, true); +}); + +for (const flag of ['--skip-worktree', '--assume-unchanged']) + test(`a tracked file marked ${flag} makes the identity dirty`, async () => { + const { repository, inRepository } = await committedRepository('unchecked'); + await inRepository(['update-index', flag, 'kept.ts']); + write(join(repository, 'kept.ts'), 'export const changed = true;\n'); + // Git's own status cannot see the change, and the copy would take the changed bytes. + assert.equal((await inRepository(['status', '--porcelain=v1'])).stdout, ''); + assert.deepEqual(await candidateFiles(repository), ['kept.ts']); + const identity = await sourceIdentity(repository); + assert.deepEqual(identity.unchecked, ['kept.ts']); + assert.equal(identity.dirty, true); + }); + +test('the candidate copy is byte-exact, keeps only the executable bit and refuses links', () => { + const repository = fresh('repository'); + write(join(repository, 'package.json'), '{"name":"root"}\n'); + write(join(repository, 'tools', 'run.mjs'), '#!/usr/bin/env node\n', 0o775); + write(join(repository, 'docs', 'note.md'), 'note\n', 0o640); + // Group and other execute bits alone do not make a file executable, as in Git's index. + write(join(repository, 'docs', 'group.sh'), 'x\n', 0o654); + const destination = fresh('copy'); + const files = ['docs/group.sh', 'docs/note.md', 'package.json', 'tools/run.mjs']; + assert.deepEqual(copyCandidateTree({ repository, destination, files }), { files: 4, bytes: 43 }); + for (const path of files) { + assert.ok(readFileSync(join(destination, path)).equals(readFileSync(join(repository, path)))); + } + if (posixModes) { + assert.equal(statSync(join(destination, 'tools', 'run.mjs')).mode & 0o777, 0o755); + assert.equal(statSync(join(destination, 'docs', 'note.md')).mode & 0o777, 0o644); + assert.equal(statSync(join(destination, 'docs', 'group.sh')).mode & 0o777, 0o644); + } + if (posixModes) { + // The modes are exact whatever the process umask. + const previous = process.umask(0o077); + let restrictive; + try { + restrictive = fresh('copy'); + copyCandidateTree({ repository, destination: restrictive, files }); + } finally { + process.umask(previous); + } + assert.equal(statSync(join(restrictive, 'tools', 'run.mjs')).mode & 0o777, 0o755); + assert.equal(statSync(join(restrictive, 'docs', 'note.md')).mode & 0o777, 0o644); + } + assert.throws( + () => copyCandidateTree({ repository, destination, files }), + /CANDIDATE_ROOT/u, + 'a non-empty destination refuses', + ); + for (const path of ['../escape.ts', 'a/../b.ts', './a.ts', 'a//b.ts', 'a\\b.ts', '']) { + assert.throws( + () => copyCandidateTree({ repository, destination: fresh('copy'), files: [path] }), + /CANDIDATE_PATH/u, + ); + } + if (posixModes) { + // A link as a parent directory refuses too, not only as the file itself. + const elsewhere = fresh('elsewhere'); + write(join(elsewhere, 'inner.ts'), 'outside\n'); + write(join(elsewhere, 'sub', 'inner.ts'), 'outside\n'); + symlinkSync(elsewhere, join(repository, 'linked-directory'), linkKind); + mkdirSync(join(repository, 'real')); + symlinkSync(elsewhere, join(repository, 'real', 'linked'), linkKind); + // The link may be the file's direct parent, a directory above it or a directory below the root. + for (const path of [ + 'linked-directory/inner.ts', + 'linked-directory/sub/inner.ts', + 'real/linked/inner.ts', + ]) { + assert.throws( + () => copyCandidateTree({ repository, destination: fresh('copy'), files: [path] }), + /CANDIDATE_PATH/u, + path, + ); + } + symlinkSync(join(repository, 'package.json'), join(repository, 'linked.json')); + assert.throws( + () => copyCandidateTree({ repository, destination: fresh('copy'), files: ['linked.json'] }), + /FILE_KIND_OR_BOUND/u, + ); + } +}); + +test('the dependency store is discovered with store-location variables only', async () => { + const store = fresh('store'); + const calls = []; + const pnpm = async (args, options) => { + calls.push({ args, options }); + return { stdout: `${store}\n`, stderr: '' }; + }; + const environment = { + HOME: '/home/someone', + XDG_DATA_HOME: '/home/someone/.local/share', + NPM_TOKEN: 'synthetic-secret', + NODE_OPTIONS: '--require=/tmp/preload.cjs', + npm_config_registry: 'https://registry.invalid/', + }; + const home = fresh('private-home'); + assert.equal( + await dependencyStore({ repository: '/repository', environment, home, pnpm }), + store, + ); + assert.deepEqual(calls[0].args, ['store', 'path']); + const env = calls[0].options.env; + assert.equal(env.HOME, '/home/someone'); + assert.equal(env.XDG_DATA_HOME, '/home/someone/.local/share'); + // Temporary files and caches (Node's compile cache, corepack) never land in the checkout. + for (const key of ['TMPDIR', 'TMP', 'TEMP', 'XDG_CACHE_HOME']) assert.equal(env[key], home, key); + for (const key of [ + 'NPM_TOKEN', + 'NODE_OPTIONS', + 'npm_config_registry', + 'USERPROFILE', + 'XDG_CONFIG_HOME', + ]) { + assert.equal(key in env, false, key); + } + // Every store-location variable reaches the discovery, which runs in the original checkout. + const locations = { + HOME: '/l/home', + USERPROFILE: '/l/profile', + XDG_DATA_HOME: '/l/data', + XDG_CONFIG_HOME: '/l/config', + LOCALAPPDATA: '/l/local', + APPDATA: '/l/appdata', + PNPM_HOME: '/l/pnpm', + npm_config_store_dir: '/l/store-lower', + NPM_CONFIG_STORE_DIR: '/l/store-upper', + }; + await dependencyStore({ + repository: '/repository', + environment: { ...locations, NPM_TOKEN: 'synthetic-secret' }, + home, + pnpm, + }); + assert.equal(calls[1].options.cwd, '/repository'); + for (const [key, value] of Object.entries(locations)) + assert.equal(calls[1].options.env[key], value, key); + assert.equal('NPM_TOKEN' in calls[1].options.env, false); + // A caller that already isolated this process names the store explicitly; pnpm is not asked. + const explicit = fresh('explicit-store'); + assert.equal( + await dependencyStore({ + repository: '/repository', + environment: { RELAVIUM_PACKAGE_STORE_DIR: explicit }, + home, + pnpm: async () => { + throw new Error('store discovery must not run'); + }, + }), + explicit, + ); + // A relative store refuses even when it exists from this process's working directory. + for (const bad of ['relative/store', 'tools', join(scratch, 'absent-store')]) { + await assert.rejects( + dependencyStore({ + repository: '/repository', + environment: { RELAVIUM_PACKAGE_STORE_DIR: bad }, + home, + pnpm, + }), + /DEPENDENCY_STORE/u, + ); + } + for (const answer of ['relative/store\n', 'tools\n', `${join(scratch, 'absent-store')}\n`]) { + await assert.rejects( + dependencyStore({ + repository: '/repository', + environment, + home, + pnpm: async () => ({ stdout: answer, stderr: '' }), + }), + /DEPENDENCY_STORE/u, + ); + } +}); + +test('store discovery refuses a private home inside the original checkout', async () => { + const repository = fresh('checkout'); + // A child whose name merely starts with two dots is inside the checkout too. + for (const home of [join(repository, 'home'), join(repository, '..home'), 'relative-home']) { + await assert.rejects( + dependencyStore({ + repository, + environment: { HOME: '/home/someone' }, + home, + pnpm: async () => { + throw new Error('store discovery must not run'); + }, + }), + /PRIVATE_HOME/u, + home, + ); + } +}); + +test('the candidate installs offline, frozen and without lifecycle scripts', async () => { + const calls = []; + await installCandidateDependencies({ + candidate: '/candidate', + store: '/store', + env: { HOME: '/home' }, + pnpm: async (args, options) => { + calls.push({ args, options }); + return { stdout: '', stderr: '' }; + }, + }); + assert.deepEqual(calls, [ + { + args: [ + 'install', + '--offline', + '--frozen-lockfile', + '--ignore-scripts', + '--store-dir', + '/store', + ], + options: { cwd: '/candidate', env: { HOME: '/home' } }, + }, + ]); +}); + +function linkedCandidate() { + const root = fresh('candidate'); + mkdirSync(join(root, 'node_modules')); + for (const name of ['shared', 'llm', 'core', 'db', 'mcp']) { + mkdirSync(join(root, 'packages', name), { recursive: true }); + } + const scope = join(root, 'apps', 'cli', 'node_modules', '@relavium'); + mkdirSync(scope, { recursive: true }); + for (const name of ['shared', 'llm', 'core', 'db', 'mcp']) { + symlinkSync(join(root, 'packages', name), join(scope, name), linkKind); + } + return root; +} + +test('every workspace link must resolve inside the candidate', () => { + const root = linkedCandidate(); + verifyCandidateLinks(root); + const original = fresh('original-checkout'); + mkdirSync(join(original, 'core')); + const link = join(root, 'apps', 'cli', 'node_modules', '@relavium', 'core'); + rmSync(link); + symlinkSync(join(original, 'core'), link, linkKind); + assert.throws(() => verifyCandidateLinks(root), /CANDIDATE_LINK/u); + + const nested = linkedCandidate(); + const coreScope = join(nested, 'packages', 'core', 'node_modules', '@relavium'); + mkdirSync(coreScope, { recursive: true }); + symlinkSync(join(original, 'core'), join(coreScope, 'shared'), linkKind); + assert.throws(() => verifyCandidateLinks(nested), /CANDIDATE_LINK/u); + + const missing = linkedCandidate(); + rmSync(join(missing, 'apps', 'cli', 'node_modules', '@relavium', 'mcp')); + assert.throws(() => verifyCandidateLinks(missing), /CANDIDATE_LINK/u); + + // The root node_modules must not resolve into the original checkout either. + const borrowed = linkedCandidate(); + rmSync(join(borrowed, 'node_modules'), { recursive: true }); + mkdirSync(join(original, 'node_modules')); + symlinkSync(join(original, 'node_modules'), join(borrowed, 'node_modules'), linkKind); + assert.throws(() => verifyCandidateLinks(borrowed), /CANDIDATE_LINK/u); +}); + +test('a candidate build leaves no build cache behind', () => { + const root = fresh('built'); + write(join(root, 'packages', 'core', 'dist', 'index.js'), 'export {};\n'); + write(join(root, 'node_modules', 'left-pad', 'index.js'), 'module.exports = 1;\n'); + assertNoBuildCache(root); + for (const marker of [ + 'packages/core/tsconfig.build.tsbuildinfo', + '.turbo/cache/abc.tar.zst', + 'apps/cli/.turbo/turbo-build.log', + 'node_modules/.cache/tool/state', + 'apps/cli/node_modules/.cache/state', + ]) { + const candidate = fresh('built'); + write(join(candidate, 'packages', 'core', 'dist', 'index.js'), 'export {};\n'); + write(join(candidate, marker), 'cache\n'); + assert.throws(() => assertNoBuildCache(candidate), /CANDIDATE_BUILD_CACHE/u, marker); + } + assert.deepEqual(readdirSync(root).sort(), ['node_modules', 'packages']); +}); + +/** A source repository plus fake git and pnpm ports; the fake install creates the workspace links. */ +/** Git's blob object id for raw bytes. */ +const blobId = (bytes) => + createHash('sha1').update(`blob ${bytes.length}\0`).update(bytes).digest('hex'); + +function preparation({ + status = '', + statusAfterCopy = status, + headAfterCopy = SHA, + unchecked = '', + uncheckedAfterCopy = unchecked, + excludes, + misLink, + convertedPath, +} = {}) { + const repository = fresh('source'); + write(join(repository, 'package.json'), '{}\n'); + for (const name of ['shared', 'llm', 'core', 'db', 'mcp']) { + write(join(repository, 'packages', name, 'package.json'), '{}\n'); + } + const files = [ + 'package.json', + ...['shared', 'llm', 'core', 'db', 'mcp'].map((name) => `packages/${name}/package.json`), + ]; + const candidate = fresh('candidate'); + const store = fresh('store'); + const home = fresh('home'); + const events = []; + let statuses = 0; + let heads = 0; + let flags = 0; + // With an excludes file, the listing and both status calls answer only when they carry it. + const prefix = READ(excludes); + const run = async (command, args, options) => { + const key = args.join(' '); + events.push(`git ${key}`); + if (key === '--no-replace-objects rev-parse HEAD') { + heads += 1; + return { stdout: `${heads === 1 ? SHA : headAfterCopy}\n` }; + } + if (key === `${prefix}status --porcelain=v1 -z --untracked-files=all`) { + statuses += 1; + return { stdout: statuses === 1 ? status : statusAfterCopy }; + } + if (key === `${prefix}ls-files -z --cached --others --exclude-standard`) + return { stdout: files.join('\0') }; + if (key === 'ls-files -z --deleted') return { stdout: '' }; + if (key === 'ls-files -v -z') { + flags += 1; + return { stdout: `H package.json\0${flags === 1 ? unchecked : uncheckedAfterCopy}` }; + } + // The commit's tree: each file's blob id, except `convertedPath`, whose committed blob differs + // from the working-tree bytes (as line-ending conversion leaves it while status stays clean). + if (key === `--no-replace-objects ls-tree -r -z --full-tree ${SHA}`) { + // Asked of the original checkout, through Git without home, global or system configuration. + assert.equal(options.cwd, repository); + assert.deepEqual(options.env, isolatedGitEnvironment()); + return { + stdout: files + .map((path) => { + const bytes = readFileSync(join(repository, ...path.split('/'))); + const committed = path === convertedPath ? Buffer.from('{}\r\n') : bytes; + return `100644 blob ${blobId(committed)}\t${path}`; + }) + .join('\0'), + }; + } + // Git names each copy's raw bytes, asked of the original checkout like the listing. + if (args[1] === 'hash-object') { + assert.deepEqual(args.slice(0, 4), [ + '--no-replace-objects', + 'hash-object', + '--no-filters', + '--', + ]); + assert.equal(options.cwd, repository); + assert.deepEqual(options.env, isolatedGitEnvironment()); + return { + stdout: args + .slice(4) + .map((path) => `${blobId(readFileSync(path))}\n`) + .join(''), + }; + } + throw new Error(`unexpected git ${key}`); + }; + let installOptions; + const pnpm = async (args, options) => { + events.push(`pnpm ${args.join(' ')}`); + if (args[0] === 'store') return { stdout: `${store}\n` }; + assert.equal(options.cwd, candidate); + installOptions = options; + // `misLink` makes the install resolve one engine package, or the root dependency directory, to + // the original checkout instead of the candidate. + if (misLink === 'root') + symlinkSync(fresh('borrowed-modules'), join(candidate, 'node_modules'), linkKind); + else mkdirSync(join(candidate, 'node_modules'), { recursive: true }); + const scope = join(candidate, 'apps', 'cli', 'node_modules', '@relavium'); + mkdirSync(scope, { recursive: true }); + for (const name of ['shared', 'llm', 'core', 'db', 'mcp']) { + const owner = misLink === 'engine' && name === 'core' ? repository : candidate; + symlinkSync(join(owner, 'packages', name), join(scope, name), linkKind); + } + return { stdout: '' }; + }; + return { + repository, + candidate, + store, + home, + events, + run, + pnpm, + installOptions: () => installOptions, + }; +} + +test('candidate preparation copies, re-checks identity, installs and verifies links in order', async () => { + const ports = preparation(); + const env = { HOME: '/isolated' }; + const identity = await prepareCandidate({ + repository: ports.repository, + candidate: ports.candidate, + environment: { + HOME: '/home/someone', + NPM_TOKEN: 'synthetic-secret', + NODE_OPTIONS: '--require=/x.cjs', + }, + home: ports.home, + env, + run: ports.run, + pnpm: ports.pnpm, + }); + // The install runs in the isolated environment it was given, never the caller's own. + assert.equal(ports.installOptions().env, env); + assert.deepEqual(identity, { source: SHA, dirty: false, files: 6 }); + assert.deepEqual(ports.events, [ + 'git --no-replace-objects rev-parse HEAD', + `git ${READ()}status --porcelain=v1 -z --untracked-files=all`, + 'git ls-files -v -z', + `git ${READ()}ls-files -z --cached --others --exclude-standard`, + 'git ls-files -z --deleted', + 'git --no-replace-objects rev-parse HEAD', + `git ${READ()}status --porcelain=v1 -z --untracked-files=all`, + 'git ls-files -v -z', + `git --no-replace-objects ls-tree -r -z --full-tree ${SHA}`, + `git --no-replace-objects hash-object --no-filters -- ${[ + 'package.json', + 'core', + 'db', + 'llm', + 'mcp', + 'shared', + ] + .map((name) => + name === 'package.json' + ? join(ports.candidate, name) + : join(ports.candidate, 'packages', name, 'package.json'), + ) + .join(' ')}`, + 'pnpm store path', + `pnpm install --offline --frozen-lockfile --ignore-scripts --store-dir ${ports.store}`, + ]); + assert.ok( + readFileSync(join(ports.candidate, 'package.json')).equals( + readFileSync(join(ports.repository, 'package.json')), + ), + ); +}); + +test('a copy that differs from the commit is dirty even when status reports the tree clean', async () => { + // Whichever copied file differs. + for (const convertedPath of [ + 'package.json', + ...['shared', 'llm', 'core', 'db', 'mcp'].map((name) => `packages/${name}/package.json`), + ]) { + const ports = preparation({ convertedPath }); + const identity = await prepareCandidate({ + repository: ports.repository, + candidate: ports.candidate, + environment: {}, + home: ports.home, + env: {}, + run: ports.run, + pnpm: ports.pnpm, + }); + assert.deepEqual(identity, { source: SHA, dirty: true, files: 6 }, convertedPath); + } + // An identity that status already calls dirty is not checked again. + const dirty = preparation({ status: ' M package.json\0', statusAfterCopy: ' M package.json\0' }); + await prepareCandidate({ + repository: dirty.repository, + candidate: dirty.candidate, + environment: {}, + home: dirty.home, + env: {}, + run: dirty.run, + pnpm: dirty.pnpm, + }); + assert.equal( + dirty.events.some((event) => event.includes('ls-tree')), + false, + ); +}); + +test('a checkout whose line endings Git converts copies bytes that are not the commit', async () => { + const { repository, inRepository } = await committedRepository('converted-line-endings'); + // core.autocrlf in the checkout's own configuration, which Git reading the checkout honours: + // the file is checked out with CRLF while its blob keeps LF, and status stays clean. + await inRepository(['config', 'core.autocrlf', 'true']); + rmSync(join(repository, 'kept.ts')); + await inRepository(['checkout', '--', 'kept.ts']); + assert.equal(readFileSync(join(repository, 'kept.ts'), 'utf8'), 'export {};\r\n'); + const identity = await sourceIdentity(repository); + assert.equal(identity.dirty, false); + const candidate = fresh('converted-copy'); + copyCandidateTree({ repository, destination: candidate, files: ['kept.ts'] }); + assert.equal( + await copyMatchesCommit({ repository, candidate, source: identity.source, files: ['kept.ts'] }), + false, + ); + // The committed bytes match; a file the commit does not hold does not. + writeFileSync(join(candidate, 'kept.ts'), 'export {};\n'); + assert.equal( + await copyMatchesCommit({ repository, candidate, source: identity.source, files: ['kept.ts'] }), + true, + ); + write(join(candidate, 'extra.ts'), 'export {};\n'); + assert.equal( + await copyMatchesCommit({ + repository, + candidate, + source: identity.source, + files: ['kept.ts', 'extra.ts'], + }), + false, + ); +}); + +test('every hash-object batch is compared, the last included', async () => { + const { repository, inRepository } = await committedRepository('hash-object-batches'); + // Enough long paths for several 16 KiB command lines, committed, then copied unchanged. + const files = Array.from( + { length: 240 }, + (_, index) => `nested/${String(index).padStart(3, '0')}-${'n'.repeat(90)}.ts`, + ); + for (const path of files) + write(join(repository, ...path.split('/')), `export const v = '${path}';\n`); + await inRepository(['add', 'nested']); + await inRepository([ + '-c', + 'user.name=t', + '-c', + 'user.email=t@example.invalid', + 'commit', + '-qm', + 'two', + ]); + const identity = await sourceIdentity(repository); + const candidate = fresh('batched-copy'); + copyCandidateTree({ repository, destination: candidate, files }); + const calls = []; + const run = (command, args, options) => { + if (args[1] === 'hash-object') calls.push(args.length - 4); + return runCommand(command, args, options); + }; + const matches = () => + copyMatchesCommit({ repository, candidate, source: identity.source, files, run }); + assert.equal(await matches(), true); + assert.ok(calls.length >= 2, `expected several batches, saw ${calls.length}`); + assert.equal( + calls.reduce((sum, count) => sum + count, 0), + files.length, + ); + // A change in the final batch's last file is still seen. + writeFileSync(join(candidate, ...files.at(-1).split('/')), 'export const v = 0;\n'); + assert.equal(await matches(), false); +}); + +test( + 'a copy that is a symbolic link to the committed bytes is refused, not hashed through', + { + skip: !posixModes, + }, + async () => { + const { repository } = await committedRepository('linked-copy'); + const identity = await sourceIdentity(repository); + const candidate = fresh('linked-copy-candidate'); + const elsewhere = join(fresh('linked-copy-target'), 'kept.ts'); + write(elsewhere, 'export {};\n'); + symlinkSync(elsewhere, join(candidate, 'kept.ts')); + await assert.rejects( + copyMatchesCommit({ repository, candidate, source: identity.source, files: ['kept.ts'] }), + /PACKED_CLI_FILE_KIND_OR_BOUND/, + ); + }, +); + +test('a hash-object answer missing a line is not a match', async () => { + const { repository } = await committedRepository('short-answer'); + const identity = await sourceIdentity(repository); + const candidate = fresh('short-answer-candidate'); + copyCandidateTree({ repository, destination: candidate, files: ['kept.ts'] }); + const run = async (command, args, options) => { + const result = await runCommand(command, args, options); + return args[1] === 'hash-object' ? { ...result, stdout: '' } : result; + }; + assert.equal( + await copyMatchesCommit({ + repository, + candidate, + source: identity.source, + files: ['kept.ts'], + run, + }), + false, + ); +}); + +test('a store the caller names reaches the install, and pnpm is never asked to discover one', async () => { + const ports = preparation(); + const explicit = fresh('explicit-store'); + const pnpm = async (args, options) => { + if (args[0] === 'store') throw new Error('store discovery must not run'); + return ports.pnpm(args, options); + }; + await prepareCandidate({ + repository: ports.repository, + candidate: ports.candidate, + environment: { RELAVIUM_PACKAGE_STORE_DIR: explicit }, + home: ports.home, + env: {}, + run: ports.run, + pnpm, + }); + assert.equal( + ports.events.at(-1), + `pnpm install --offline --frozen-lockfile --ignore-scripts --store-dir ${explicit}`, + ); +}); + +test('a modified or unchecked source makes the prepared identity dirty', async () => { + for (const source of [ + { status: ' M package.json\0' }, + { unchecked: 'S packages/core/package.json\0' }, + ]) { + const ports = preparation(source); + const identity = await prepareCandidate({ + repository: ports.repository, + candidate: ports.candidate, + environment: {}, + home: ports.home, + env: {}, + run: ports.run, + pnpm: ports.pnpm, + }); + assert.deepEqual(identity, { source: SHA, dirty: true, files: 6 }); + } +}); + +test('an installed link outside the candidate refuses after the install', async () => { + for (const misLink of ['engine', 'root']) { + const ports = preparation({ misLink }); + await assert.rejects( + prepareCandidate({ + repository: ports.repository, + candidate: ports.candidate, + environment: {}, + home: ports.home, + env: {}, + run: ports.run, + pnpm: ports.pnpm, + }), + /CANDIDATE_LINK/u, + ); + assert.match(ports.events.at(-1), /^pnpm install /u); + } +}); + +test('a source change during the copy refuses before anything is installed', async () => { + // A changed status, a commit that moved while the status stayed the same, or a file Git was told + // not to check during the copy. + for (const change of [ + { statusAfterCopy: ' M package.json\0' }, + { headAfterCopy: 'b'.repeat(40) }, + { uncheckedAfterCopy: 'h tools/run.mjs\0' }, + ]) { + const ports = preparation(change); + await assert.rejects( + prepareCandidate({ + repository: ports.repository, + candidate: ports.candidate, + environment: {}, + home: ports.home, + env: {}, + run: ports.run, + pnpm: ports.pnpm, + }), + /SOURCE_CHANGED_DURING_COPY/u, + ); + assert.equal( + ports.events.some((event) => event.startsWith('pnpm')), + false, + ); + } +}); + +test("candidate preparation applies the developer's global excludes to every listing and status", async () => { + const ports = preparation({ excludes: '/developer/ignore' }); + const identity = await prepareCandidate({ + repository: ports.repository, + candidate: ports.candidate, + environment: {}, + home: ports.home, + env: {}, + excludes: '/developer/ignore', + run: ports.run, + pnpm: ports.pnpm, + }); + assert.deepEqual(identity, { source: SHA, dirty: false, files: 6 }); + assert.deepEqual( + ports.events.filter((event) => event.includes('status') || event.includes('--cached')), + [ + `git ${READ('/developer/ignore')}status --porcelain=v1 -z --untracked-files=all`, + `git ${READ('/developer/ignore')}ls-files -z --cached --others --exclude-standard`, + `git ${READ('/developer/ignore')}status --porcelain=v1 -z --untracked-files=all`, + ], + ); +}); + +test('an isolated producer receives the store and, when found, the excludes file', () => { + assert.deepEqual(producerEnvironment({ HOME: '/isolated' }, { store: '/store' }), { + HOME: '/isolated', + RELAVIUM_PACKAGE_STORE_DIR: '/store', + }); + assert.deepEqual( + producerEnvironment({ HOME: '/isolated' }, { store: '/store', excludes: '/developer/ignore' }), + { + HOME: '/isolated', + RELAVIUM_PACKAGE_STORE_DIR: '/store', + RELAVIUM_GIT_EXCLUDES_FILE: '/developer/ignore', + }, + ); +}); + +test('the candidate refuses a file list or a tree over its bounds', async () => { + const listing = git({ + [`${READ()}ls-files -z --cached --others --exclude-standard`]: 'a.ts\0b.ts\0c.ts\0', + 'ls-files -z --deleted': '', + }); + assert.deepEqual(await candidateFiles('/repository', listing.run, undefined, { files: 3 }), [ + 'a.ts', + 'b.ts', + 'c.ts', + ]); + await assert.rejects( + candidateFiles('/repository', listing.run, undefined, { files: 2 }), + /CANDIDATE_FILES/u, + ); + const repository = fresh('bounded'); + write(join(repository, 'a.ts'), 'abcdef'); + write(join(repository, 'b.ts'), 'ghijkl'); + const files = ['a.ts', 'b.ts']; + assert.deepEqual( + copyCandidateTree({ + repository, + destination: fresh('copy'), + files, + limits: { fileBytes: 6, totalBytes: 12 }, + }), + { files: 2, bytes: 12 }, + ); + assert.throws( + () => + copyCandidateTree({ + repository, + destination: fresh('copy'), + files, + limits: { fileBytes: 6, totalBytes: 11 }, + }), + /CANDIDATE_BYTES/u, + ); + assert.throws( + () => + copyCandidateTree({ + repository, + destination: fresh('copy'), + files, + limits: { fileBytes: 5, totalBytes: 12 }, + }), + /FILE_KIND_OR_BOUND/u, + ); +}); + +test('the production bounds are the documented ones and apply by default', () => { + assert.deepEqual( + { ...CANDIDATE_LIMITS }, + { files: 20_000, fileBytes: 64 * 1024 * 1024, totalBytes: 512 * 1024 * 1024 }, + ); +}); + +test('candidate files and the copy apply the production bounds by default', async () => { + const names = Array.from({ length: 20_001 }, (_, index) => `f${index}.ts`).join('\0'); + const listing = git({ + [`${READ()}ls-files -z --cached --others --exclude-standard`]: names, + 'ls-files -z --deleted': '', + }); + await assert.rejects(candidateFiles('/repository', listing.run), /CANDIDATE_FILES/u); + // A sparse file one byte over the per-file bound is refused from its size, before any read. + const repository = fresh('sparse'); + const descriptor = openSync(join(repository, 'large.bin'), 'w'); + try { + ftruncateSync(descriptor, 64 * 1024 * 1024 + 1); + } finally { + closeSync(descriptor); + } + assert.throws( + () => copyCandidateTree({ repository, destination: fresh('copy'), files: ['large.bin'] }), + /FILE_KIND_OR_BOUND/u, + ); +}); + +test('the copy needs an absolute, empty destination', () => { + const repository = fresh('repository'); + write(join(repository, 'a.ts'), 'a\n'); + assert.throws( + () => + copyCandidateTree({ + repository, + destination: relative(process.cwd(), fresh('copy')), + files: ['a.ts'], + }), + /CANDIDATE_ROOT/u, + ); + const occupied = fresh('copy'); + writeFileSync(join(occupied, 'one'), ''); + assert.throws( + () => copyCandidateTree({ repository, destination: occupied, files: ['a.ts'] }), + /CANDIDATE_ROOT/u, + ); +}); + +test('only the five engine packages may be linked as @relavium workspace packages', () => { + // A link to another workspace package, even one inside the candidate, refuses. + for (const owner of [join('apps', 'cli'), join('packages', 'core')]) { + const root = linkedCandidate(); + mkdirSync(join(root, 'packages', 'ui'), { recursive: true }); + const scope = join(root, ...owner.split('/'), 'node_modules', '@relavium'); + mkdirSync(scope, { recursive: true }); + symlinkSync(join(root, 'packages', 'ui'), join(scope, 'ui'), linkKind); + assert.throws(() => verifyCandidateLinks(root), /CANDIDATE_LINK/u, owner); + } +}); + +test( + 'the excludes lookup never runs a git found in the checkout when the caller has no PATH', + { skip: !posixModes }, + async () => { + // With an empty PATH a spawn would look the command up in its working directory, the checkout. + const repository = fresh('planted-git'); + const marker = join(fresh('marker'), 'ran'); + write(join(repository, 'git'), `#!/bin/sh\necho ran > ${JSON.stringify(marker)}\n`, 0o755); + await globalExcludesFile({ repository, environment: { HOME: fresh('home') } }).catch(() => {}); + assert.equal(existsSync(marker), false); + }, +); + +for (const [label, scope] of [ + [ + 'GIT_CONFIG_COUNT', + (ignore) => ({ + GIT_CONFIG_COUNT: '1', + GIT_CONFIG_KEY_0: 'core.excludesFile', + GIT_CONFIG_VALUE_0: ignore, + }), + ], + [ + 'GIT_CONFIG_PARAMETERS', + (ignore) => ({ GIT_CONFIG_PARAMETERS: `'core.excludesFile'='${ignore}'` }), + ], +]) + test(`a real core.excludesFile set through ${label} is found`, async () => { + const root = fresh('scoped'); + const ignore = join(root, 'scoped-ignore'); + write(ignore, 'secret.local\n'); + mkdirSync(join(root, 'empty-home')); + assert.equal( + await globalExcludesFile({ + repository: fresh('repository'), + environment: { + PATH: process.env.PATH, + HOME: join(root, 'empty-home'), + GIT_CONFIG_NOSYSTEM: '1', + ...scope(ignore), + }, + }), + ignore, + ); + }); diff --git a/tools/cli-package/check.mjs b/tools/cli-package/check.mjs index 3aaba4dd..208b3a3a 100644 --- a/tools/cli-package/check.mjs +++ b/tools/cli-package/check.mjs @@ -1,20 +1,49 @@ /** Local full-CI counterpart: one new candidate and one actual-host consumer, never a platform aggregate. */ import assert from 'node:assert/strict'; import { mkdtempSync, readFileSync, realpathSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; +import { constants, tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { isolatedEnvironment, runCommand } from './process.mjs'; +import { dependencyStore, globalExcludesFile, producerEnvironment } from './candidate.mjs'; +import { forwardInterrupts, isolatedEnvironment, runCommand } from './process.mjs'; assert.equal(process.argv.length, 2, 'no arguments required'); +/** Shortens each stage's deadline (milliseconds, at most the default 600000). */ +const DEADLINE_ENV = 'RELAVIUM_LOCAL_CHECK_DEADLINE_MS'; const directory = fileURLToPath(new URL('.', import.meta.url)); const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-local-package-check-'))); +// The producer and the consumer run detached, and each stops its own child trees and removes its +// temporary directories on an interrupt, so the interrupt is forwarded to them instead of ending this +// check first. The check root, which holds those directories, goes once they have exited, and no +// later stage starts after an interrupt arrived. +const interrupts = forwardInterrupts(); +const proceed = () => + assert.equal(interrupts.received(), undefined, 'PACKED_CLI_LOCAL_CHECK_INTERRUPTED'); try { + // The producer runs with an isolated HOME, so the store and the developer's global Git excludes + // file are found here, from the caller's own location variables, and handed over explicitly. + const repository = fileURLToPath(new URL('../../', import.meta.url)); + const store = await dependencyStore({ repository, environment: process.env, home: root }); + const excludes = await globalExcludesFile({ repository, environment: process.env }); const artifact = join(root, 'artifact'), reports = join(root, 'reports'); - const options = { env: isolatedEnvironment(root), timeoutMs: 600_000 }; - await runCommand(process.execPath, [join(directory, 'pack.mjs'), artifact], options); + // A stage past its deadline is interrupted first (SIGTERM), so it stops its own detached build + // children and removes its private directories; only a stage that ignores that is killed. The + // deadline can only be shortened, for the tests. + const deadline = Number(process.env[DEADLINE_ENV] ?? 600_000); + assert.ok( + Number.isSafeInteger(deadline) && deadline > 0 && deadline <= 600_000, + 'PACKED_CLI_LOCAL_CHECK_DEADLINE', + ); + const options = { env: isolatedEnvironment(root), timeoutMs: deadline, graceMs: 30_000 }; + proceed(); + await runCommand(process.execPath, [join(directory, 'pack.mjs'), artifact], { + ...options, + env: producerEnvironment(options.env, { store, excludes }), + }); + proceed(); await runCommand(process.execPath, [join(directory, 'consume.mjs'), artifact, reports], options); + proceed(); const record = JSON.parse(readFileSync(join(reports, 'installed-consumer.json'), 'utf8')); assert.equal(record.leg, 'local'); console.log( @@ -28,6 +57,12 @@ try { smokeCases: record.smoke.cases.length, }), ); +} catch (error) { + if (interrupts.received() === undefined) throw error; } finally { - rmSync(root, { recursive: true, force: true }); + // Retries absorb a child killed by the interrupt that was still finishing a write. + rmSync(root, { recursive: true, force: true, maxRetries: 3 }); + await interrupts.dispose(); } +const signal = interrupts.received(); +if (signal !== undefined) process.exit(128 + (constants.signals[signal] ?? 0)); diff --git a/tools/cli-package/check.test.mjs b/tools/cli-package/check.test.mjs new file mode 100644 index 00000000..735a347a --- /dev/null +++ b/tools/cli-package/check.test.mjs @@ -0,0 +1,612 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { devNull, tmpdir } from 'node:os'; +import { basename, delimiter, dirname, join } from 'node:path'; +import { after, test } from 'node:test'; +import { setTimeout as delay } from 'node:timers/promises'; +import { fileURLToPath } from 'node:url'; + +// The local check runs from a copied tooling tree whose pack.mjs and consume.mjs are stand-ins, so no +// case can build, pack or install the real CLI even if the orchestration regresses. +const tooling = fileURLToPath(new URL('../', import.meta.url)); +const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-local-check-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); +let count = 0; + +const COPIED = Object.freeze([ + 'cli-bootstrap/child.mjs', + 'cli-bootstrap/policy.mjs', + 'cli-bootstrap/process.mjs', + 'cli-package/smoke/index.mjs', + 'cli-package/check.mjs', + 'cli-package/candidate.mjs', + 'cli-package/consumer-lifetime.mjs', + 'cli-package/files.mjs', + 'cli-package/finalize.mjs', + 'cli-package/source-map.mjs', + 'cli-package/paths.mjs', + 'cli-package/process.mjs', + 'bundle-closure/closure.mjs', + 'bundle-closure/inventory.mjs', +]); + +/** + * A long-lived process that records its pid, then tries to write a file into `directory` every 5 ms; + * it outlives the directory's removal unless it is killed. + */ +function writer(directory, pidFile) { + return ( + `const { writeFileSync } = require('node:fs');` + + `const { join } = require('node:path');` + + `writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + `let n = 0; setInterval(() => { try { writeFileSync(join(${JSON.stringify(directory)}, 'f' + n++), 'x'); } catch {} }, 5);` + ); +} + +/** The consumer stand-in: the real consumer lifetime around a `pnpm add`-like writer in its home. */ +const CONSUMER = (markers) => ` +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { completeConsumer } from './consumer-lifetime.mjs'; +import { isolatedEnvironment, runCommand } from './process.mjs'; +writeFileSync(${JSON.stringify(join(markers, 'consume-started'))}, String(process.pid)); +const reports = process.argv[3]; +mkdirSync(reports, { recursive: true }); +const home = mkdtempSync(join(tmpdir(), 'relavium-package-consumer-')); +const writer = (${writer.toString()})(home, ${JSON.stringify(join(markers, 'writer.pid'))}); +await completeConsumer({ + exercise: () => + runCommand(process.execPath, ['-e', writer], { + env: isolatedEnvironment(home), + timeoutMs: 60_000, + }), + supportRoot: reports, + cleanup: () => rmSync(home, { recursive: true, force: true, maxRetries: 3 }), + cleanupFailed: () => {}, + publish: () => {}, +}); +`; + +/** A consumer stand-in that publishes the record `pnpm smoke:cli-package` summarizes. */ +const PUBLISHING_CONSUMER = (leg) => (markers) => ` +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +writeFileSync(${JSON.stringify(join(markers, 'consume-started'))}, String(process.pid)); +const reports = process.argv[3]; +mkdirSync(reports, { recursive: true }); +writeFileSync(join(reports, 'installed-consumer.json'), ${JSON.stringify( + JSON.stringify({ + leg, + source: 'c'.repeat(40), + tarballSha256: 'd'.repeat(64), + node: 'v22.0.0', + platform: 'synthetic', + bootstrap: { controls: 7 }, + smoke: { cases: [{ id: 'one', passed: true }] }, + }), +)}); +`; + +/** A producer stand-in that records the store and excludes variables it received, then succeeds. */ +const RECORDING_PRODUCER = (markers) => ` +import { mkdirSync, writeFileSync } from 'node:fs'; +writeFileSync(${JSON.stringify(join(markers, 'pack-environment'))}, JSON.stringify({ + store: process.env.RELAVIUM_PACKAGE_STORE_DIR ?? null, + excludes: process.env.RELAVIUM_GIT_EXCLUDES_FILE ?? null, +})); +mkdirSync(process.argv[2]); +`; + +/** + * Start the copied check with stand-ins for its producer and consumer. The store is named + * explicitly, so the check runs no pnpm, unless a `pnpm` stand-in is given for its store discovery; + * so is the excludes file unless `excludes` is false, when the check's Git lookup sees an empty home + * and no global or system configuration. + */ +function check({ pack, consume = CONSUMER, excludes: named = true, git, pnpm, env: extra = {} }) { + count += 1; + const root = join(scratch, `repository-${count}`); + for (const path of COPIED) { + mkdirSync(dirname(join(root, 'tools', path)), { recursive: true }); + copyFileSync(join(tooling, path), join(root, 'tools', path)); + } + symlinkSync( + fileURLToPath(new URL('../../node_modules', import.meta.url)), + join(root, 'node_modules'), + process.platform === 'win32' ? 'junction' : 'dir', + ); + const markers = join(root, 'markers'); + const temporary = join(root, 'tmp'); + const store = join(root, 'store'); + for (const directory of [markers, temporary, store]) mkdirSync(directory); + const excludes = join(root, 'ignore'); + writeFileSync(excludes, ''); + const home = join(root, 'home'); + mkdirSync(home); + // Optional `git` and `pnpm` stand-ins, first on PATH, for the check's own excludes lookup and + // store discovery. + const bin = join(root, 'bin'); + mkdirSync(bin); + if (git !== undefined) writeFileSync(join(bin, 'git'), git, { mode: 0o755 }); + if (pnpm !== undefined) writeFileSync(join(bin, 'pnpm'), pnpm(markers, store), { mode: 0o755 }); + writeFileSync(join(root, 'tools', 'cli-package', 'pack.mjs'), pack(markers)); + writeFileSync(join(root, 'tools', 'cli-package', 'consume.mjs'), consume(markers)); + const child = spawn(process.execPath, [join(root, 'tools', 'cli-package', 'check.mjs')], { + env: { + PATH: `${bin}${delimiter}${process.env.PATH}`, + TMPDIR: temporary, + ...(pnpm === undefined ? { RELAVIUM_PACKAGE_STORE_DIR: store } : {}), + ...(named + ? { RELAVIUM_GIT_EXCLUDES_FILE: excludes } + : { HOME: home, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: devNull }), + ...extra, + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => (stdout += chunk)); + child.stderr.on('data', (chunk) => (stderr += chunk)); + const exited = new Promise((resolve) => + child.once('exit', (code, signal) => resolve({ code, signal, stdout, stderr })), + ); + return { child, exited, root, markers, temporary, store, excludes }; +} + +/** The check's exit, bounded so that a regressed check fails its test instead of hanging it. */ +async function exitOf(run) { + // Unreferenced, so a pending bound never keeps the test process alive after the check exits. + const bound = delay(60_000, undefined, { ref: false }).then(() => undefined); + const outcome = await Promise.race([run.exited, bound]); + assert.ok(outcome !== undefined, 'the check did not exit within 60 s'); + return outcome; +} + +async function waitForFile(file) { + for (let waited = 0; !existsSync(file) || readFileSync(file, 'utf8') === ''; waited += 25) { + assert.ok(waited < 20_000, `${file} never appeared`); + await delay(25); + } + return readFileSync(file, 'utf8'); +} + +function alive(pid) { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +/** A killed process can still answer for a moment after its killer exits: wait, with a bound. */ +async function assertGone(pid, message) { + for (let waited = 0; alive(pid); waited += 25) { + assert.ok(waited < 5_000, message); + await delay(25); + } +} + +/** + * Stop the check and every stand-in a regressed check could leave behind: the stand-ins run in their + * own process groups, so they would outlive a check that died without forwarding its interrupt. + */ +function stopAll(run) { + if (run.child.exitCode === null && run.child.signalCode === null) run.child.kill('SIGKILL'); + for (const name of [ + 'pack.pid', + 'consume-started', + 'writer.pid', + 'grandchild.pid', + 'helper.pid', + ]) { + const file = join(run.markers, name); + const pid = existsSync(file) ? Number(readFileSync(file, 'utf8')) : NaN; + if (Number.isSafeInteger(pid) && pid > 0 && alive(pid)) process.kill(pid, 'SIGKILL'); + } +} + +const checkRoots = (temporary) => + readdirSync(temporary).filter((name) => name.startsWith('relavium-local-package-check-')); + +test( + 'an interrupt during the consumer stops its children, removes the check root and exits', + { skip: process.platform === 'win32' }, + async () => { + const run = check({ + pack: () => `import { mkdirSync } from 'node:fs';\nmkdirSync(process.argv[2]);\n`, + }); + let pid; + try { + pid = Number(await waitForFile(join(run.markers, 'writer.pid'))); + assert.equal(checkRoots(run.temporary).length, 1); + run.child.kill('SIGINT'); + const outcome = await exitOf(run); + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 130); + assert.equal(outcome.stdout, ''); + // The consumer received the forwarded interrupt and stopped its writer before its home went. + await assertGone(pid, 'the consumer writer outlived the interrupt'); + assert.deepEqual(checkRoots(run.temporary), []); + } finally { + stopAll(run); + } + }, +); + +test( + 'an interrupt during the producer reaches it, and the consumer never starts', + { skip: process.platform === 'win32' }, + async () => { + const run = check({ + pack: (markers) => ` +import { writeFileSync } from 'node:fs'; +process.on('SIGINT', () => { + writeFileSync(${JSON.stringify(join(markers, 'pack-interrupted'))}, 'SIGINT'); + process.exit(130); +}); +writeFileSync(${JSON.stringify(join(markers, 'pack.pid'))}, String(process.pid)); +setInterval(() => {}, 1000); +`, + }); + try { + await waitForFile(join(run.markers, 'pack.pid')); + run.child.kill('SIGINT'); + const outcome = await exitOf(run); + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 130); + assert.equal(readFileSync(join(run.markers, 'pack-interrupted'), 'utf8'), 'SIGINT'); + assert.equal(existsSync(join(run.markers, 'consume-started')), false); + assert.deepEqual(checkRoots(run.temporary), []); + } finally { + stopAll(run); + } + }, +); + +test( + 'an interrupt the producer outlives still stops the check before the consumer', + { skip: process.platform === 'win32' }, + async () => { + // The producer signals the check, ignores the interrupt forwarded back to it and finishes. + const run = check({ + pack: (markers) => ` +import { mkdirSync, writeFileSync } from 'node:fs'; +process.on('SIGINT', () => writeFileSync(${JSON.stringify(join(markers, 'pack-forwarded'))}, 'SIGINT')); +mkdirSync(process.argv[2]); +process.kill(process.ppid, 'SIGINT'); +setTimeout(() => process.exit(0), 500); +`, + }); + try { + const outcome = await exitOf(run); + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 130); + assert.equal(readFileSync(join(run.markers, 'pack-forwarded'), 'utf8'), 'SIGINT'); + assert.equal(existsSync(join(run.markers, 'consume-started')), false); + assert.deepEqual(checkRoots(run.temporary), []); + } finally { + stopAll(run); + } + }, +); + +test('a successful producer and consumer give the summary and leave no check root', async () => { + const run = check({ pack: RECORDING_PRODUCER, consume: PUBLISHING_CONSUMER('local') }); + try { + const outcome = await exitOf(run); + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 0); + assert.deepEqual(JSON.parse(outcome.stdout), { + scope: 'actual local host only; required CI owns four platform legs', + source: 'c'.repeat(40), + sha256: 'd'.repeat(64), + node: 'v22.0.0', + platform: 'synthetic', + controls: 7, + smokeCases: 1, + }); + // The isolated producer receives the store and the excludes file the check found. + assert.deepEqual(JSON.parse(readFileSync(join(run.markers, 'pack-environment'), 'utf8')), { + store: run.store, + excludes: run.excludes, + }); + assert.deepEqual(checkRoots(run.temporary), []); + } finally { + stopAll(run); + } +}); + +test('without a named store the check asks pnpm in the repository, with its own private home', async () => { + // The private home must lie outside the repository, so the temporary directory does too. + const outside = mkdtempSync(join(scratch, 'outside-')); + const developer = join(outside, 'developer-home'); + const run = check({ + env: { TMPDIR: outside, HOME: developer }, + pack: RECORDING_PRODUCER, + consume: PUBLISHING_CONSUMER('local'), + pnpm: (markers, store) => + '#!/usr/bin/env node\n' + + `require('node:fs').writeFileSync(${JSON.stringify(join(markers, 'pnpm-call'))}, ` + + 'JSON.stringify({ args: process.argv.slice(2), cwd: process.cwd(), ' + + 'home: process.env.HOME, temporary: process.env.TMPDIR, cache: process.env.XDG_CACHE_HOME }));\n' + + `process.stdout.write(${JSON.stringify(`${store}\n`)});\n`, + }); + try { + const outcome = await exitOf(run); + assert.equal(outcome.code, 0, outcome.stderr); + const call = JSON.parse(readFileSync(join(run.markers, 'pnpm-call'), 'utf8')); + assert.deepEqual(call.args, ['store', 'path']); + assert.equal(realpathSync(call.cwd), realpathSync(run.root)); + // The store location follows the developer's own HOME; temporary files and caches go to the + // check's owner-only root. + assert.equal(call.home, developer); + assert.equal(dirname(call.temporary), outside); + assert.match(basename(call.temporary), /^relavium-local-package-check-/u); + assert.equal(call.cache, call.temporary); + assert.equal( + JSON.parse(readFileSync(join(run.markers, 'pack-environment'), 'utf8')).store, + run.store, + ); + assert.deepEqual(checkRoots(outside), []); + } finally { + stopAll(run); + } +}); + +test('without a global excludes file the producer receives none', async () => { + const run = check({ + pack: RECORDING_PRODUCER, + consume: PUBLISHING_CONSUMER('local'), + excludes: false, + }); + try { + assert.equal((await exitOf(run)).code, 0); + assert.deepEqual(JSON.parse(readFileSync(join(run.markers, 'pack-environment'), 'utf8')), { + store: run.store, + excludes: null, + }); + } finally { + stopAll(run); + } +}); + +for (const [label, options] of [ + ['a failed producer', { pack: () => 'process.exit(3);\n' }], + ['a failed consumer', { pack: RECORDING_PRODUCER, consume: () => 'process.exit(4);\n' }], + [ + 'a record from another leg', + { pack: RECORDING_PRODUCER, consume: PUBLISHING_CONSUMER('linux') }, + ], +]) + test(`${label} fails the check without a summary and leaves no check root`, async () => { + const run = check(options); + try { + const outcome = await exitOf(run); + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 1); + assert.equal(outcome.stdout, ''); + if (label === 'a failed producer') + assert.equal(existsSync(join(run.markers, 'consume-started')), false); + assert.deepEqual(checkRoots(run.temporary), []); + } finally { + stopAll(run); + } + }); + +test( + 'an interrupt the consumer outlives still gives no summary', + { skip: process.platform === 'win32' }, + async () => { + // The consumer signals the check, ignores the interrupt forwarded back to it and publishes. + const outliving = (markers) => + PUBLISHING_CONSUMER('local')(markers) + + `process.on('SIGINT', () => writeFileSync(${JSON.stringify(join(markers, 'consume-forwarded'))}, 'SIGINT'));\n` + + `process.kill(process.ppid, 'SIGINT');\nsetTimeout(() => process.exit(0), 500);\n`; + const run = check({ pack: RECORDING_PRODUCER, consume: outliving }); + try { + const outcome = await exitOf(run); + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 130); + assert.equal(outcome.stdout, ''); + assert.equal(readFileSync(join(run.markers, 'consume-forwarded'), 'utf8'), 'SIGINT'); + assert.deepEqual(checkRoots(run.temporary), []); + } finally { + stopAll(run); + } + }, +); + +test( + 'an interrupt during discovery stops the check before the producer', + { skip: process.platform === 'win32' }, + async () => { + // The excludes lookup's Git signals the check, ignores the interrupt forwarded back to it and + // exits 1 (no setting), so discovery itself completes normally. + const run = check({ + pack: RECORDING_PRODUCER, + consume: PUBLISHING_CONSUMER('local'), + excludes: false, + git: `#!/bin/sh\ntrap 'exit 1' INT\nkill -INT "$PPID"\nsleep 1 &\nwait\nexit 1\n`, + }); + try { + const outcome = await exitOf(run); + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 130); + assert.equal(outcome.stdout, ''); + assert.equal(existsSync(join(run.markers, 'pack-environment')), false); + assert.equal(existsSync(join(run.markers, 'consume-started')), false); + assert.deepEqual(checkRoots(run.temporary), []); + } finally { + stopAll(run); + } + }, +); + +test( + 'a stage past its deadline is interrupted first, so its own children never outlive the check', + { skip: process.platform === 'win32' }, + async () => { + // A producer that handles the interrupt as the real one does, around a detached build child that + // would write a map into the check root after the check removed it. + const run = check({ + env: { RELAVIUM_LOCAL_CHECK_DEADLINE_MS: '1500' }, + pack: (markers) => ` +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { writeFileSync } from 'node:fs'; +import { exitOnInterrupt, isolatedEnvironment, runCommand } from './process.mjs'; +const late = join(tmpdir(), 'late'); +await exitOnInterrupt( + () => writeFileSync(${JSON.stringify(join(markers, 'pack-interrupted'))}, 'cleaned'), + () => + runCommand(process.execPath, ['-e', ${JSON.stringify( + `const fs = require('node:fs');` + + `fs.writeFileSync(${JSON.stringify(join(markers, 'grandchild.pid'))}, String(process.pid));`, + )} + 'setTimeout(() => { require("node:fs").mkdirSync(' + JSON.stringify(late) + ', { recursive: true }); require("node:fs").writeFileSync(' + JSON.stringify(join(late, 'index.js.map')) + ', "map"); }, 3000); setInterval(() => {}, 1000);'], { + env: isolatedEnvironment(tmpdir()), + timeoutMs: 60_000, + }), +); +`, + }); + try { + const grandchild = Number(await waitForFile(join(run.markers, 'grandchild.pid'))); + const [checkRoot] = checkRoots(run.temporary); + assert.ok(checkRoot !== undefined); + const outcome = await exitOf(run); + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 1); + assert.equal(outcome.stdout, ''); + assert.equal(readFileSync(join(run.markers, 'pack-interrupted'), 'utf8'), 'cleaned'); + await assertGone(grandchild, 'the stage build child outlived the deadline'); + // Past the moment the build child would have written: nothing recreated the check root. + await delay(3500); + assert.equal(existsSync(join(run.temporary, checkRoot)), false); + } finally { + stopAll(run); + } + }, +); + +test('only a shorter, positive stage deadline is admitted', async () => { + for (const deadline of ['0', '-1', '600001', 'soon', '']) { + const run = check({ + env: { RELAVIUM_LOCAL_CHECK_DEADLINE_MS: deadline }, + pack: RECORDING_PRODUCER, + consume: PUBLISHING_CONSUMER('local'), + }); + try { + const outcome = await exitOf(run); + assert.equal(outcome.code, 1, deadline); + assert.equal(existsSync(join(run.markers, 'pack-environment')), false, deadline); + assert.deepEqual(checkRoots(run.temporary), []); + } finally { + stopAll(run); + } + } +}); + +test( + 'an interrupt during the root removal of the check still ends it with the signal', + { skip: process.platform === 'win32' }, + async () => { + // The consumer publishes, leaves a large tree in the check root and starts a helper that signals + // the check as soon as that tree starts to shrink, that is during the check's own removal. + const removal = (markers) => + PUBLISHING_CONSUMER('local')(markers) + + ` +const bulk = join(reports, 'bulk'); +mkdirSync(bulk); +for (let i = 0; i < 20000; i += 1) writeFileSync(join(bulk, String(i)), ''); +const { spawn } = await import('node:child_process'); +const helper = spawn(process.execPath, ['-e', ${JSON.stringify( + `const fs = require('node:fs');` + + `const [bulk, check, marker] = process.argv.slice(1);` + + `const poll = () => { let count; try { count = fs.readdirSync(bulk).length; } catch { process.exit(0); }` + + ` if (count < 20000) { process.kill(Number(check), 'SIGINT'); fs.writeFileSync(marker, 'sent'); process.exit(0); }` + + ` setTimeout(poll, 1); }; poll();`, + )}, bulk, String(process.ppid), ${JSON.stringify(join(markers, 'removal-signalled'))}], { detached: true, stdio: 'ignore' }); +writeFileSync(${JSON.stringify(join(markers, 'helper.pid'))}, String(helper.pid)); +helper.unref(); +`; + const run = check({ pack: RECORDING_PRODUCER, consume: removal }); + try { + const outcome = await exitOf(run); + assert.equal(readFileSync(join(run.markers, 'removal-signalled'), 'utf8'), 'sent'); + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 130); + assert.deepEqual(checkRoots(run.temporary), []); + } finally { + stopAll(run); + } + }, +); + +test( + 'a consumer past its deadline is interrupted first too, so its own children never outlive the check', + { skip: process.platform === 'win32' }, + async () => { + // The real consumer lifetime around a detached child that would write into the check root later. + const lateConsumer = (markers) => ` +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { completeConsumer } from './consumer-lifetime.mjs'; +import { isolatedEnvironment, runCommand } from './process.mjs'; +const reports = process.argv[3]; +mkdirSync(reports, { recursive: true }); +const home = mkdtempSync(join(tmpdir(), 'relavium-package-consumer-')); +const late = join(tmpdir(), 'late'); +await completeConsumer({ + supportRoot: reports, + exercise: () => + runCommand(process.execPath, ['-e', ${JSON.stringify( + `require('node:fs').writeFileSync(${JSON.stringify(join(markers, 'grandchild.pid'))}, String(process.pid));`, + )} + 'setTimeout(() => { require("node:fs").mkdirSync(' + JSON.stringify(late) + ', { recursive: true }); require("node:fs").writeFileSync(' + JSON.stringify(join(late, 'installed.log')) + ', "late"); }, 3000); setInterval(() => {}, 1000);'], { + env: isolatedEnvironment(home), + timeoutMs: 60_000, + }), + cleanup: () => { + rmSync(home, { recursive: true, force: true, maxRetries: 3 }); + writeFileSync(${JSON.stringify(join(markers, 'consume-cleaned'))}, 'cleaned'); + }, + cleanupFailed: () => {}, + publish: () => {}, +}); +`; + const run = check({ + env: { RELAVIUM_LOCAL_CHECK_DEADLINE_MS: '1500' }, + pack: RECORDING_PRODUCER, + consume: lateConsumer, + }); + try { + const grandchild = Number(await waitForFile(join(run.markers, 'grandchild.pid'))); + const [checkRoot] = checkRoots(run.temporary); + assert.ok(checkRoot !== undefined); + const outcome = await exitOf(run); + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 1); + assert.equal(outcome.stdout, ''); + assert.equal(readFileSync(join(run.markers, 'consume-cleaned'), 'utf8'), 'cleaned'); + await assertGone(grandchild, 'the stage build child outlived the deadline'); + await delay(3500); + assert.equal(existsSync(join(run.temporary, checkRoot)), false); + } finally { + stopAll(run); + } + }, +); diff --git a/tools/cli-package/child-exit.mjs b/tools/cli-package/child-exit.mjs new file mode 100644 index 00000000..50df296a --- /dev/null +++ b/tools/cli-package/child-exit.mjs @@ -0,0 +1,24 @@ +/** + * Test-only: a spawned tool's exit within a bound, so that a regressed tool fails its test instead of + * hanging it, which would let the runner kill the test and leave the tool's detached descendants + * behind. Call it before signalling the child. + */ +import assert from 'node:assert/strict'; +import { setTimeout as delay } from 'node:timers/promises'; + +/** + * @param {import('node:child_process').ChildProcess} child + * @param {number} [timeoutMs] + * @returns {Promise<[number | null, string | null]>} the exit code and signal + */ +export function exitWithin(child, timeoutMs = 30_000) { + const exit = new Promise((resolve) => + child.once('exit', (code, signal) => resolve([code, signal])), + ); + // Unreferenced, so a pending bound never keeps the test process alive after the child exits. + const bound = delay(timeoutMs, undefined, { ref: false }); + return Promise.race([exit, bound]).then((outcome) => { + assert.ok(outcome !== undefined, `the child did not exit within ${timeoutMs} ms`); + return outcome; + }); +} diff --git a/tools/cli-package/consume.mjs b/tools/cli-package/consume.mjs index 581e0e2c..a76cb822 100644 --- a/tools/cli-package/consume.mjs +++ b/tools/cli-package/consume.mjs @@ -12,7 +12,7 @@ import { writeSync, } from 'node:fs'; import { tmpdir } from 'node:os'; -import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; +import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { runBootstrap } from '../cli-bootstrap/check.mjs'; import { inspectPackageArchive } from './archive.mjs'; @@ -22,7 +22,14 @@ import { validateConsumerRecord, } from './binding.mjs'; import { installedInventory, oneTarball, readOrdinaryFile, sha256 } from './files.mjs'; -import { exerciseInstalledCommand, isolatedEnvironment, runCommand, runPnpm } from './process.mjs'; +import { placedWithin, within } from './paths.mjs'; +import { + exerciseInstalledCommand, + isolatedEnvironment, + isolatedGitEnvironment, + runCommand, + runPnpm, +} from './process.mjs'; import { runInstalledSmoke } from './smoke/index.mjs'; import { completeConsumer } from './consumer-lifetime.mjs'; @@ -35,12 +42,13 @@ const receipt = JSON.parse( readOrdinaryFile(join(artifactDirectory, 'package-test-receipt.json'), 512 * 1024), ); const origin = originFromEnvironment(process.env); +// CI always names the source; only a local run asks Git. const source = process.env.RELAVIUM_PACKAGE_SOURCE_SHA ?? ( await runCommand('git', ['--no-replace-objects', 'rev-parse', 'HEAD'], { cwd: repository, - env: isolatedEnvironment(repository), + env: isolatedGitEnvironment(), }) ).stdout.trim(); const nodeMajor = Number(readFileSync(join(repository, '.nvmrc'), 'utf8').trim()); @@ -57,11 +65,8 @@ inspectPackageArchive(bytes, { expectedPackage: receipt.package, }); const home = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-package-consumer-'))); -const inside = (parent, child) => { - const part = relative(parent, child); - return part === '' || (!isAbsolute(part) && part !== '..' && !part.startsWith(`..${sep}`)); -}; -assert.equal(inside(realpathSync(repository), home), false, 'PACKED_CLI_INSTALL_IN_WORKSPACE'); +// Compared on the stored letter case, so a TMPDIR naming the checkout in another case is refused. +assert.equal(placedWithin(repository, home), false, 'PACKED_CLI_INSTALL_IN_WORKSPACE'); await completeConsumer({ async exercise() { const env = isolatedEnvironment(home); @@ -92,7 +97,7 @@ await completeConsumer({ ); writeFileSync(join(reportDirectory, 'install.log'), installation.stdout + installation.stderr); const packageRoot = realpathSync(join(home, 'node_modules/relavium')); - assert.ok(inside(home, packageRoot), 'PACKED_CLI_INSTALL_ESCAPED'); + assert.ok(within(home, packageRoot), 'PACKED_CLI_INSTALL_ESCAPED'); const before = installedInventory(packageRoot); const wanted = receipt.files .map(({ path, bytes, sha256 }) => ({ path, bytes, sha256 })) @@ -116,10 +121,10 @@ await completeConsumer({ name === '@modelcontextprotocol/sdk' ? '@modelcontextprotocol/sdk/client/index.js' : name, ), ); - assert.ok(inside(home, entry), 'PACKED_CLI_DEPENDENCY_OUTSIDE_INSTALL'); + assert.ok(within(home, entry), 'PACKED_CLI_DEPENDENCY_OUTSIDE_INSTALL'); let directory = dirname(entry), found = false; - while (inside(home, directory)) { + while (within(home, directory)) { let content; try { content = readOrdinaryFile(join(directory, 'package.json'), 1024 * 1024); @@ -176,7 +181,10 @@ await completeConsumer({ validateConsumerRecord(record, receipt, { ...expected, allowLocal: true }); return record; }, - cleanup: () => rmSync(home, { recursive: true, force: true }), + // An interrupt also abandons the bootstrap witness's fixtures and the installed smoke's worker roots + // in the report directory; retries absorb a killed child that was still finishing a write. + supportRoot: reportDirectory, + cleanup: () => rmSync(home, { recursive: true, force: true, maxRetries: 3 }), cleanupFailed: () => writeSync(2, 'PACKED_CLI_SECONDARY_INSTALLATION_CLEANUP_FAILED\n'), publish(record) { writeFileSync( diff --git a/tools/cli-package/consumer-lifetime.mjs b/tools/cli-package/consumer-lifetime.mjs index 1ba77b7b..6b11bab7 100644 --- a/tools/cli-package/consumer-lifetime.mjs +++ b/tools/cli-package/consumer-lifetime.mjs @@ -1,24 +1,60 @@ -/** A success publication includes actual worker settlement and owned installation removal. */ -export async function completeConsumer({ exercise, cleanup, cleanupFailed, publish }) { - let result; - let primary; - let primaryFailure = false; - try { - result = await exercise(); - } catch (error) { - primary = error; - primaryFailure = true; - } - try { - await cleanup(); - } catch (error) { - if (!primaryFailure) throw error; +import assert from 'node:assert/strict'; +import { isAbsolute } from 'node:path'; +import { abandonFixtures } from '../cli-bootstrap/child.mjs'; +import { exitOnInterrupt } from './process.mjs'; +import { abandonInstalledSmoke } from './smoke/index.mjs'; + +/** + * A success publication includes actual worker settlement and owned installation removal. An + * interrupt (SIGINT, SIGTERM, SIGHUP) is the primary outcome: it stops every `runCommand` child tree, + * then the bootstrap witness's bounded children, removes its live fixtures and the installed smoke's + * worker roots under `supportRoot` (the native fixture targets end with that worker), removes the + * installation and exits with the signal's code, reporting any failed removal only through + * `cleanupFailed`. `cleanup` is therefore synchronous. + * @template T + * @param {{exercise: () => Promise, cleanup: () => void, cleanupFailed: () => void, + * publish: (result: T) => unknown, supportRoot: string}} options + */ +export async function completeConsumer({ exercise, cleanup, cleanupFailed, publish, supportRoot }) { + assert.ok( + typeof supportRoot === 'string' && isAbsolute(supportRoot), + 'PACKED_CLI_CONSUMER_SUPPORT_ROOT', + ); + const secondary = () => { try { cleanupFailed(); } catch { - // A failed secondary diagnostic cannot replace the original exercise failure. + // A failed secondary diagnostic cannot replace the primary outcome. } - } - if (primaryFailure) throw primary; - return publish(result); + }; + const interrupted = () => { + let failed = false; + for (const step of [abandonFixtures, () => abandonInstalledSmoke(supportRoot), cleanup]) { + try { + step(); + } catch { + failed = true; + } + } + if (failed) secondary(); + }; + return exitOnInterrupt(interrupted, async () => { + let result; + let primary; + let primaryFailure = false; + try { + result = await exercise(); + } catch (error) { + primary = error; + primaryFailure = true; + } + try { + cleanup(); // synchronous by contract, see above + } catch (error) { + if (!primaryFailure) throw error; + secondary(); + } + if (primaryFailure) throw primary; + return publish(result); + }); } diff --git a/tools/cli-package/consumer-lifetime.test.mjs b/tools/cli-package/consumer-lifetime.test.mjs index df66448c..91f9ffb0 100644 --- a/tools/cli-package/consumer-lifetime.test.mjs +++ b/tools/cli-package/consumer-lifetime.test.mjs @@ -6,16 +6,26 @@ import { mkdirSync, mkdtempSync, readFileSync, + readdirSync, + realpathSync, rmSync, writeFileSync, } from 'node:fs'; +import { spawn } from 'node:child_process'; import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; -import { test } from 'node:test'; +import { after, test } from 'node:test'; +import { setTimeout as delay } from 'node:timers/promises'; import { pathToFileURL } from 'node:url'; +import { exitWithin } from './child-exit.mjs'; import { completeConsumer } from './consumer-lifetime.mjs'; import { REQUIRED_SMOKE_CASES } from './smoke/cases.mjs'; +import { abandonInstalledSmoke, createInstalledSmokeRoot } from './smoke/index.mjs'; + +// A support directory for the cases that never reach an interrupt. +const synthetic = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-consumer-support-'))); +after(() => rmSync(synthetic, { recursive: true, force: true })); async function fixture( t, @@ -44,6 +54,7 @@ async function fixture( join(smoke, 'worker.mjs'), ` import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; import { createRequire } from 'node:module'; import { closeSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; @@ -107,6 +118,7 @@ test('real addon worker closes before installation removal and success publicati // On Windows an in-process or still-running addon owner keeps this copy locked. rmSync(own.installation, { recursive: true, force: true }); }, + supportRoot: own.support, cleanupFailed: () => assert.fail('unexpected secondary cleanup failure'), publish: (report) => { assert.equal(existsSync(own.installation), false); @@ -141,6 +153,7 @@ for (const diagnosticSinkFailure of [false, true]) assert.equal(readFileSync(join(own.support, 'worker-exit'), 'utf8'), 'observed'); rmSync(own.installation, { recursive: true, force: true }); }, + supportRoot: own.support, cleanupFailed: () => assert.fail('unexpected secondary cleanup failure'), publish: () => { published = true; @@ -186,6 +199,7 @@ for (const [name, options, expected] of [ completeConsumer({ exercise: own.run, cleanup: () => rmSync(own.installation, { recursive: true, force: true }), + supportRoot: own.support, cleanupFailed: () => assert.fail('unexpected secondary cleanup failure'), publish: () => { published = true; @@ -207,6 +221,7 @@ test('failed required cleanup prevents publication of a successful exercise', as cleanup: () => { throw cleanupFailure; }, + supportRoot: synthetic, cleanupFailed: () => assert.fail('cleanup is primary here'), publish: () => { published = true; @@ -231,6 +246,7 @@ for (const primary of [new Error('synthetic worker refusal'), undefined, null]) cleaned = true; throw new Error('synthetic secondary cleanup refusal'); }, + supportRoot: synthetic, cleanupFailed: () => { diagnosed = true; throw new Error('synthetic failed diagnostic sink'); @@ -245,3 +261,237 @@ for (const primary of [new Error('synthetic worker refusal'), undefined, null]) assert.equal(diagnosed, true); assert.equal(published, false); }); + +test('a consumer without an absolute support directory refuses before it exercises', async () => { + for (const supportRoot of [undefined, 'relative/reports']) { + let exercised = false; + await assert.rejects( + completeConsumer({ + exercise: async () => { + exercised = true; + }, + cleanup: () => {}, + cleanupFailed: () => {}, + publish: () => {}, + supportRoot, + }), + /PACKED_CLI_CONSUMER_SUPPORT_ROOT/u, + ); + assert.equal(exercised, false); + } +}); + +function alive(pid) { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +/** + * Code for a long-lived process that records its pid, then tries to write into the directory + * `directory` (a JavaScript expression) every 5 ms; it outlives that directory unless it is killed. + * With `grandchildFile`, it first starts a second such process in its own process group. + */ +function writer(directory, pidFile, grandchildFile) { + const own = + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + `let n = 0; setInterval(() => { try { require('node:fs').writeFileSync(` + + `require('node:path').join(${directory}, 'f' + n++), 'x'); } catch {} }, 5);`; + if (grandchildFile === undefined) return own; + return ( + `require('node:child_process').spawn(process.execPath, ['-e', ${JSON.stringify( + writer(directory, grandchildFile), + )}], { stdio: 'ignore' });` + own + ); +} + +async function waitForPid(file) { + for (let waited = 0; !existsSync(file) || readFileSync(file, 'utf8') === ''; waited += 25) { + assert.ok(waited < 20_000, `${file} never appeared`); + await delay(25); + } + return Number(readFileSync(file, 'utf8')); +} + +/** + * A consumer process interrupted mid-exercise: a `runCommand` child keeps writing into the owned + * installation, as `pnpm add` does, while a running bootstrap control's bounded child (with a + * grandchild in its process group) keeps writing into its fixture, and a smoke worker root, whose + * worker an interrupt kills before its own removal, waits in the support directory. + */ +async function interruptedConsumer(t, { cleanupFails = false } = {}) { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-consumer-interrupt-'))); + t.after(() => rmSync(root, { recursive: true, force: true })); + const installation = join(root, 'installation'); + const support = join(root, 'support'); + const temporary = join(root, 'tmp'); + for (const directory of [installation, support, temporary]) mkdirSync(directory); + const smoke = join(support, 'installed-smoke-planted'); + mkdirSync(smoke); + writeFileSync(join(smoke, 'guard.ndjson'), '{}\n'); + // Report entries that are not worker roots, a near miss included, which the interrupt keeps. + const kept = ['install.log', 'installed-smoke.json', 'installed-smokeX', 'bootstrap']; + writeFileSync(join(support, 'install.log'), 'log\n'); + writeFileSync(join(support, 'installed-smoke.json'), '{}\n'); + writeFileSync(join(support, 'installed-smokeX'), 'near miss\n'); + mkdirSync(join(support, 'bootstrap')); + const log = join(root, 'events'); + const fixtureFile = join(root, 'fixture-root'); + const pids = ['install.pid', 'bounded.pid', 'grandchild.pid'].map((name) => join(root, name)); + const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); + const script = ` +import { appendFileSync, existsSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { withFixture } from ${url('../cli-bootstrap/child.mjs')}; +import { spawnBounded } from ${url('../cli-bootstrap/process.mjs')}; +import { completeConsumer } from ${url('./consumer-lifetime.mjs')}; +import { isolatedEnvironment, runCommand } from ${url('./process.mjs')}; +const record = (event) => appendFileSync(${JSON.stringify(log)}, event + '\\n'); +const state = (path) => (existsSync(path) ? 'live' : 'gone'); +await completeConsumer({ + supportRoot: ${JSON.stringify(support)}, + exercise: () => + Promise.all([ + runCommand(process.execPath, ['-e', ${JSON.stringify(writer(JSON.stringify(installation), pids[0]))}], { + env: isolatedEnvironment(${JSON.stringify(root)}), + timeoutMs: 60_000, + }), + withFixture((fixture) => { + writeFileSync(${JSON.stringify(fixtureFile)}, fixture.root); + return spawnBounded( + ['-e', ${JSON.stringify(writer('process.env.TMPDIR', pids[1], pids[2]))}], + { env: { TMPDIR: fixture.temporary } }, + { timeoutMs: 60_000 }, + ); + }), + ]), + cleanup: () => { + // Everything the interrupt abandons is gone before the installation's own removal. + record('cleanup fixture:' + state(readFileSync(${JSON.stringify(fixtureFile)}, 'utf8')) + ' smoke:' + state(${JSON.stringify(smoke)})); + if (${cleanupFails}) throw new Error('synthetic cleanup refusal'); + rmSync(${JSON.stringify(installation)}, { recursive: true, force: true, maxRetries: 3 }); + }, + cleanupFailed: () => record('cleanupFailed'), + publish: () => record('publish'), +}); +`; + const consumer = spawn(process.execPath, ['--input-type=module', '-e', script], { + env: { PATH: process.env.PATH, TMPDIR: temporary }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const children = []; + try { + for (const file of pids) children.push(await waitForPid(file)); + const exited = exitWithin(consumer); + consumer.kill('SIGINT'); + const [code, signal] = await exited; + for (let waited = 0; children.some(alive); waited += 25) { + assert.ok(waited < 5_000, 'a child outlived the interrupted consumer'); + await delay(25); + } + return { + code, + signal, + installation, + smoke, + fixture: readFileSync(fixtureFile, 'utf8'), + events: readFileSync(log, 'utf8'), + support: readdirSync(support).sort(), + kept: kept.sort(), + }; + } finally { + if (consumer.exitCode === null && consumer.signalCode === null) consumer.kill('SIGKILL'); + for (const pid of children) if (alive(pid)) process.kill(pid, 'SIGKILL'); + } +} + +test( + 'an interrupted consumer stops every child, abandons the bootstrap and smoke roots, removes its installation and exits', + { skip: process.platform === 'win32' }, + async (t) => { + const outcome = await interruptedConsumer(t); + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 130); + for (const path of [outcome.installation, outcome.smoke, outcome.fixture]) + assert.equal(existsSync(path), false, path); + // Only the worker roots go: every other report entry stays. + assert.deepEqual(outcome.support, outcome.kept); + assert.equal(outcome.events, 'cleanup fixture:gone smoke:gone\n'); + }, +); + +test( + 'a failed removal after an interrupt is only the secondary diagnostic', + { skip: process.platform === 'win32' }, + async (t) => { + const outcome = await interruptedConsumer(t, { cleanupFails: true }); + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 130); + assert.equal(outcome.events, 'cleanup fixture:gone smoke:gone\ncleanupFailed\n'); + }, +); + +test( + 'an interrupt still removes the installation when an earlier abandonment step fails', + { skip: process.platform === 'win32' }, + async (t) => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-consumer-steps-'))); + t.after(() => rmSync(root, { recursive: true, force: true })); + const installation = join(root, 'installation'); + mkdirSync(installation); + const log = join(root, 'events'); + const pidFile = join(root, 'child.pid'); + const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); + // An absolute support directory that does not exist, so abandoning the smoke roots throws. + const script = ` +import { appendFileSync, rmSync } from 'node:fs'; +import { completeConsumer } from ${url('./consumer-lifetime.mjs')}; +import { isolatedEnvironment, runCommand } from ${url('./process.mjs')}; +const record = (event) => appendFileSync(${JSON.stringify(log)}, event + '\\n'); +await completeConsumer({ + supportRoot: ${JSON.stringify(join(root, 'missing-support'))}, + exercise: () => + runCommand(process.execPath, ['-e', ${JSON.stringify(writer(JSON.stringify(installation), pidFile))}], { + env: isolatedEnvironment(${JSON.stringify(root)}), + timeoutMs: 60_000, + }), + cleanup: () => { + rmSync(${JSON.stringify(installation)}, { recursive: true, force: true, maxRetries: 3 }); + record('cleanup'); + }, + cleanupFailed: () => record('cleanupFailed'), + publish: () => record('publish'), +}); +`; + const consumer = spawn(process.execPath, ['--input-type=module', '-e', script], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + let child; + try { + child = await waitForPid(pidFile); + const exited = exitWithin(consumer); + consumer.kill('SIGINT'); + assert.deepEqual(await exited, [130, null]); + assert.equal(existsSync(installation), false); + assert.equal(readFileSync(log, 'utf8'), 'cleanup\ncleanupFailed\n'); + } finally { + if (consumer.exitCode === null && consumer.signalCode === null) consumer.kill('SIGKILL'); + if (child !== undefined && alive(child)) process.kill(child, 'SIGKILL'); + } + }, +); + +test('abandoning the installed smoke removes exactly the roots the worker creates', (t) => { + const support = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-smoke-roots-'))); + t.after(() => rmSync(support, { recursive: true, force: true })); + const roots = [createInstalledSmokeRoot(support), createInstalledSmokeRoot(support)]; + for (const root of roots) { + assert.equal(dirname(root), support); + writeFileSync(join(root, 'guard.ndjson'), '{}\n'); + } + writeFileSync(join(support, 'installed-smoke.json'), '{}\n'); + abandonInstalledSmoke(support); + assert.deepEqual(readdirSync(support), ['installed-smoke.json']); +}); diff --git a/tools/cli-package/container.mjs b/tools/cli-package/container.mjs new file mode 100644 index 00000000..1b5bb159 --- /dev/null +++ b/tools/cli-package/container.mjs @@ -0,0 +1,223 @@ +/** + * The private custody container (Step 9b.1; packaging.md, "Recipient custody" and the private map + * manifest). One strict owner-only USTAR of the sealed staging — `finalized.json`, `manifest.json`, + * `metafile-esm.json` and `maps/.js.map`, staging-relative — so a decrypted, extracted container + * is a staging directory the manifest reader and symbolication consume unchanged. It is built only from + * a staging whose every map is the one the manifest and the finalizer's record name, and read back with + * the same checks. + */ +import { Buffer } from 'node:buffer'; +import { lstatSync, readdirSync } from 'node:fs'; +import { join, posix } from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; +import { CustodyError, refuseCustody } from './custody-error.mjs'; +import { readOrdinaryFile, sha256, byCodeUnit } from './files.mjs'; +import { MANIFEST_FILE, RECORD_BYTES, validateManifest } from './manifest.mjs'; +import { readUstar, writeUstar } from './ustar.mjs'; + +export const CONTAINER_LIMITS = Object.freeze({ + entryBytes: 16 * 1024 * 1024, + // The three records plus one map for each of the manifest's at most 256 outputs. + entries: 259, + totalBytes: 128 * 1024 * 1024, +}); +const RECORDS = Object.freeze(['finalized.json', MANIFEST_FILE, 'metafile-esm.json']); +const MAP = /^maps\/[A-Za-z0-9_-]+\.js\.map$/u; +const ENTRY_MODE = 0o600; + +/** @param {unknown} value @returns {value is Record} */ +function isRecord(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +/** @param {Buffer} bytes @param {string} code */ +function json(bytes, code) { + try { + return JSON.parse(bytes.toString('utf8')); + } catch { + return refuseCustody(code); + } +} + +/** + * The finalizer's record of `{path, bytes, sha256}` entries, sorted by path. + * @param {unknown} list + * @returns {{path: string, bytes: number, sha256: string}[]} + */ +function recorded(list) { + if ( + !Array.isArray(list) || + !list.every( + (item) => + isRecord(item) && + isDeepStrictEqual(Object.keys(item).sort(byCodeUnit), ['bytes', 'path', 'sha256']) && + typeof item.path === 'string' && + Number.isSafeInteger(item.bytes) && + typeof item.sha256 === 'string', + ) + ) { + refuseCustody('CONTAINER_RECORD'); + } + return [...list].sort((left, right) => (left.path < right.path ? -1 : 1)); +} + +/** + * Check a container's entries as a whole: exactly the three records, each within the bound its reader + * applies, and one map per manifest output, each map the manifest's and the finalizer record's, the + * record naming exactly the manifest's files. + * @param {Map} files + */ +function checkContents(files) { + for (const name of RECORDS) if (!files.has(name)) refuseCustody('CONTAINER_SET'); + for (const name of RECORDS) { + if (/** @type {Buffer} */ (files.get(name)).length > RECORD_BYTES[name]) { + refuseCustody('CONTAINER_RECORD'); + } + } + const manifest = (() => { + try { + return validateManifest( + json(/** @type {Buffer} */ (files.get(MANIFEST_FILE)), 'CONTAINER_RECORD'), + ); + } catch { + return refuseCustody('CONTAINER_RECORD'); + } + })(); + const maps = manifest.outputs.map((output) => `maps/${posix.basename(output.map.path)}`); + if ( + !isDeepStrictEqual( + [...files.keys()].filter((path) => !RECORDS.includes(path)).sort(byCodeUnit), + [...maps].sort(byCodeUnit), + ) + ) { + refuseCustody('CONTAINER_SET'); + } + for (const [index, output] of manifest.outputs.entries()) { + if (sha256(/** @type {Buffer} */ (files.get(maps[index]))) !== output.map.sha256) { + refuseCustody('CONTAINER_DRIFT'); + } + } + // The manifest records no digest for the metafile; it must at least be the finalizer's JSON object. + if (!isRecord(json(/** @type {Buffer} */ (files.get('metafile-esm.json')), 'CONTAINER_RECORD'))) { + refuseCustody('CONTAINER_RECORD'); + } + const record = json(/** @type {Buffer} */ (files.get('finalized.json')), 'CONTAINER_RECORD'); + if ( + !isRecord(record) || + !isDeepStrictEqual(Object.keys(record).sort(byCodeUnit), [ + 'maps', + 'outputs', + 'upstreamMaps', + 'version', + ]) || + record.version !== 1 || + !Number.isSafeInteger(record.upstreamMaps) + ) { + refuseCustody('CONTAINER_RECORD'); + } + const outputs = recorded(record.outputs); + const recordedMaps = recorded(record.maps); + if ( + !isDeepStrictEqual( + outputs.map((item) => [item.path, item.sha256]), + manifest.outputs.map((output) => [output.path, output.sha256]), + ) || + !isDeepStrictEqual( + recordedMaps.map((item) => [item.path, item.sha256, item.bytes]), + manifest.outputs.map((output, index) => [ + output.map.path, + output.map.sha256, + /** @type {Buffer} */ (files.get(maps[index])).length, + ]), + ) + ) { + refuseCustody('CONTAINER_RECORD'); + } + return manifest; +} + +/** @param {Map} files */ +function containerBytes(files) { + // A header block per entry, padding to the block, and the two-block end. + return [...files.values()].reduce( + (total, bytes) => total + 512 + Math.ceil(bytes.length / 512) * 512, + 1024, + ); +} + +/** + * Read the sealed staging into one container. The staging must hold exactly the three records and + * `maps/` with exactly the manifest's maps, all regular files within the entry bound. + * @param {string} staging + * @param {typeof CONTAINER_LIMITS} [limits] + * @returns {{bytes: Buffer, manifest: import('./manifest.mjs').PrivateManifest}} + */ +export function buildPrivateContainer(staging, limits = CONTAINER_LIMITS) { + try { + return build(staging, limits); + } catch (error) { + // A host error's message names the staging path: only fixed custody codes leave. + if (error instanceof CustodyError) throw error; + return refuseCustody('CONTAINER_UNREADABLE'); + } +} + +/** @param {string} staging @param {typeof CONTAINER_LIMITS} limits */ +function build(staging, limits) { + /** @param {string} path */ + const read = (path) => { + try { + return readOrdinaryFile(path, limits.entryBytes); + } catch { + return refuseCustody('CONTAINER_ENTRY'); + } + }; + if ( + !isDeepStrictEqual(readdirSync(staging).sort(byCodeUnit), [...RECORDS, 'maps'].sort(byCodeUnit)) + ) { + refuseCustody('CONTAINER_SET'); + } + // lstat: a linked maps directory is not a directory. + if (!lstatSync(join(staging, 'maps')).isDirectory()) refuseCustody('CONTAINER_SET'); + const names = readdirSync(join(staging, 'maps')).sort(); + if (names.length > limits.entries - RECORDS.length) refuseCustody('CONTAINER_SIZE'); + /** @type {Map} */ + const files = new Map(); + for (const name of RECORDS) files.set(name, read(join(staging, name))); + for (const name of names) { + if (!MAP.test(`maps/${name}`)) refuseCustody('CONTAINER_SET'); + files.set(`maps/${name}`, read(join(staging, 'maps', name))); + } + const manifest = checkContents(files); + // Refused before the archive is assembled in memory; the read-back check would refuse it after. + if (containerBytes(files) > limits.totalBytes) refuseCustody('CONTAINER_TOTAL'); + const sorted = [...files].sort(([left], [right]) => (left < right ? -1 : 1)); + const bytes = writeUstar(sorted, () => ENTRY_MODE); + // The written container must read back as exactly these entries. + const again = readPrivateContainer(bytes, limits); + if (!isDeepStrictEqual([...again.files], sorted)) refuseCustody('CONTAINER_SELF_CHECK'); + return { bytes, manifest }; +} + +/** + * Read and check a container: the strict owner-only dialect, the bounds, and the contents as a whole. + * @param {Buffer} bytes + * @param {typeof CONTAINER_LIMITS} [limits] + * @returns {{files: Map, manifest: import('./manifest.mjs').PrivateManifest}} + */ +export function readPrivateContainer(bytes, limits = CONTAINER_LIMITS) { + if (!Buffer.isBuffer(bytes) || bytes.length > limits.totalBytes) refuseCustody('CONTAINER_SIZE'); + /** @type {Map} */ + const files = new Map(); + readUstar(bytes, { + admitPath: (path) => { + if (!RECORDS.includes(path) && !MAP.test(path)) refuseCustody('CONTAINER_PATH'); + }, + modeFor: () => ENTRY_MODE, + maxFiles: limits.entries, + maxFileBytes: limits.entryBytes, + refuse: (code) => refuseCustody(`CONTAINER_${code}`), + onFile: ({ path, contents }) => files.set(path, Buffer.from(contents)), + }); + return { files, manifest: checkContents(files) }; +} diff --git a/tools/cli-package/container.test.mjs b/tools/cli-package/container.test.mjs new file mode 100644 index 00000000..d7ddceb7 --- /dev/null +++ b/tools/cli-package/container.test.mjs @@ -0,0 +1,386 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { Buffer } from 'node:buffer'; +import { + cpSync, + mkdirSync, + readFileSync, + readdirSync, + renameSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { join, posix } from 'node:path'; +import { before, test } from 'node:test'; +import { candidate, fresh, posixModes, seal } from './candidate-fixture.mjs'; +import { CONTAINER_LIMITS, buildPrivateContainer, readPrivateContainer } from './container.mjs'; +import { CustodyError } from './custody-error.mjs'; +import { RECORD_BYTES, readPrivateManifest } from './manifest.mjs'; +import { writeUstar } from './ustar.mjs'; + +const refuses = (code) => (error) => error instanceof CustodyError && error.code === code; + +/** @type {string} */ +let sealed; +before(async () => { + // One synthetic candidate sealed as the producer seals it, plus the metafile the finalizer stages. + const fixture = await candidate(); + await seal(fixture); + writeFileSync(join(fixture.staging, 'metafile-esm.json'), '{"inputs":{},"outputs":{}}'); + sealed = fixture.staging; +}); + +/** A fresh copy of the sealed staging, changed by `change`. */ +function staging(change = () => {}) { + const copy = fresh('container-staging'); + cpSync(sealed, copy, { recursive: true }); + change(copy); + return copy; +} + +const mapName = (copy) => readdirSync(join(copy, 'maps'))[0]; + +test('the sealed staging becomes one container that reads back and extracts to the same staging', () => { + const copy = staging(); + const { bytes, manifest } = buildPrivateContainer(copy); + assert.deepEqual(manifest, readPrivateManifest(copy)); + const read = readPrivateContainer(bytes); + assert.deepEqual(read.manifest, manifest); + const expected = [ + 'finalized.json', + 'manifest.json', + ...readdirSync(join(copy, 'maps')) + .sort() + .map((name) => `maps/${name}`), + 'metafile-esm.json', + ]; + assert.deepEqual([...read.files.keys()], expected); + // Recovery is `gpg -d | tar -x`: the platform tar's extraction is a staging the manifest reader reads. + const archive = join(fresh('container-archive'), 'container.tar'); + mkdirSync(join(archive, '..')); + writeFileSync(archive, bytes); + const out = fresh('container-extracted'); + mkdirSync(out); + assert.equal(spawnSync('tar', ['-xf', archive, '-C', out]).status, 0); + for (const path of expected) { + assert.deepEqual( + readFileSync(join(out, ...path.split('/'))), + readFileSync(join(copy, ...path.split('/'))), + ); + } + assert.deepEqual(readPrivateManifest(out), manifest); +}); + +test('a staging that is not exactly the sealed records and maps refuses', () => { + for (const [code, change] of [ + ['CONTAINER_SET', (copy) => writeFileSync(join(copy, 'notes.txt'), 'extra')], + ['CONTAINER_SET', (copy) => rmSync(join(copy, 'metafile-esm.json'))], + ['CONTAINER_SET', (copy) => rmSync(join(copy, 'manifest.json'))], + ['CONTAINER_SET', (copy) => writeFileSync(join(copy, 'maps', 'extra.js.map'), '{}')], + ['CONTAINER_SET', (copy) => rmSync(join(copy, 'maps', mapName(copy)))], + ['CONTAINER_SET', (copy) => writeFileSync(join(copy, 'maps', 'notes.txt'), '{}')], + // Same count, another valid chunk name. + [ + 'CONTAINER_SET', + (copy) => + renameSync(join(copy, 'maps', mapName(copy)), join(copy, 'maps', 'zzzrenamed.js.map')), + ], + [ + 'CONTAINER_DRIFT', + (copy) => { + const path = join(copy, 'maps', mapName(copy)); + writeFileSync(path, Buffer.concat([readFileSync(path), Buffer.from(' ')])); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.outputs[0].sha256 = '0'.repeat(64); + writeFileSync(path, JSON.stringify(record)); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.outputs[0].path = 'dist/other.js'; + writeFileSync(path, JSON.stringify(record)); + }, + ], + ['CONTAINER_RECORD', (copy) => writeFileSync(join(copy, 'manifest.json'), '{"version":1}')], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.maps[0].bytes += 1; + writeFileSync(path, JSON.stringify(record)); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.maps[0].sha256 = '0'.repeat(64); + writeFileSync(path, JSON.stringify(record)); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.maps[0].path = 'dist/other.js.map'; + writeFileSync(path, JSON.stringify(record)); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.version = 2; + writeFileSync(path, JSON.stringify(record)); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.extra = true; + writeFileSync(path, JSON.stringify(record)); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.upstreamMaps = 1.5; + writeFileSync(path, JSON.stringify(record)); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + delete record.outputs[0].bytes; + writeFileSync(path, JSON.stringify(record)); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.outputs[0].bytes = '1'; + writeFileSync(path, JSON.stringify(record)); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.maps = {}; + writeFileSync(path, JSON.stringify(record)); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.maps[0].extra = true; + writeFileSync(path, JSON.stringify(record)); + }, + ], + ['CONTAINER_RECORD', (copy) => writeFileSync(join(copy, 'finalized.json'), 'null')], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.outputs[0] = null; + writeFileSync(path, JSON.stringify(record)); + }, + ], + ['CONTAINER_RECORD', (copy) => writeFileSync(join(copy, 'metafile-esm.json'), 'not json')], + ['CONTAINER_RECORD', (copy) => writeFileSync(join(copy, 'metafile-esm.json'), '[]')], + ]) { + assert.throws(() => buildPrivateContainer(staging(change)), refuses(code), code); + } +}); + +test('a staging that cannot be read refuses with a fixed code and no path', () => { + const missing = join(fresh('no-staging'), 'staging'); + const file = `${fresh('file-staging')}.json`; + writeFileSync(file, '{}'); + for (const path of [missing, file]) { + assert.throws( + () => buildPrivateContainer(path), + (error) => + error instanceof CustodyError && + error.code === 'CONTAINER_UNREADABLE' && + !error.message.includes(path), + ); + } +}); + +test('a linked record or maps directory refuses', { skip: !posixModes }, () => { + assert.throws( + () => + buildPrivateContainer( + staging((copy) => { + const target = `${fresh('linked-metafile')}.json`; + writeFileSync(target, '{}'); + rmSync(join(copy, 'metafile-esm.json')); + symlinkSync(target, join(copy, 'metafile-esm.json')); + }), + ), + refuses('CONTAINER_ENTRY'), + ); + assert.throws( + () => + buildPrivateContainer( + staging((copy) => { + const maps = fresh('linked-maps'); + cpSync(join(copy, 'maps'), maps, { recursive: true }); + rmSync(join(copy, 'maps'), { recursive: true }); + symlinkSync(maps, join(copy, 'maps')); + }), + ), + refuses('CONTAINER_SET'), + ); +}); + +test('the entry, count and total bounds refuse before a container is written', () => { + const copy = staging(); + const largest = Math.max( + ...['finalized.json', 'manifest.json', 'metafile-esm.json'].map( + (name) => readFileSync(join(copy, name)).length, + ), + ...readdirSync(join(copy, 'maps')).map((name) => readFileSync(join(copy, 'maps', name)).length), + ); + assert.throws( + () => buildPrivateContainer(copy, { ...CONTAINER_LIMITS, entryBytes: largest - 1 }), + refuses('CONTAINER_ENTRY'), + ); + assert.throws( + () => buildPrivateContainer(copy, { ...CONTAINER_LIMITS, entries: 3 }), + refuses('CONTAINER_SIZE'), + ); + const { bytes } = buildPrivateContainer(copy); + // Exactly at every bound is admitted, on building and on reading. + const exact = { + entryBytes: largest, + entries: 3 + readdirSync(join(copy, 'maps')).length, + totalBytes: bytes.length, + }; + assert.deepEqual(buildPrivateContainer(copy, exact).bytes, bytes); + assert.equal(readPrivateContainer(bytes, exact).files.size, exact.entries); + assert.throws( + () => readPrivateContainer(bytes, { ...exact, entries: exact.entries - 1 }), + refuses('CONTAINER_FILE_COUNT'), + ); + // Reading bounds each entry as building does. + assert.throws( + () => readPrivateContainer(bytes, { ...CONTAINER_LIMITS, entryBytes: largest - 1 }), + refuses('CONTAINER_FILE_SIZE'), + ); + // The total is refused before the archive is assembled, not by the read-back check. + assert.throws( + () => buildPrivateContainer(copy, { ...CONTAINER_LIMITS, totalBytes: bytes.length - 1 }), + refuses('CONTAINER_TOTAL'), + ); + assert.throws( + () => readPrivateContainer(bytes, { ...CONTAINER_LIMITS, entries: 3 }), + refuses('CONTAINER_FILE_COUNT'), + ); + assert.throws( + () => readPrivateContainer(bytes, { ...CONTAINER_LIMITS, totalBytes: bytes.length - 1 }), + refuses('CONTAINER_SIZE'), + ); +}); + +test("each record stays within its reader's bound, on building and on reading", () => { + /** @param {string} name @param {number} extra */ + const padded = (name, extra) => + staging((copy) => { + const path = join(copy, name); + const bytes = readFileSync(path); + // Trailing whitespace keeps the JSON valid, so only the size is judged. + const padding = Buffer.alloc(RECORD_BYTES[name] - bytes.length + extra, 0x20); + writeFileSync(path, Buffer.concat([bytes, padding])); + }); + for (const name of ['finalized.json', 'manifest.json', 'metafile-esm.json']) { + buildPrivateContainer(padded(name, 0)); + assert.throws(() => buildPrivateContainer(padded(name, 1)), refuses('CONTAINER_RECORD'), name); + } + // The manifest reader recovery uses refuses that manifest too. + assert.throws( + () => readPrivateManifest(padded('manifest.json', 1)), + (error) => error instanceof Error && 'code' in error && error.code === 'UNREADABLE', + ); + const files = readPrivateContainer(buildPrivateContainer(staging()).bytes).files; + const over = new Map(files); + over.set( + 'manifest.json', + Buffer.concat([files.get('manifest.json'), Buffer.alloc(RECORD_BYTES['manifest.json'], 0x20)]), + ); + assert.throws( + () => readPrivateContainer(writeUstar([...over], () => 0o600)), + refuses('CONTAINER_RECORD'), + ); +}); + +test('a written container that does not read back refuses: the self-check', () => { + // A chunk name too long for the dialect's 100-byte name field is truncated by the writer, and + // only the read-back of the written container notices. + const copy = staging((dir) => { + const manifest = JSON.parse(readFileSync(join(dir, 'manifest.json'), 'utf8')); + const record = JSON.parse(readFileSync(join(dir, 'finalized.json'), 'utf8')); + const [output] = manifest.outputs; + const long = `dist/${'x'.repeat(100)}.js`; + renameSync( + join(dir, 'maps', posix.basename(output.map.path)), + join(dir, 'maps', `${posix.basename(long)}.map`), + ); + for (const item of record.outputs) if (item.path === output.path) item.path = long; + for (const item of record.maps) if (item.path === output.map.path) item.path = `${long}.map`; + output.path = long; + output.map.path = `${long}.map`; + manifest.outputs.sort((left, right) => (left.path < right.path ? -1 : 1)); + writeFileSync(join(dir, 'manifest.json'), JSON.stringify(manifest)); + writeFileSync(join(dir, 'finalized.json'), JSON.stringify(record)); + }); + assert.throws(() => buildPrivateContainer(copy), refuses('CONTAINER_TAR_NAME')); +}); + +test('a container outside the owner-only dialect, or whose contents disagree, refuses on reading', () => { + const files = readPrivateContainer(buildPrivateContainer(staging()).bytes).files; + const write = (entries, mode = 0o600) => writeUstar(entries, () => mode); + assert.throws( + () => readPrivateContainer(write([...files, ['notes.txt', Buffer.from('x')]])), + refuses('CONTAINER_PATH'), + ); + assert.throws( + () => readPrivateContainer(write([...files], 0o644)), + refuses('CONTAINER_TAR_MODE'), + ); + const [map] = [...files.keys()].filter((path) => path.startsWith('maps/')); + const drifted = new Map(files); + drifted.set(map, Buffer.concat([files.get(map), Buffer.from(' ')])); + assert.throws(() => readPrivateContainer(write([...drifted])), refuses('CONTAINER_DRIFT')); + const missing = new Map(files); + missing.delete('metafile-esm.json'); + assert.throws(() => readPrivateContainer(write([...missing])), refuses('CONTAINER_SET')); + assert.throws(() => readPrivateContainer(Uint8Array.from([1])), refuses('CONTAINER_SIZE')); +}); diff --git a/tools/cli-package/custody-error.mjs b/tools/cli-package/custody-error.mjs new file mode 100644 index 00000000..408cb728 --- /dev/null +++ b/tools/cli-package/custody-error.mjs @@ -0,0 +1,14 @@ +/** The custody tools' one refusal (Step 9b.1): a fixed code, never a path, key, map byte or child output. */ +export class CustodyError extends Error { + /** @param {string} code */ + constructor(code) { + super(`PACKED_CLI_CUSTODY_${code}`); + this.name = 'CustodyError'; + this.code = code; + } +} + +/** @param {string} code @returns {never} */ +export function refuseCustody(code) { + throw new CustodyError(code); +} diff --git a/tools/cli-package/custody-fixtures/expired-primary.asc b/tools/cli-package/custody-fixtures/expired-primary.asc new file mode 100644 index 00000000..33d7f7d1 --- /dev/null +++ b/tools/cli-package/custody-fixtures/expired-primary.asc @@ -0,0 +1,15 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEXgvhABYJKwYBBAHaRw8BAQdA4EjzLwY9BoBLpq5V0JHMtZxzJXpNVuK+FSiV +ujKVQzu0MWV4cGlyZWQtcHJpbWFyeSA8ZXhwaXJlZC1wcmltYXJ5QGZpeHR1cmUu +aW52YWxpZD6ItQQTFgoAXRYhBOU2Am1dlOSxXumEAFcN+Q5G2HLdBQJeC+EAGxSA +AAAAAAQADm1hbnUyLDIuNSsxLjEyLDAsMwIbAQUJAAFRgAULCQgHAgIiAgYVCgkI +CwIEFgIDAQIeBwIXgAAKCRBXDfkORthy3fGTAP97iZY7uSZCmDjGdzDTxICFAxcz +08w4FTgMt4MzNq/7JwEA1mX7T+bkq3U3FGQ2ZH3C5JNen041M/WwuS+vYxhDxQO4 +OAReC+EAEgorBgEEAZdVAQUBAQdAEUr8zfwnwvMx8WF4bB6gp1uXpGaL6VNDfi5e +zws9+WADAQgHiJQEGBYKADwWIQTlNgJtXZTksV7phABXDfkORthy3QUCXgvhABsU +gAAAAAAEAA5tYW51MiwyLjUrMS4xMiwwLDMCGwwACgkQVw35DkbYct0epQEAgcTs +Ew7PUofTu7gorjBuJwJKD/KClhgVfwFwJ3GYEW8BAI1xbQo1YLQC4Zv5fFNgZ/9L +kTHVZmTjEzLP32pdp6gC +=mAOL +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/expired-subkey.asc b/tools/cli-package/custody-fixtures/expired-subkey.asc new file mode 100644 index 00000000..110a6b29 --- /dev/null +++ b/tools/cli-package/custody-fixtures/expired-subkey.asc @@ -0,0 +1,15 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEXgvhABYJKwYBBAHaRw8BAQdAT7eN1lZthk5wYgrMnitVnGNPnEsHrYODB0Cu +To0VmuO0L2V4cGlyZWQtc3Via2V5IDxleHBpcmVkLXN1YmtleUBmaXh0dXJlLmlu +dmFsaWQ+iK8EExYKAFcWIQTumbKL6S4sns/QUEN7BuSkoQP2OQUCXgvhABsUgAAA +AAAEAA5tYW51MiwyLjUrMS4xMiwwLDMCGwEFCwkIBwICIgIGFQoJCAsCBBYCAwEC +HgcCF4AACgkQewbkpKED9jlLUAD9HVjUpkciDwaN4d1Uh1E7G7IrktHgf7E1FoPU +70a6iIMA/3t1NSUSlUE68tvdvlCohePX61k4o057Gw3cXUHpXVcAuDgEXgvhABIK +KwYBBAGXVQEFAQEHQBsWrx9liPO7pusMyDmq+LKF4M9W/lUssiD/IEksTtZEAwEI +B4iaBBgWCgBCFiEE7pmyi+kuLJ7P0FBDewbkpKED9jkFAl4L4QAbFIAAAAAABAAO +bWFudTIsMi41KzEuMTIsMCwzAhsMBQkAAVGAAAoJEHsG5KShA/Y52t8BALLdyWEr +bX0wKNvONJrsddx3JrXEayrpeTVqba6KM1+sAP9EcTa2tdIfK1s58sEFQEkw+l4d +cuz9Osg6pZ4hEJvvCg== +=BA/e +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/generate.mjs b/tools/cli-package/custody-fixtures/generate.mjs new file mode 100644 index 00000000..38ac5f59 --- /dev/null +++ b/tools/cli-package/custody-fixtures/generate.mjs @@ -0,0 +1,159 @@ +/** + * Maintainer-only generator of the committed synthetic custody fixtures (Step 9b.1). It is never run in + * CI: generation is not deterministic, so the armored public keys and `pins.json` it writes are + * committed and reviewed instead. Every key is a synthetic OpenPGP v4 key under `fixture.invalid`; no + * secret key leaves the throwaway home, which is removed with its agent stopped. Only this generator + * may fake the system time, to create keys that expired in 2020. + * + * node tools/cli-package/custody-fixtures/generate.mjs + */ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { resolveGpg } from '../gnupg.mjs'; + +const here = fileURLToPath(new URL('.', import.meta.url)); +// A short home: the agent's socket path must fit the platform's limit (104 bytes on macOS). +const home = mkdtempSync(join(tmpdir(), 'rg-')); +// The installation from absolute PATH entries only, as the release path resolves it. +const installation = resolveGpg(); +const PAST = '20200101T000000!'; +const NO_AEAD = 'AES256 AES192 AES SHA512 SHA384 SHA256 ZLIB ZIP Uncompressed'; + +/** @param {string[]} args @param {{input?: string, past?: boolean}} [options] */ +function gpg(args, { input, past = false } = {}) { + const run = spawnSync( + installation.command, + [ + '--homedir', + home, + '--batch', + '--no-tty', + '--pinentry-mode', + 'loopback', + '--passphrase', + '', + ...(past ? ['--faked-system-time', PAST] : []), + ...args, + ], + { encoding: 'utf8', input, env: { PATH: process.env.PATH, GNUPGHOME: home, LANG: 'C' } }, + ); + assert.equal(run.status, 0, `gpg ${args.join(' ')} failed:\n${run.stderr}`); + return run.stdout; +} + +/** The primary and subkey fingerprints, in listing order, of the key with this UID. */ +function fingerprints(uid) { + const fprs = []; + let current; + for (const line of gpg(['--with-colons', '--fixed-list-mode', '--list-keys', uid]).split('\n')) { + const fields = line.split(':'); + if (fields[0] === 'pub' || fields[0] === 'sub') current = fields[0]; + if (fields[0] === 'fpr') fprs.push({ kind: current, fpr: fields[9] }); + } + return { + primary: fprs.find((item) => item.kind === 'pub').fpr, + subkeys: fprs.filter((item) => item.kind === 'sub').map((item) => item.fpr), + }; +} + +/** @param {string} name @param {{past?: boolean, primary?: string, usage?: string}} [options] */ +function key(name, { past = false, primary = 'ed25519', usage = 'cert' } = {}) { + const uid = `${name} <${name}@fixture.invalid>`; + gpg(['--quick-gen-key', uid, primary, usage, past ? '1d' : 'never'], { past }); + return uid; +} + +/** @param {string} uid @param {string} algo @param {{past?: boolean, expire?: string}} [options] */ +function subkey(uid, algo, { past = false, expire = 'never' } = {}) { + gpg(['--quick-add-key', fingerprints(uid).primary, algo, 'encr', expire], { past }); +} + +/** Drop the AEAD feature, so encryption to this key uses SEIPDv1 with MDC (the approved profile). */ +function withoutAead(uid) { + gpg(['--command-fd', '0', '--edit-key', fingerprints(uid).primary], { + input: `setpref ${NO_AEAD}\ny\nsave\n`, + }); +} + +const fixtures = {}; +function record(name, uid, pin = (keys) => keys.subkeys[0]) { + const keys = fingerprints(uid); + writeFileSync(join(here, `${name}.asc`), gpg(['--armor', '--export', keys.primary])); + fixtures[name] = { primary: keys.primary, subkey: pin(keys) }; +} + +try { + let uid = key('good'); + subkey(uid, 'cv25519'); + withoutAead(uid); + record('good', uid); + + uid = key('other'); + subkey(uid, 'cv25519'); + withoutAead(uid); + record('other', uid); + + uid = key('good-aead'); + subkey(uid, 'cv25519'); + record('good-aead', uid); + + uid = key('rsa3072', { primary: 'rsa3072' }); + subkey(uid, 'rsa3072'); + withoutAead(uid); + record('rsa3072', uid); + + uid = key('rsa2048', { primary: 'rsa3072' }); + subkey(uid, 'rsa2048'); + withoutAead(uid); + record('rsa2048', uid); + + uid = key('expired-primary', { past: true }); + subkey(uid, 'cv25519', { past: true }); + record('expired-primary', uid); + + // A subkey cannot be added in the past to a key created now, so both are created in 2020. + uid = `expired-subkey `; + gpg(['--quick-gen-key', uid, 'ed25519', 'cert', 'never'], { past: true }); + subkey(uid, 'cv25519', { past: true, expire: '1d' }); + record('expired-subkey', uid); + + uid = key('revoked-primary'); + subkey(uid, 'cv25519'); + withoutAead(uid); + const revoked = fingerprints(uid).primary; + const certificate = readFileSync(join(home, 'openpgp-revocs.d', `${revoked}.rev`), 'utf8'); + gpg(['--import'], { input: certificate.replace(/^:-----BEGIN/mu, '-----BEGIN') }); + record('revoked-primary', uid); + + uid = key('revoked-subkey'); + subkey(uid, 'cv25519'); + withoutAead(uid); + gpg(['--command-fd', '0', '--edit-key', fingerprints(uid).primary], { + input: 'key 1\nrevkey\ny\n0\n\ny\nsave\n', + }); + record('revoked-subkey', uid); + + uid = key('sign-only', { usage: 'sign' }); + withoutAead(uid); + record('sign-only', uid, (keys) => keys.primary); + + uid = key('two-encryption-subkeys'); + subkey(uid, 'cv25519'); + subkey(uid, 'cv25519'); + withoutAead(uid); + record('two-encryption-subkeys', uid); + + writeFileSync(join(here, 'pins.json'), `${JSON.stringify(fixtures, null, 2)}\n`); + for (const name of readdirSync(here).filter((file) => file.endsWith('.asc'))) { + assert.ok(!readFileSync(join(here, name), 'utf8').includes('PRIVATE KEY'), name); + } +} finally { + const gpgconf = installation.gpgconf.command; + spawnSync(gpgconf, ['--homedir', home, '--kill', 'all'], { stdio: 'ignore' }); + spawnSync(gpgconf, ['--homedir', home, '--remove-socketdir'], { stdio: 'ignore' }); + rmSync(home, { recursive: true, force: true }); +} diff --git a/tools/cli-package/custody-fixtures/good-aead.asc b/tools/cli-package/custody-fixtures/good-aead.asc new file mode 100644 index 00000000..26a01ecf --- /dev/null +++ b/tools/cli-package/custody-fixtures/good-aead.asc @@ -0,0 +1,14 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEat2WvhYJKwYBBAHaRw8BAQdAokSGPLhvXsbRjAjXgDsQIAV/PVVdF8wSQ/k2 +S2rlbxS0JWdvb2QtYWVhZCA8Z29vZC1hZWFkQGZpeHR1cmUuaW52YWxpZD6IrwQT +FgoAVxYhBEDvL/eVF4dpAtpA7QOQEPqAVWbWBQJq3Za+GxSAAAAAAAQADm1hbnUy +LDIuNSsxLjEyLDAsMwIbAQULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAKCRAD +kBD6gFVm1sCuAP45PAWVAvHneGtF1LkTKDgUcAG1ek59lsc3Ip0710fERwD+Ntae +o3P41B1X3bbBpjRfAUXzFqRg59R4qlOWagW4egS4OARq3Za+EgorBgEEAZdVAQUB +AQdAhOq0LdJOTD0hwThHIc95Nq8e0q6l7zU8C/keUEaz+kMDAQgHiJQEGBYKADwW +IQRA7y/3lReHaQLaQO0DkBD6gFVm1gUCat2WvhsUgAAAAAAEAA5tYW51MiwyLjUr +MS4xMiwwLDMCGwwACgkQA5AQ+oBVZtYOrQD/fEmoTIeq9y1uUSdOwAOBaCvsNwIc +Tp8pHASeA+zbK1sBAOLL1uDoEoHE8LCEOzI9rhoT1ygvTGuPRx92zueCBYoG +=AEJk +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/good.asc b/tools/cli-package/custody-fixtures/good.asc new file mode 100644 index 00000000..a1b035f6 --- /dev/null +++ b/tools/cli-package/custody-fixtures/good.asc @@ -0,0 +1,14 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEat2WuxYJKwYBBAHaRw8BAQdAEeSA+thutMGa/RNuUGSIlsrNCdIlvuVfL7vU +0Zt/NhO0G2dvb2QgPGdvb2RAZml4dHVyZS5pbnZhbGlkPoipBBMWCgBRGxSAAAAA +AAQADm1hbnUyLDIuNSsxLjEyLDAsMwIbAQIXgBYhBO4GgcqIG3Sq5zqBtIO2uabd +MVVHBQJq3Za9BAsJCAcEFQoJCAQWAgEAAh4FAAoJEIO2uabdMVVHMwQBALApldn5 +z4MkMfYFH+MecSmW3VTLacxNbV9PePJksnnJAQCYYvTWWTZIavACiUzVznn8h3TI +xAWNMT3lOVRyz+dGAbg4BGrdlrwSCisGAQQBl1UBBQEBB0A1IbBf35QcZVaAsZsB +6dNA0Htnxozzm/aUH1zypb51WwMBCAeIlAQYFgoAPBYhBO4GgcqIG3Sq5zqBtIO2 +uabdMVVHBQJq3Za8GxSAAAAAAAQADm1hbnUyLDIuNSsxLjEyLDAsMwIbDAAKCRCD +trmm3TFVRwdIAP9vkzEvZbkbrrPq+3siG5Zfe+JGgGRFoSP/HkijhC3Z3gD+O98s +oAoEBcHwtAJHDRfBDg8t7uDk3IskjjlCGeFHiAA= +=csdU +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/other.asc b/tools/cli-package/custody-fixtures/other.asc new file mode 100644 index 00000000..bc155cc8 --- /dev/null +++ b/tools/cli-package/custody-fixtures/other.asc @@ -0,0 +1,14 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEat2WvRYJKwYBBAHaRw8BAQdAAmv01jf91maMkpr2vIbVvWYF1fKi7dK6udWh +kZcNbjy0HW90aGVyIDxvdGhlckBmaXh0dXJlLmludmFsaWQ+iKkEExYKAFEbFIAA +AAAABAAObWFudTIsMi41KzEuMTIsMCwzAhsBAheAFiEEIUHtrrAwMYKW0f3tIQmz +3KSCa9UFAmrdlr4ECwkIBwQVCgkIBBYCAQACHgUACgkQIQmz3KSCa9U9VQEA1iuR +FMeHA75QQYyQrXsyFHyekW2w/h+zJfQ4WWieWEIA/jNp8tU/HRDqK+c2ix0yFn2l ++wjppRJa4Dcs4wJytX4BuDgEat2WvRIKKwYBBAGXVQEFAQEHQMnOIYQieOy4V1SF +p2nwvrqda7RFyFvJBzdiZiayCI1hAwEIB4iUBBgWCgA8FiEEIUHtrrAwMYKW0f3t +IQmz3KSCa9UFAmrdlr0bFIAAAAAABAAObWFudTIsMi41KzEuMTIsMCwzAhsMAAoJ +ECEJs9ykgmvV92AA/18cC4M9kz751q9WeRNEgLtoQ+2lPCk9Z7+OroRJnEpyAP4/ +++lEOf/ea4mqlBLzBsY24cPSbuN8Jp6seEbz/+lVBA== +=uHao +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/pins.json b/tools/cli-package/custody-fixtures/pins.json new file mode 100644 index 00000000..ac167578 --- /dev/null +++ b/tools/cli-package/custody-fixtures/pins.json @@ -0,0 +1,46 @@ +{ + "good": { + "primary": "EE0681CA881B74AAE73A81B483B6B9A6DD315547", + "subkey": "F2975EB87A2A368097B3E2107DB14DAAA7F5853C" + }, + "other": { + "primary": "2141EDAEB030318296D1FDED2109B3DCA4826BD5", + "subkey": "0CB8BAB8E07E7FBECF4FB0698460B65F8FE5950C" + }, + "good-aead": { + "primary": "40EF2FF79517876902DA40ED039010FA805566D6", + "subkey": "378B6ADB6E8075395972FB73E06A4FB6DF7C698D" + }, + "rsa3072": { + "primary": "CB42858849CC31B9FD9E57D5CC6C94646B58DF14", + "subkey": "4F263156BC6D479C1155C6BFCABD5BC5AA2A354D" + }, + "rsa2048": { + "primary": "72ADEBD7526307F22475E4BF3E7BB55F4618AE3F", + "subkey": "9E1CB6EF6717695C78DD88EED4A281B5C0659AA8" + }, + "expired-primary": { + "primary": "E536026D5D94E4B15EE98400570DF90E46D872DD", + "subkey": "5063075D48B8F4A2807BA2E70BF054F449288B59" + }, + "expired-subkey": { + "primary": "EE99B28BE92E2C9ECFD050437B06E4A4A103F639", + "subkey": "E391FD0050FC3DFEC2BB9059E5C47BF3D6755E83" + }, + "revoked-primary": { + "primary": "948BC49F079C6015ECF03F93EE9216602EC57C79", + "subkey": "70517C46844F3641AC36ACDE97B2ECCEC091E981" + }, + "revoked-subkey": { + "primary": "C18A44CD2063D08ED2FE89490C0B99F8A32B2B99", + "subkey": "C65206D133BC917C7C74DEDD3497A20162C01EDB" + }, + "sign-only": { + "primary": "169B64C7A5E3F70681DE919275127ED14F938C6B", + "subkey": "169B64C7A5E3F70681DE919275127ED14F938C6B" + }, + "two-encryption-subkeys": { + "primary": "9DC1C32A1AD28FAA2961DA244718609DBBAAAAEC", + "subkey": "88F65CC8EFF54D6573EE9E6861C9EC83CE7526C2" + } +} diff --git a/tools/cli-package/custody-fixtures/revoked-primary.asc b/tools/cli-package/custody-fixtures/revoked-primary.asc new file mode 100644 index 00000000..0822fa2f --- /dev/null +++ b/tools/cli-package/custody-fixtures/revoked-primary.asc @@ -0,0 +1,18 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEat2WwBYJKwYBBAHaRw8BAQdAADIA3uUrKdTYtrCCJGABKLhAHBIJcCnwqwRC +/BfCJa6IjwQgFgoANxYhBJSLxJ8HnGAV7PA/k+6SFmAuxXx5BQJq3ZbAFiMElIvE +nwecYBXs8D+T7pIWYC7FfHkCHQAACgkQ7pIWYC7FfHmjEwEAuW3zR2FJylii03s+ +kk5ntI3TlTcALxptv4zPEbBICzsA/3pjoKqJkfiGFjbxyOjnNuMGNoOwepctwGRt +X4LXmfkAtDFyZXZva2VkLXByaW1hcnkgPHJldm9rZWQtcHJpbWFyeUBmaXh0dXJl +LmludmFsaWQ+iKkEExYKAFEbFIAAAAAABAAObWFudTIsMi41KzEuMTIsMCwzAhsB +AheAFiEElIvEnwecYBXs8D+T7pIWYC7FfHkFAmrdlsEECwkIBwQVCgkIBBYCAQAC +HgUACgkQ7pIWYC7FfHk65wEA1Kbs4+R2h5/aXQ2671g+zGnwsxPb2UbY3b+tlst3 +FjkBAOq+VYgnbxPUVRDmZy30TJJILVZ8m4Xd0tXV3paVZxwEuDgEat2WwBIKKwYB +BAGXVQEFAQEHQFVn2Nk0296KF3+1m68fkqzJKcJo1HvUosITb6mnHMd2AwEIB4iU +BBgWCgA8FiEElIvEnwecYBXs8D+T7pIWYC7FfHkFAmrdlsAbFIAAAAAABAAObWFu +dTIsMi41KzEuMTIsMCwzAhsMAAoJEO6SFmAuxXx5HnoA/R0dH75WZHtTveyk9TsV +lBG2b3wWOLVlqt4Rd/3oXTNQAP9F/fCqWN+eRlcMm0jSg6l5mV+hD8lyCBusDykg +eCrqDg== +=+COW +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/revoked-subkey.asc b/tools/cli-package/custody-fixtures/revoked-subkey.asc new file mode 100644 index 00000000..c9c84283 --- /dev/null +++ b/tools/cli-package/custody-fixtures/revoked-subkey.asc @@ -0,0 +1,17 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEat2WwRYJKwYBBAHaRw8BAQdALhxOGRC0XdinhS/b3a+N50OgHtXop0hOr4Zf +i6Ru1ia0L3Jldm9rZWQtc3Via2V5IDxyZXZva2VkLXN1YmtleUBmaXh0dXJlLmlu +dmFsaWQ+iKkEExYKAFEbFIAAAAAABAAObWFudTIsMi41KzEuMTIsMCwzAhsBAheA +FiEEwYpEzSBj0I7S/olJDAuZ+KMrK5kFAmrdlsIECwkIBwQVCgkIBBYCAQACHgUA +CgkQDAuZ+KMrK5kRHQD/YfjCv+yB3RiDN049UQwMqfbis0367PKR2EC72iYUC/YB +AIkbNskO5fyUPdF/gkHQcHrAm0MtzxZ+xCUltCiWzEYBuDgEat2WwRIKKwYBBAGX +VQEFAQEHQL+g+kiOwZvQdRxFvB20Q4nBUh5hp6UBGmx1wbcN/7AgAwEIB4h4BCgW +CgAgFiEEwYpEzSBj0I7S/olJDAuZ+KMrK5kFAmrdlsMCHQAACgkQDAuZ+KMrK5ll +RgD/RVVXl0lSbq06DiRfa/vl+eMv7Uw8b/c3neReJyy6/XIA/AgcwY3t8LTaaa7c +NbQEWDitBJk51dZA/+tYmI+js18OiJQEGBYKADwWIQTBikTNIGPQjtL+iUkMC5n4 +oysrmQUCat2WwRsUgAAAAAAEAA5tYW51MiwyLjUrMS4xMiwwLDMCGwwACgkQDAuZ ++KMrK5l8lwD/ftWUHj6NO4yLPlqmtaUbJZyCBNXaDhzrs2lRzJjQChkA/2VaOnfV +uNHkWkRQLmpsTXKyzugnsRMxjCdcN8PjgvUB +=MNM3 +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/rsa2048.asc b/tools/cli-package/custody-fixtures/rsa2048.asc new file mode 100644 index 00000000..7235b983 --- /dev/null +++ b/tools/cli-package/custody-fixtures/rsa2048.asc @@ -0,0 +1,39 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQGNBGrdlr8BDADeFkpg6V9UvTWp3T9MwSwqDi6mO71EcBXw8vAWr2WSibFEbdJg +nB2BVPV5GsT+ZaDJzaTjuhcxLcEjeZNHAFh6/6eKFWO4lQ9T1IPovH1gD4REZknp +p4QtcmxONjpFBLxKZt1tfSAWRM7q+fqqCRk+3nVk9HhAslqk+2/cBuVVaQppBYra +QVuh2f/Xs735oaBFw10K/fHnETWtByx1ugQgNzizhvfU7ST820V2oEimya0SQqcR +2WAUgYYBViCdz1j2SbCuXl0m69DaMXbOo3FX6xzCbGdKt9Ydr73g10ZO6zOXA/u/ +014ViO+jG8j8kR16X/FISg8dNGjj6ziB3iONYdloLWOy6yLwXX0Ebw6dgLozsnBa +P2CKw8LTdkQEHoNLxSM5Gb7e8h54w/VYDcdLBgd7jFA7YRfKg/IV5NyoJt9dSJOG +6fpedgk6IEqru7FEssBKHO3oWYrYSoBDOgqxm/VvsL8eEUdsmgFX5XUcR2Qfrsh3 +D9To9xcFTT/jxd0AEQEAAbQhcnNhMjA0OCA8cnNhMjA0OEBmaXh0dXJlLmludmFs +aWQ+iQHnBBMBCABRGxSAAAAAAAQADm1hbnUyLDIuNSsxLjEyLDAsMwIbAQIXgBYh +BHKt69dSYwfyJHXkvz57tV9GGK4/BQJq3ZbABAsJCAcEFQoJCAQWAgEAAh4FAAoJ +ED57tV9GGK4/fT0MAJiy3xH33wdTYWqEhXPiQ5H7U2vZi6EL/DiEF1TNJBtPuIFE +0j5hpHBiBragMvEbz+lpanMBY1639q54deFhWoNkImzw1klO6ANAAFBWbwsFob0w +Is4Mr0tO2INDVAjUyyrtLeF4igF+ybzCatxIn9Rr+liSJO/69ssjjvTaSB+JF/MX +S1dKvP5gHL6yEE8chPYTT9PHGr9k5GE55xW0ptP1tBesOjoHt91JjRW1HE3OEG96 +3h3rNbmk8Sy75SJ1owzaAYuhnFhzQusGg4PfahV7tfhHK255dM+GAUtGrSx4C2Q+ +L6KGaMNInFUQb1k4oBKn40Nso+z+np+5bjimVCC5pMVGHYvMKadq3yC+uZLmOlKo +G0LchhSrpXOuGroCrZ7C1u1BBzmIaV20dsqSCKGe/5d+EEMY3xoxYlerRQcUeIFZ +bQ9+CeC3REXAr0qP+7CUryfxzj7UJI4MkGY0lb8pmlirMk0dttANSDDycMeylAxj +3rLHUITtSSER05OxTLkBDQRq3ZbAAQgA6w7+ogqC8ws0ohCgrtrASoiwBqVHUENE +u7Va7v6w55WH/aqCcs3+FjmmOr7rUMdC3Lh3ktW4s7Y21yF452uzKuFO91ZKoinb +aRwBk/jXv3Wd11/KBC3Or2RD3NJAPhffTeCRcRWe5rg7fQ6r/Luk0VhI5dOeyDF7 +Us1eMFAPlolQrjGo9D67A4v+TTwEKp5LhIYMz30G0apK+y730nT/Fs0Bck1oVbFi +s0w6dG363S8a/k94jwY5csFI+9NR+YnGIIcHWm7113vJxpamzS6wI6OApGXBPWTR ++IPt3G1k04MwYxGDZCSSVOVWKIh1PJ9PIjVUriB5DQ1O4dubJNApRQARAQABiQHS +BBgBCAA8FiEEcq3r11JjB/IkdeS/Pnu1X0YYrj8FAmrdlsAbFIAAAAAABAAObWFu +dTIsMi41KzEuMTIsMCwzAhsMAAoJED57tV9GGK4/UHoL/3fKq0JDRmGl3g4ugABX +kDnjjuXg7YJIM+2SriHrnJKpqi/m+heoKtWWReo/QQAo4iEIK4+LR/9dmo29+FUv +B1XAAb4UBab4vn+mwsiqPkw+yIcgWK9SpNckt5zLU9Hh/HTvG/Is9yUb+caksfZM +De0LxmVu86YRgAPv+L0t3BjX9lqGsPSx6VD/z3FoZeBsYEnCMOuh1OjAyn/iRIh1 +pwJX/ySvd2ngACZBmT/4PwQTwJbnV9X68+JuIJpBKmaXGNurv/6tM00LUlOYuVtV +KanIuE85upkJvp0r8EeuCfhS/Ra757ZL6kaTQoiM9nIhjKflzI2s3ipTZwihaRI/ +dQm4FYjw8d7My6SDEiqwePYwHNcuZ7alXzcZVXufXV5zrrRY1Axuy5pXJ+jWS/Zd +JxjmdPFBW1PXv/R5clvzghB2fUHCI0oaP18zP8XusxYC2lXj5x3Ea024U4tZ4Vn8 +VgP0OaujPsdQEppwTvdDCtcLB6z2aWj3xTEdyPJF92s8lg== +=+VgZ +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/rsa3072.asc b/tools/cli-package/custody-fixtures/rsa3072.asc new file mode 100644 index 00000000..145dcf7a --- /dev/null +++ b/tools/cli-package/custody-fixtures/rsa3072.asc @@ -0,0 +1,42 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQGNBGrdlr4BDACt3W1a3dBJebX1ZlnzR/PgP7MAJPi0Gpz4/ZEs4dl7t9/EjAVn +eJCaZGlfQtj2SwnmURvHWHPXTiLvQVaQcjwa6cfjf2/yIJsZcUcq4vPJjjV1IMGl +lZooVfITRVkBZdOlV/uXh7HBhjW9GQ/l2Jhe6HN4EK96vgk0jf9qWdEqqHjtXnSw +7PPYej2fxCny1kHRDFw5w8sAQmd4ZDmPhbm56G7rbxMqEcDkftdYIC4sarcn7b25 +7iXyDcgmuqyM1Mevi8RdbuCR/WVW6ZnqtylJ6q/iPXyagXFfwcJXWM8lJBU+5lBE +O1uCr+LVlJ/Vjlj1bLETq1rs/YdKd6IGZOgcfS8cOIsy66m7dcHZcdM1h52wiOMn +DTBWzIW6ionfWMtonThRrsSk0d/bM+U6+bCGf+V4jlby2+GLEkezfqP5+hngechA +SWJ4NuUUenwP9cjla12VVSr9KiA4IJoH/sdjUM2lFNXr+iJPqVy9nAxtq0LU8SUn +KKazdRyoD4f83/kAEQEAAbQhcnNhMzA3MiA8cnNhMzA3MkBmaXh0dXJlLmludmFs +aWQ+iQHnBBMBCABRGxSAAAAAAAQADm1hbnUyLDIuNSsxLjEyLDAsMwIbAQIXgBYh +BMtChYhJzDG5/Z5X1cxslGRrWN8UBQJq3Za/BAsJCAcEFQoJCAQWAgEAAh4FAAoJ +EMxslGRrWN8ULnAMAKQU+TngA84brOHZ/Y+8iXLHaIDpnKoBxCSXaKsWM8x7+g7J +PI5R2/Ok2uU1W9HI1genJjXwZnzExf2AoGzb0aXk/FvhjeXHT4sR1L1RNQJc2XAO +uyEf3/vWLDYWyJkS2TGt7d2eYmGWklrxsnvKVcGzc/mVCvY7dACjAditAJPmQ/rH +z41QYn1eKxDLqO2Y7+Dk8JuuisfQHHGCAOmriIGVmB+F+DWKatavx6BhtX6lGArX +n0iwpg1fTu0MpNLemQ6kXmEyVNJO51Gxp87Y5JABhCYxcerti45hB4gIkD3lx8v0 +zBdHnNvdCSzEsR247p4d+uaovvK4XSOUjFP04hPIaBwR/nsoqcinyhYStj7HOBL8 +iQ6ug85t/DWofUk/gEAmb1DCujb6VRSn8NOkDnebnCfQ8jBKW4flhFMPBaBCfviN +O1IrxQBKgvcSHWm2sbpAxy2GQMNFajVtCOOgHMq8ds77X+RDO845qfu0je+vZmkC +vEc7rS6QmMG87kZ89LkBjQRq3Za+AQwAxKgLJFYyz3n6GTymTdRlE3ZDYZgESUE4 +fYBP/APpJ2Nvo/UDAT7+5pgQ2aYba7Ovyzc03AJ0P+yGDttx3hFp25xZRz9n6qw1 +hUCWRDXCT1haYiYOWYuj8+CmuQN5IHJaFHsfF7foWinqPigS/P5Xe4yzGzoPPKpe +WPiA9kKLeUxqir5QKjNUmGIfaUIGrODbwbSST1pm/s842rM1xti6ghXr5pk7Uqq0 +8YzTUg6QJd2X95ssvvC7Ht4+viIRy9ldJnru5h1tuEtBAiU387Zqf+T7nDDKoNyQ +j0yY0IGJ8mBXuHJ4Xbjh87Esn0odhTgBqMan4jQg73Hg0AXlUT+daGEoWvKnVQrt +zDMyn90mMy8JZ9/LfZziNNOriM49+i6v1cf06KsbQkSZ9aXWnJOX+QtA8dfZymZh +h2qxj11mgkTICzDvdwKo1ngQJtn4VGNgYESOpo6kAIOfVig51mGf1MuHq2squo5L +BX2OuhDdTmRGC2ZTbbOaegMEy5wiWrvlABEBAAGJAdIEGAEIADwWIQTLQoWIScwx +uf2eV9XMbJRka1jfFAUCat2WvhsUgAAAAAAEAA5tYW51MiwyLjUrMS4xMiwwLDMC +GwwACgkQzGyUZGtY3xR3Mgv/QJo4zuU6c//uKPLA65zoH7WySjsl/iXwXUgCP7VJ +90LF9pbrfCmCz2uOSM4DhAlrFgtS0G43ZYN1yrfUENbZ1omrje8zPKUAAdsgE/Tq +A2phoxxvCsZF0d/QZtWIce9Til7w2tyBI3xMdg3nHXQ24LNgArQl6YY+MjuXAsIi +WJe4fycqEiyZtnAjkBvastl1D9RTUytqlTbHfjBuSQN2k6x//08FEzCP/qi2Vw1L +9Qi5+vOpOl1ukLtlYMlFMv9jNfn+z7kpOT/CEs9vlCH2LykORxmASurX/FhxmeyR +JkXOdzHR4rTVO/XLltWHHrHvQ1YV/+tLybkAXDFxBbE//Pk4ZBg/fczdkErRiTsJ +NbKGjgacDoUBAusWryN/egkJi+f73VN9YlKTCGueNHM5OEol2i49eTbTQmUeE2Is +G8F+yIY0MvPBvICfLfmgbUlp0hNYnDphQptuwgkXJ8yQlGbONuq8HHcxB/aDoR6R +MZTE5tsCyD89p/Wr9u6XU+Ks +=GO/z +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/sign-only.asc b/tools/cli-package/custody-fixtures/sign-only.asc new file mode 100644 index 00000000..435141b5 --- /dev/null +++ b/tools/cli-package/custody-fixtures/sign-only.asc @@ -0,0 +1,10 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEat2WwxYJKwYBBAHaRw8BAQdAGfsw2NVWKZsXoJ3mayMQoqGRmDGPXWosOe57 +8OCGEQq0JXNpZ24tb25seSA8c2lnbi1vbmx5QGZpeHR1cmUuaW52YWxpZD6IqQQT +FgoAURsUgAAAAAAEAA5tYW51MiwyLjUrMS4xMiwwLDMCGwMCF4AWIQQWm2THpeP3 +BoHekZJ1En7RT5OMawUCat2WxAQLCQgHBBUKCQgEFgIBAAIeBQAKCRB1En7RT5OM +ax7QAQDWs8vQhT20VQif75MxOKuuJ+5F0qhOJasSYNZmpda1ywD/QiAP8TRB+E8X +x9X9DnNPEngLC8u1dBhtuq0pxLkN9Ag= +=K6zy +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/two-encryption-subkeys.asc b/tools/cli-package/custody-fixtures/two-encryption-subkeys.asc new file mode 100644 index 00000000..938105a4 --- /dev/null +++ b/tools/cli-package/custody-fixtures/two-encryption-subkeys.asc @@ -0,0 +1,19 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEat2WxBYJKwYBBAHaRw8BAQdAj9pjkGVc96gIzspDm/Avvhz4Gmw/R05P7h5F +5KP3Jcm0P3R3by1lbmNyeXB0aW9uLXN1YmtleXMgPHR3by1lbmNyeXB0aW9uLXN1 +YmtleXNAZml4dHVyZS5pbnZhbGlkPoipBBMWCgBRGxSAAAAAAAQADm1hbnUyLDIu +NSsxLjEyLDAsMwIbAQIXgBYhBJ3Bwyoa0o+qKWHaJEcYYJ27qqrsBQJq3ZbFBAsJ +CAcEFQoJCAQWAgEAAh4FAAoJEEcYYJ27qqrsDv4BALGZRcIh5gxL8xBuqDW8Cfrv +fMJjKLqEvQLkrLYPqKv8AQC8F5+GeKVO5lP4AQyUq8IcPk/mXoH/zut6W8iRVSXj +BLg4BGrdlsQSCisGAQQBl1UBBQEBB0BF5owCYus+Zt9BlsRveQcp9ajIl/XjGe18 +45wVSTvRdQMBCAeIlAQYFgoAPBYhBJ3Bwyoa0o+qKWHaJEcYYJ27qqrsBQJq3ZbE +GxSAAAAAAAQADm1hbnUyLDIuNSsxLjEyLDAsMwIbDAAKCRBHGGCdu6qq7H44AP9e +OmONEnukZGfF4SpwqVEA2pklQIIf4aV2pqIvZEM2UgD+O3x+FqlA+pf/9+Av4o75 +M146LG+aglO5eeEDKDBDWQy4OARq3ZbEEgorBgEEAZdVAQUBAQdAL0hGquRkWGKp +lwNnMT+knhxxQ509QnwOL90d6yLOrE4DAQgHiJQEGBYKADwWIQSdwcMqGtKPqilh +2iRHGGCdu6qq7AUCat2WxBsUgAAAAAAEAA5tYW51MiwyLjUrMS4xMiwwLDMCGwwA +CgkQRxhgnbuqquwfFwD/SQDfpajkcPFt77Lcvg9WEnNP0MJXLDjB+4GX+EDY62gA +/2TjAiKPad4XtWJ92oJM46C4GF49XxBtB2UzTOlLYZgN +=+Sh7 +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody.mjs b/tools/cli-package/custody.mjs new file mode 100644 index 00000000..138e4a43 --- /dev/null +++ b/tools/cli-package/custody.mjs @@ -0,0 +1,408 @@ +/** + * Recipient custody (Step 9b.1; ADR-0111; packaging.md, "Recipient custody"): encrypt a private + * container once to one pinned recipient. The recipient file must hold exactly one OpenPGP v4 key + * whose only usable encryption subkey is the pinned one, with a cv25519 or RSA ≥ 3072 encryption + * subkey, neither expired, revoked nor disabled; a colon listing classifies it before GnuPG's own + * refusal, a second, independent layer. The ciphertext must be SEIPDv1 with MDC and AES-256, carry + * exactly one recipient — the pinned subkey — and is published once, never over an existing file. + * Plaintext lives only in an owner-only work directory under the caller's private root, removed on + * every path; the caller's interrupt scope stops GnuPG and removes that root. This library holds no + * production recipient: Step 9b uses synthetic recipients only. + */ +import { randomUUID } from 'node:crypto'; +import { + linkSync, + lstatSync, + mkdtempSync, + readdirSync, + realpathSync, + rmSync, + statSync, +} from 'node:fs'; +import { dirname, join } from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; +import { readPrivateContainer } from './container.mjs'; +import { CustodyError, refuseCustody } from './custody-error.mjs'; +import { readOrdinaryFile, sha256, byCodeUnit } from './files.mjs'; +import { writePrivateFile } from './finalize.mjs'; +import { + GNUPG_CALL, + assertHomeState, + assertNoGlobalConfig, + createGnupgHome, + encryptArgs, + gnupgEnvironment, + gnupgVersion, + listArgs, + parseColons, + parseStatus, + recipientsArgs, + resolveGpg, +} from './gnupg.mjs'; +import { runCommand } from './process.mjs'; + +const FINGERPRINT = /^[0-9A-F]{40}$/u; +/** + * An armored secret key, refused from the bytes before any listing: whether GnuPG lists secret input + * (as `sec`/`ssb`, refused too) or fails depends on the home's agent socket path. + */ +const SECRET_ARMOR = ['-----BEGIN PGP', 'PRIVATE KEY BLOCK-----'].join(' '); +const RECIPIENT_BYTES = 256 * 1024; +/** Literal-packet and packet framing overhead a ciphertext may add to its container. */ +const CIPHERTEXT_OVERHEAD = 64 * 1024; +/** The keywords a successful encryption never reports. */ +const FAILURE_STATUS = Object.freeze([ + 'INV_RECP', + 'NO_RECP', + 'FAILURE', + 'ERROR', + 'KEYEXPIRED', + 'KEYREVOKED', + 'INV_SGNR', +]); + +/** The ciphertext's file name for a package version. */ +export const ciphertextName = (version) => `relavium-${version}-private-maps.tar.gpg`; + +/** + * Maintainer-confirmed pins: the primary and encryption-subkey fingerprints, 40 uppercase hex digits + * each (OpenPGP v4). A user ID, a short key ID or a `!` suffix refuses. + * @param {unknown} pins + * @returns {{primary: string, subkey: string}} + */ +export function parsePins(pins) { + if (pins === null || typeof pins !== 'object') return refuseCustody('PIN_FORMAT'); + const { primary, subkey } = /** @type {Record} */ (pins); + if ( + !isDeepStrictEqual(Object.keys(pins).sort(byCodeUnit), ['primary', 'subkey']) || + typeof primary !== 'string' || + typeof subkey !== 'string' || + !FINGERPRINT.test(primary) || + !FINGERPRINT.test(subkey) + ) { + refuseCustody('PIN_FORMAT'); + } + return Object.freeze({ primary, subkey }); +} + +/** + * The keys of a colon listing, each with its fingerprint. + * @param {string[][]} records + */ +function keysOf(records) { + /** @type {{type: string, fields: string[], fpr?: string}[]} */ + const keys = []; + for (const fields of records) { + if (['pub', 'sec', 'sub', 'ssb'].includes(fields[0])) keys.push({ type: fields[0], fields }); + else if (fields[0] === 'fpr' && keys.length > 0 && keys.at(-1).fpr === undefined) { + keys.at(-1).fpr = fields[9]; + } + } + return keys; +} + +/** @param {string[]} fields @param {number} now */ +const expired = (fields, now) => + fields[1] === 'e' || (fields[6] !== '' && Number(fields[6]) <= now); + +/** + * A usable subkey with any of `capabilities`: `e` encrypts, and `r` marks an additional decryption + * subkey (ADSK, GnuPG 2.4.1 and later), which GnuPG adds as a further recipient. + * @param {string[]} fields @param {number} now @param {string} [capabilities] + */ +const usableEncryption = (fields, now, capabilities = 'e') => + !['e', 'r', 'i', 'n', 'd'].includes(fields[1]) && + !expired(fields, now) && + [...capabilities].some((capability) => fields[11].includes(capability)) && + !fields[11].includes('D'); + +/** The approved encryption-subkey algorithms: cv25519 ECDH, or RSA of at least 3072 bits. */ +const approvedAlgorithm = (fields) => + (fields[3] === '18' && fields[16] === 'cv25519') || + (fields[3] === '1' && Number(fields[2]) >= 3072); + +/** + * Classify a recipient listing against its pins, in a fixed order: secret material, a missing primary, + * more than one key, a missing subkey, revocation, expiry, unusability, then another usable + * encryption or additional decryption subkey. + * @param {string[][]} records colon records of `--show-keys` + * @param {{primary: string, subkey: string}} pins + * @param {number} now seconds since the epoch + */ +export function inspectRecipient(records, pins, now) { + const keys = keysOf(records); + if (keys.some((key) => key.type === 'sec' || key.type === 'ssb')) { + refuseCustody('RECIPIENT_SECRET'); + } + const primaries = keys.filter((key) => key.type === 'pub'); + const primary = primaries.find((key) => key.fpr === pins.primary); + if (!primary) refuseCustody('RECIPIENT_MISSING'); + if (primaries.length !== 1) refuseCustody('RECIPIENT_UNEXPECTED'); + const subkeys = keys.filter((key) => key.type === 'sub'); + const pinned = keys.find((key) => key.fpr === pins.subkey); + if (!pinned) refuseCustody('RECIPIENT_MISSING'); + if (primary.fields[1] === 'r' || pinned.fields[1] === 'r') refuseCustody('RECIPIENT_REVOKED'); + if (expired(primary.fields, now) || expired(pinned.fields, now)) { + refuseCustody('RECIPIENT_EXPIRED'); + } + if ( + pinned.type !== 'sub' || + ['i', 'n', 'd'].includes(primary.fields[1]) || + primary.fields[11].includes('D') || + !primary.fields[11].includes('E') || + !usableEncryption(pinned.fields, now) || + Number(primary.fields[5]) > now || + Number(pinned.fields[5]) > now || + !approvedAlgorithm(pinned.fields) + ) { + refuseCustody('RECIPIENT_UNUSABLE'); + } + if (subkeys.some((key) => key !== pinned && usableEncryption(key.fields, now, 'er'))) { + refuseCustody('RECIPIENT_UNEXPECTED'); + } + return pins; +} + +/** @param {string} root @param {string} path */ +function within(root, path) { + // Compared by identity, not spelling: a case-insensitive volume or a bind mount can give one + // directory two spellings that no string comparison relates. + const { dev, ino } = statSync(root, { bigint: true }); + for (let current = path; ; current = dirname(current)) { + const info = statSync(current, { bigint: true }); + if (info.dev === dev && info.ino === ino) return true; + if (dirname(current) === current) return false; + } +} + +/** A directory, not a link; a missing path is not one. */ +function plainDirectory(path) { + try { + return lstatSync(path).isDirectory(); + } catch { + return false; + } +} + +/** + * An owner-only directory of this user, not a link; a missing path is not one. Windows modes say + * nothing about access and no ACL check exists, so no directory there is one. + */ +function privateDirectory(path) { + if (process.platform === 'win32') return false; + let info; + try { + info = lstatSync(path); + } catch { + return false; + } + return info.isDirectory() && (info.mode & 0o077) === 0 && info.uid === process.getuid?.(); +} + +/** + * Publish once and atomically: the bytes go to a fresh sibling, which is then hard-linked to the + * final name. A link never replaces an existing file, and a failed write never leaves a partial + * file under the final name. + * @param {string} directory @param {string} name @param {Buffer} bytes + */ +function publish(directory, name, bytes) { + const temporary = join(directory, `.${name}.${randomUUID()}.partial`); + try { + writePrivateFile(temporary, bytes); + try { + linkSync(temporary, join(directory, name)); + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'EEXIST') { + refuseCustody('ALREADY_ENCRYPTED'); + } + throw error; + } + } finally { + rmSync(temporary, { force: true }); + } +} + +/** + * @typedef {{name: string, bytes: number, sha256: string, + * recipient: {primary: string, subkey: string}, gnupg: string}} CustodyRecord + */ + +/** + * Encrypt one container once to one pinned recipient and publish the ciphertext into `destination`. + * @param {{ + * container: Buffer, + * recipient: {file: string, pins: unknown}, + * destination: string, + * privateRoot: string, + * now?: number, + * run?: typeof runCommand, + * gpg?: import('./gnupg.mjs').GnupgInstallation, + * }} options + * @returns {Promise} + */ +export async function encryptOnce(options) { + /** @type {{work?: string}} */ + const scope = {}; + /** @type {{record?: CustodyRecord, error?: CustodyError}} */ + let outcome; + try { + outcome = { record: await encrypt(options, scope) }; + } catch (error) { + // A host error's message can name a private path: only fixed custody codes leave. + outcome = { error: error instanceof CustodyError ? error : new CustodyError('UNEXPECTED') }; + } + if (scope.work !== undefined) { + try { + rmSync(scope.work, { recursive: true, force: true }); + } catch { + // The plaintext may remain: refuse with a fixed code rather than a host message. + outcome = { error: new CustodyError('CLEANUP') }; + } + } + if (outcome.error) throw outcome.error; + return /** @type {CustodyRecord} */ (outcome.record); +} + +/** + * The encryption itself; `scope.work` names its work directory as soon as it exists. + * @param {Parameters[0]} options + * @param {{work?: string}} scope + * @returns {Promise} + */ +async function encrypt( + { + container, + recipient, + destination, + privateRoot, + now = Math.floor(Date.now() / 1000), + run = runCommand, + gpg, + }, + scope, +) { + const pins = parsePins(recipient.pins); + const { manifest } = readPrivateContainer(container); + const name = ciphertextName(manifest.packageVersion); + if (!plainDirectory(destination)) refuseCustody('PLACEMENT'); + if (readdirSync(destination).includes(name)) refuseCustody('ALREADY_ENCRYPTED'); + if (!privateDirectory(privateRoot)) refuseCustody('PRIVATE_ROOT'); + const destinationPath = realpathSync(destination); + const root = realpathSync(privateRoot); + if (within(root, destinationPath) || within(destinationPath, root)) { + refuseCustody('PLACEMENT'); + } + const binary = gpg ?? resolveGpg(); + + const work = mkdtempSync(join(root, 'custody-')); + scope.work = work; + const env = gnupgEnvironment(work); + const listing = createGnupgHome(work, 'listing'); + const encryption = createGnupgHome(work, 'encryption'); + const recipientsHome = createGnupgHome(work, 'recipients'); + const call = async (args, code) => { + try { + return await run(binary.command, [...binary.prefix, ...args], { ...GNUPG_CALL, env }); + } catch { + return refuseCustody(code); + } + }; + const version = await gnupgVersion(binary, { run, env, home: encryption }); + await assertNoGlobalConfig(binary, { run, env }); + + // Every later step reads this copy, so the bytes checked are the bytes encrypted to. + let key; + try { + key = readOrdinaryFile(recipient.file, RECIPIENT_BYTES); + } catch { + refuseCustody('RECIPIENT_UNREADABLE'); + } + if (key.includes(SECRET_ARMOR)) refuseCustody('RECIPIENT_SECRET'); + const recipientPath = join(work, 'recipient.asc'); + writePrivateFile(recipientPath, key); + const listed = await call(listArgs(listing, recipientPath), 'RECIPIENT_UNREADABLE'); + inspectRecipient(parseColons(String(listed.stdout)), pins, now); + assertHomeState(listing, ['trustdb.gpg']); + + const input = join(work, 'container.tar'); + const output = join(work, 'ciphertext.gpg'); + writePrivateFile(input, container); + const encrypted = await call( + encryptArgs(encryption, recipientPath, input, output), + 'ENCRYPTION_FAILED', + ); + const status = parseStatus(String(encrypted.stdout)); + if (status.some((line) => FAILURE_STATUS.includes(line.keyword))) refuseCustody('STATUS'); + const begins = status.filter((line) => line.keyword === 'BEGIN_ENCRYPTION'); + const ends = status.filter((line) => line.keyword === 'END_ENCRYPTION'); + const considered = status.filter((line) => line.keyword === 'KEY_CONSIDERED'); + if ( + begins.length !== 1 || + ends.length !== 1 || + status.indexOf(ends[0]) < status.indexOf(begins[0]) || + considered.length !== 1 || + !isDeepStrictEqual(considered[0].args.slice(0, 2), [pins.primary, '0']) + ) { + refuseCustody('STATUS'); + } + // SEIPDv1 with MDC (method 2) and AES-256 (algorithm 9); an AEAD algorithm refuses. + if (begins[0].args[0] !== '2' || begins[0].args[1] !== '9' || begins[0].args.length > 2) { + refuseCustody('PROFILE'); + } + assertHomeState(encryption, ['random_seed']); + + if ( + !isDeepStrictEqual(readdirSync(work).sort(), [ + 'ciphertext.gpg', + 'container.tar', + 'encryption', + 'listing', + 'recipient.asc', + 'recipients', + ]) + ) { + refuseCustody('ASSET'); + } + let ciphertext; + try { + ciphertext = readOrdinaryFile(output, container.length + CIPHERTEXT_OVERHEAD); + } catch { + refuseCustody('ASSET'); + } + const listedRecipients = await call(recipientsArgs(recipientsHome, output), 'RECIPIENTS'); + const recipients = parseStatus(String(listedRecipients.stdout)).filter( + (line) => line.keyword === 'ENC_TO', + ); + if ( + recipients.length !== 1 || + // A hidden recipient's all-zero key ID never equals the pinned subkey's. + recipients[0].args[0] !== pins.subkey.slice(-16) + ) { + refuseCustody('RECIPIENTS'); + } + assertHomeState(recipientsHome, []); + // The listed file must still be the bytes about to be published. + let listedBytes; + try { + listedBytes = readOrdinaryFile(output, container.length + CIPHERTEXT_OVERHEAD); + } catch { + refuseCustody('ASSET'); + } + if (!listedBytes.equals(ciphertext)) refuseCustody('ASSET'); + + // The plaintext goes before anything is published, so a failed removal publishes nothing. + try { + rmSync(work, { recursive: true, force: true }); + } catch { + refuseCustody('CLEANUP'); + } + scope.work = undefined; + publish(destinationPath, name, ciphertext); + return Object.freeze({ + name, + bytes: ciphertext.length, + sha256: sha256(ciphertext), + recipient: Object.freeze({ primary: pins.primary, subkey: pins.subkey }), + gnupg: version, + }); +} diff --git a/tools/cli-package/custody.test.mjs b/tools/cli-package/custody.test.mjs new file mode 100644 index 00000000..6126516c --- /dev/null +++ b/tools/cli-package/custody.test.mjs @@ -0,0 +1,1057 @@ +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import { Buffer } from 'node:buffer'; +import { createHash } from 'node:crypto'; +import { + appendFileSync, + chmodSync, + existsSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { basename, dirname, join } from 'node:path'; +import { before, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { candidate, fresh, posixModes, seal } from './candidate-fixture.mjs'; +import { buildPrivateContainer } from './container.mjs'; +import { CustodyError } from './custody-error.mjs'; +import { ciphertextName, encryptOnce, inspectRecipient, parsePins } from './custody.mjs'; +import { + GNUPG_CALL, + createGnupgHome, + gnupgEnvironment, + gnupgVersion, + listArgs, + parseColons, + resolveGpg, +} from './gnupg.mjs'; +import { PINS, ephemeralKey, fixture, processesNaming } from './gpg-fixture.mjs'; +import { readPrivateManifest } from './manifest.mjs'; +import { runCommand } from './process.mjs'; + +const refuses = (code) => (error) => error instanceof CustodyError && error.code === code; +const NOW = Math.floor(Date.now() / 1000); + +/** The installed GnuPG's version, read independently of `encryptOnce`. */ +async function installedVersion() { + const work = fresh('version'); + mkdirSync(work); + const home = createGnupgHome(work, 'home'); + return gnupgVersion(resolveGpg(), { run: runCommand, env: gnupgEnvironment(work), home }); +} + +/** @type {Buffer} */ +let container; +/** @type {string} */ +let staging; +before(async () => { + // A synthetic candidate sealed as the producer seals it: synthetic maps only, never a product's. + const fixtureCandidate = await candidate(); + await seal(fixtureCandidate); + writeFileSync(join(fixtureCandidate.staging, 'metafile-esm.json'), '{"inputs":{},"outputs":{}}'); + staging = fixtureCandidate.staging; + container = buildPrivateContainer(staging).bytes; +}); +/** The contract's ciphertext name, spelled independently of `ciphertextName`. */ +const NAME = () => `relavium-${readPrivateManifest(staging).packageVersion}-private-maps.tar.gpg`; + +/** A fresh owner-only private root and an empty destination. */ +function places() { + const root = fresh('custody-root'); + mkdirSync(root); + chmodSync(root, 0o700); + const destination = fresh('custody-destination'); + mkdirSync(destination); + return { root, destination }; +} + +/** @param {string[]} args */ +const kindOf = (args) => + args.includes('--list-dirs') + ? 'config' + : args.includes('--version') + ? 'version' + : args.includes('--show-keys') + ? 'list' + : args.includes('--encrypt') + ? 'encrypt' + : args.includes('--list-only') + ? 'recipients' + : 'other'; +/** @param {string[]} args @param {string} flag */ +const valueOf = (args, flag) => args[args.indexOf(flag) + 1]; + +/** + * A run port that records every GnuPG call and lets a test act before one, replace it, or rewrite + * its result; everything else is the real GnuPG. + * @param {Record} [hooks] + */ +function recording(hooks = {}) { + const calls = []; + /** @type {string | undefined} */ + let work; + /** @type {typeof runCommand} */ + const run = async (command, args, options) => { + const kind = kindOf(args); + calls.push(kind); + // Every call, the gpgconf query without a home included, runs within the call bounds in the + // isolated C-locale environment of the one work directory that holds every home. + const parent = args.includes('--homedir') ? dirname(valueOf(args, '--homedir')) : undefined; + if (kind === 'version') work = parent; + assert.ok(work !== undefined, 'the version call did not come first'); + assert.ok(parent === undefined || parent === work, 'a home outside the work directory'); + assert.match(basename(work), /^custody-[A-Za-z0-9]{6}$/u); + assert.deepEqual(options.env, gnupgEnvironment(work)); + assert.equal(options.timeoutMs, GNUPG_CALL.timeoutMs); + assert.equal(options.maxBytes, GNUPG_CALL.maxBytes); + const hook = hooks[kind] ?? {}; + if (hook.before) await hook.before(args); + if (hook.replace) return hook.replace(args); + const result = await runCommand(command, args, options); + return hook.after ? hook.after(result, args) : result; + }; + return { run, calls }; +} + +/** One encryption attempt with fresh places; good recipient unless overridden. */ +async function attempt(overrides = {}) { + const { root, destination } = places(); + try { + const record = await encryptOnce({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + destination, + privateRoot: root, + ...overrides, + }); + return { root, destination, record }; + } catch (error) { + return { root, destination, error }; + } +} + +/** A refusal with its code that removed its plaintext work directory and published nothing. */ +function refused(outcome, code) { + assert.ok(outcome.error instanceof CustodyError, `expected ${code}, got ${outcome.error}`); + assert.equal(outcome.error.code, code); + assert.deepEqual(readdirSync(outcome.root), [], 'plaintext work directory left'); + assert.deepEqual(readdirSync(outcome.destination), [], 'something was published'); +} + +/** A colon key record with the shape of a good v4 key unless overridden. */ +function key(type, overrides = {}) { + const fields = Array(20).fill(''); + fields[0] = type; + fields[1] = '-'; + fields[2] = type === 'pub' ? '255' : '255'; + fields[3] = type === 'pub' ? '22' : '18'; + fields[4] = 'KEYID'; + fields[5] = String(NOW - 86_400); + fields[8] = '-'; + fields[11] = type === 'pub' ? 'cEC' : 'e'; + fields[16] = type === 'pub' ? 'ed25519' : 'cv25519'; + for (const [index, value] of Object.entries(overrides)) fields[Number(index)] = value; + return fields; +} +/** @param {string} value */ +const fpr = (value) => Object.assign(Array(11).fill(''), { 0: 'fpr', 9: value }); +const PRIMARY = 'A'.repeat(40); +const SUBKEY = 'B'.repeat(40); +/** A good key's records with the given field overrides on its primary and subkey. */ +const records = (pub = {}, sub = {}) => [ + key('pub', pub), + fpr(PRIMARY), + key('sub', sub), + fpr(SUBKEY), +]; + +test('pins are exactly two 40-digit uppercase fingerprints', () => { + assert.deepEqual(parsePins({ primary: PRIMARY, subkey: SUBKEY }), { + primary: PRIMARY, + subkey: SUBKEY, + }); + for (const pins of [ + undefined, + { primary: PRIMARY }, + { primary: PRIMARY, subkey: SUBKEY, extra: SUBKEY }, + { primary: PRIMARY.toLowerCase(), subkey: SUBKEY }, + { primary: PRIMARY, subkey: `${SUBKEY}!` }, + { primary: PRIMARY, subkey: SUBKEY.slice(-16) }, + { primary: 'good ', subkey: SUBKEY }, + { primary: 'C'.repeat(64), subkey: SUBKEY }, + // A value that only converts to a fingerprint is not one. + { primary: [PRIMARY], subkey: [SUBKEY] }, + { primary: PRIMARY, subkey: [SUBKEY] }, + { primary: { toString: () => PRIMARY }, subkey: SUBKEY }, + ]) { + assert.throws(() => parsePins(pins), refuses('PIN_FORMAT'), JSON.stringify(pins)); + } +}); + +test('the recipient classification refuses in its fixed order', () => { + const pins = { primary: PRIMARY, subkey: SUBKEY }; + assert.equal(inspectRecipient(records(), pins, NOW), pins); + for (const [code, listing, override] of [ + ['RECIPIENT_SECRET', [key('sec'), fpr(PRIMARY), key('ssb'), fpr(SUBKEY)]], + ['RECIPIENT_SECRET', [key('sec'), fpr(PRIMARY), key('sub'), fpr(SUBKEY)]], + ['RECIPIENT_MISSING', []], + ['RECIPIENT_MISSING', records(), { primary: 'C'.repeat(40), subkey: SUBKEY }], + ['RECIPIENT_MISSING', records(), { primary: PRIMARY, subkey: 'C'.repeat(40) }], + ['RECIPIENT_UNEXPECTED', [...records(), key('pub'), fpr('D'.repeat(40))]], + ['RECIPIENT_REVOKED', records({ 1: 'r' })], + ['RECIPIENT_REVOKED', records({}, { 1: 'r' })], + ['RECIPIENT_EXPIRED', records({ 1: 'e' })], + ['RECIPIENT_EXPIRED', records({}, { 6: String(NOW) })], + ['RECIPIENT_UNUSABLE', records({ 1: 'i' })], + ['RECIPIENT_UNUSABLE', records({ 11: 'cECD' })], + ['RECIPIENT_UNUSABLE', records({ 11: 'cC' })], + ['RECIPIENT_UNUSABLE', records({}, { 11: 's' })], + ['RECIPIENT_UNUSABLE', records({}, { 1: 'n' })], + ['RECIPIENT_UNUSABLE', records({}, { 1: 'i' })], + ['RECIPIENT_UNUSABLE', records({}, { 11: 'eD' })], + ['RECIPIENT_UNUSABLE', records({ 1: 'n' })], + ['RECIPIENT_UNUSABLE', records({ 1: 'd' })], + ['RECIPIENT_UNUSABLE', records({ 5: String(NOW + 1) })], + ['RECIPIENT_UNUSABLE', records({}, { 5: String(NOW + 1) })], + ['RECIPIENT_UNUSABLE', records({}, { 16: 'nistp256' })], + ['RECIPIENT_UNUSABLE', records({}, { 3: '1', 2: '2048', 16: '' })], + // An ElGamal subkey of any size is not an approved algorithm. + ['RECIPIENT_UNUSABLE', records({}, { 3: '16', 2: '4096', 16: '' })], + ['RECIPIENT_UNUSABLE', records({}, { 3: '16', 2: '255', 16: 'cv25519' })], + // Revocation is reported before expiry. + ['RECIPIENT_REVOKED', records({ 1: 'r', 6: String(NOW) })], + ['RECIPIENT_SECRET', [...records(), key('ssb'), fpr('E'.repeat(40))]], + [ + 'RECIPIENT_UNUSABLE', + records({ 3: '18', 11: 'ecEC', 16: 'cv25519' }), + { primary: PRIMARY, subkey: PRIMARY }, + ], + ['RECIPIENT_UNEXPECTED', [...records(), key('sub'), fpr('E'.repeat(40))]], + // An additional decryption subkey (capability `r`) is a further recipient GnuPG adds. + ['RECIPIENT_UNEXPECTED', [...records(), key('sub', { 11: 'r' }), fpr('E'.repeat(40))]], + ]) { + assert.throws( + () => inspectRecipient(listing, override ?? pins, NOW), + refuses(code), + `${code} ${JSON.stringify(listing.map((fields) => fields.slice(0, 12)))}`, + ); + } + // RSA of at least 3072 bits is approved; an expired or revoked second subkey is not unexpected. + assert.equal(inspectRecipient(records({}, { 3: '1', 2: '3072', 16: '' }), pins, NOW), pins); + assert.equal( + inspectRecipient([...records(), key('sub', { 1: 'r' }), fpr('E'.repeat(40))], pins, NOW), + pins, + ); + assert.equal( + inspectRecipient([...records(), key('sub', { 1: 'e' }), fpr('E'.repeat(40))], pins, NOW), + pins, + ); + assert.equal( + inspectRecipient( + [...records(), key('sub', { 6: String(NOW) }), fpr('E'.repeat(40))], + pins, + NOW, + ), + pins, + ); + assert.equal(inspectRecipient(records({}, { 3: '1', 2: '4096', 16: '' }), pins, NOW), pins); + // A fingerprint record before any key is ignored. + assert.equal(inspectRecipient([fpr(PRIMARY), ...records()], pins, NOW), pins); + // Created in the current second is not in the future; a revoked additional decryption subkey is + // not a recipient. + assert.equal(inspectRecipient(records({ 5: String(NOW) }, { 5: String(NOW) }), pins, NOW), pins); + assert.equal( + inspectRecipient( + [...records(), key('sub', { 1: 'r', 11: 'r' }), fpr('E'.repeat(40))], + pins, + NOW, + ), + pins, + ); +}); + +test('the committed synthetic fixtures list as their names say under the real GnuPG', async () => { + const gpg = resolveGpg(); + const work = fresh('fixture-listing'); + mkdirSync(work); + let count = 0; + const classify = async (file, pins) => { + const home = createGnupgHome(work, `home-${(count += 1)}`); + const listed = await runCommand(gpg.command, [...gpg.prefix, ...listArgs(home, file)], { + timeoutMs: 60_000, + maxBytes: 256 * 1024, + env: gnupgEnvironment(work), + }); + try { + inspectRecipient(parseColons(listed.stdout), pins, NOW); + return 'ok'; + } catch (error) { + if (error instanceof CustodyError) return error.code; + throw error; + } + }; + for (const [name, expected] of Object.entries({ + good: 'ok', + other: 'ok', + 'good-aead': 'ok', + rsa3072: 'ok', + rsa2048: 'RECIPIENT_UNUSABLE', + 'expired-primary': 'RECIPIENT_EXPIRED', + 'expired-subkey': 'RECIPIENT_EXPIRED', + 'revoked-primary': 'RECIPIENT_REVOKED', + 'revoked-subkey': 'RECIPIENT_REVOKED', + 'sign-only': 'RECIPIENT_UNUSABLE', + 'two-encryption-subkeys': 'RECIPIENT_UNEXPECTED', + })) { + assert.match(PINS[name].primary, /^[0-9A-F]{40}$/u, name); + assert.match(PINS[name].subkey, /^[0-9A-F]{40}$/u, name); + assert.equal(await classify(fixture(name), PINS[name]), expected, name); + } + const both = join(work, 'two-keys.asc'); + writeFileSync( + both, + Buffer.concat([readFileSync(fixture('good')), readFileSync(fixture('other'))]), + ); + assert.equal(await classify(both, PINS.good), 'RECIPIENT_UNEXPECTED'); + assert.equal(await classify(fixture('good'), PINS.other), 'RECIPIENT_MISSING'); + // A rotation that changed the key without updating the subkey pin. + assert.equal( + await classify(fixture('good'), { primary: PINS.good.primary, subkey: PINS.other.subkey }), + 'RECIPIENT_MISSING', + ); +}); + +test('no tracked file holds a PGP private-key block', () => { + const marker = ['BEGIN PGP', 'PRIVATE KEY BLOCK'].join(' '); + const repository = fileURLToPath(new URL('../../', import.meta.url)); + const found = spawnSync('git', ['grep', '-l', '-F', marker], { + cwd: repository, + encoding: 'utf8', + }); + assert.equal(found.status, 1, `tracked private-key blocks:\n${found.stdout}`); +}); + +test('encryption publishes exactly one ciphertext for the pinned subkey and leaves no plaintext', async () => { + const { run, calls } = recording(); + const outcome = await attempt({ run }); + assert.equal(outcome.error, undefined); + const name = NAME(); + assert.deepEqual(readdirSync(outcome.destination), [name]); + const bytes = readFileSync(join(outcome.destination, name)); + assert.deepEqual(outcome.record, { + name, + bytes: bytes.length, + sha256: createHash('sha256').update(bytes).digest('hex'), + recipient: PINS.good, + gnupg: await installedVersion(), + }); + assert.ok(Object.isFrozen(outcome.record) && Object.isFrozen(outcome.record.recipient)); + assert.deepEqual(calls, ['version', 'config', 'list', 'encrypt', 'recipients']); + assert.deepEqual(readdirSync(outcome.root), []); + assert.equal(ciphertextName('1.2.3'), 'relavium-1.2.3-private-maps.tar.gpg'); +}); + +test('an ephemeral key decrypts the ciphertext to the container, and tar extracts the staging', async () => { + const ephemeral = ephemeralKey(); + const outcome = await attempt({ recipient: { file: ephemeral.file, pins: ephemeral.pins } }); + assert.equal(outcome.error, undefined); + const out = fresh('round-trip'); + mkdirSync(out); + const plain = join(out, 'container.tar'); + ephemeral.decrypt(join(outcome.destination, NAME()), plain); + assert.deepEqual(readFileSync(plain), container); + const extracted = join(out, 'staging'); + mkdirSync(extracted); + assert.equal(spawnSync('tar', ['-xf', plain, '-C', extracted]).status, 0); + assert.deepEqual(readPrivateManifest(extracted), readPrivateManifest(staging)); + for (const name of readdirSync(join(staging, 'maps'))) { + assert.deepEqual( + readFileSync(join(extracted, 'maps', name)), + readFileSync(join(staging, 'maps', name)), + ); + } + // Secret key material is never a recipient. + refused( + await attempt({ recipient: { file: ephemeral.secret, pins: ephemeral.pins } }), + 'RECIPIENT_SECRET', + ); + ephemeral.dispose(); + assert.deepEqual(processesNaming(ephemeral.home), [], 'the ephemeral agent is still running'); + assert.equal(existsSync(ephemeral.home), false); +}); + +test('every committed refusal fixture is refused before encryption, and AEAD after it', async () => { + for (const [name, code] of [ + ['expired-primary', 'RECIPIENT_EXPIRED'], + ['expired-subkey', 'RECIPIENT_EXPIRED'], + ['revoked-primary', 'RECIPIENT_REVOKED'], + ['revoked-subkey', 'RECIPIENT_REVOKED'], + ['sign-only', 'RECIPIENT_UNUSABLE'], + ['rsa2048', 'RECIPIENT_UNUSABLE'], + ['two-encryption-subkeys', 'RECIPIENT_UNEXPECTED'], + ]) { + const { run, calls } = recording(); + refused(await attempt({ run, recipient: { file: fixture(name), pins: PINS[name] } }), code); + assert.deepEqual(calls, ['version', 'config', 'list'], name); + } + // GnuPG 2.3 and later pick OCB for a key that advertises AEAD, a profile that is refused; GnuPG 2.2 + // has no AEAD and encrypts SEIPDv1, which the profile check admits. + const [major, minor] = (await installedVersion()).split('.').map(Number); + const aead = await attempt({ + recipient: { file: fixture('good-aead'), pins: PINS['good-aead'] }, + }); + if (major > 2 || (major === 2 && minor >= 3)) refused(aead, 'PROFILE'); + else assert.equal(aead.error, undefined, String(aead.error)); +}); + +test("GnuPG's own refusal is a second layer behind the pre-check", async () => { + // A forged listing passes the pre-check for an expired key; the real GnuPG still refuses it. + const pins = PINS['expired-subkey']; + const forged = [key('pub'), fpr(pins.primary), key('sub'), fpr(pins.subkey)] + .map((fields) => fields.join(':')) + .join('\n'); + const { run, calls } = recording({ list: { replace: () => ({ stdout: forged, stderr: '' }) } }); + refused( + await attempt({ run, recipient: { file: fixture('expired-subkey'), pins } }), + 'ENCRYPTION_FAILED', + ); + assert.deepEqual(calls, ['version', 'config', 'list', 'encrypt']); +}); + +test('the status must be exactly one pinned, completed SEIPDv1 AES-256 encryption', async () => { + const rewrite = (change) => ({ + encrypt: { after: (result) => ({ ...result, stdout: change(result.stdout) }) }, + }); + for (const [code, change] of [ + ['STATUS', (text) => `${text}gpg: encrypted\n`], + ['STATUS', (text) => text.replace(/\[GNUPG:\] END_ENCRYPTION\n/u, '')], + ['STATUS', (text) => text.replace(/(\[GNUPG:\] BEGIN_ENCRYPTION[^\n]*\n)/u, '$1$1')], + [ + 'STATUS', + (text) => + text.replace(/(\[GNUPG:\] BEGIN_ENCRYPTION[^\n]*\n)(\[GNUPG:\] END_ENCRYPTION\n)/u, '$2$1'), + ], + ['STATUS', (text) => `${text}[GNUPG:] INV_RECP 0 recipient\n`], + ['STATUS', (text) => `${text}[GNUPG:] FAILURE encrypt 53\n`], + ...['NO_RECP 0', 'ERROR encrypt 1', 'KEYEXPIRED 1', 'KEYREVOKED', 'INV_SGNR 0 x'].map( + (line) => ['STATUS', (text) => `${text}[GNUPG:] ${line}\n`], + ), + ['STATUS', (text) => `${text}[GNUPG:] END_ENCRYPTION\n`], + ['STATUS', (text) => text.replace(/\[GNUPG:\] KEY_CONSIDERED[^\n]*\n/u, '')], + ['STATUS', (text) => text.replace(/(\[GNUPG:\] KEY_CONSIDERED[^\n]*\n)/u, '$1$1')], + ['STATUS', (text) => `${text}[GNUPG:] KEY_CONSIDERED ${'A'.repeat(40)} 0\n`], + [ + 'STATUS', + (text) => + text.replace(/KEY_CONSIDERED [0-9A-F]{40} 0/u, `KEY_CONSIDERED ${'A'.repeat(40)} 0`), + ], + ['STATUS', (text) => text.replace(/KEY_CONSIDERED ([0-9A-F]{40}) 0/u, 'KEY_CONSIDERED $1 1')], + ['PROFILE', (text) => text.replace(/BEGIN_ENCRYPTION 2 9/u, 'BEGIN_ENCRYPTION 0 9 2')], + ['PROFILE', (text) => text.replace(/BEGIN_ENCRYPTION 2 9/u, 'BEGIN_ENCRYPTION 2 7')], + ['PROFILE', (text) => text.replace(/BEGIN_ENCRYPTION 2 9/u, 'BEGIN_ENCRYPTION 2 9 2')], + ['PROFILE', (text) => text.replace(/BEGIN_ENCRYPTION 2 9/u, 'BEGIN_ENCRYPTION 0 9')], + ['PROFILE', (text) => text.replace(/BEGIN_ENCRYPTION 2 9/u, 'BEGIN_ENCRYPTION 1 9')], + ]) { + refused(await attempt(recording(rewrite(change))), code); + } +}); + +test('the ciphertext must name exactly the pinned subkey and nothing else', async () => { + const rewrite = (change) => ({ + recipients: { after: (result) => ({ ...result, stdout: change(result.stdout) }) }, + }); + for (const change of [ + (text) => text.replace(/(\[GNUPG:\] ENC_TO[^\n]*\n)/u, '$1$1'), + (text) => text.replace(/ENC_TO [0-9A-F]{16}/u, `ENC_TO ${'1'.repeat(16)}`), + (text) => text.replace(/ENC_TO [0-9A-F]{16}/u, `ENC_TO ${'0'.repeat(16)}`), + (text) => text.replace(/\[GNUPG:\] ENC_TO[^\n]*\n/u, ''), + // The pinned subkey with a hidden or another recipient beside it. + (text) => + text.replace(/(\[GNUPG:\] ENC_TO[^\n]*\n)/u, `$1[GNUPG:] ENC_TO ${'0'.repeat(16)} 18 0\n`), + (text) => + text.replace(/(\[GNUPG:\] ENC_TO[^\n]*\n)/u, `$1[GNUPG:] ENC_TO ${'1'.repeat(16)} 18 0\n`), + ]) { + refused(await attempt(recording(rewrite(change))), 'RECIPIENTS'); + } + refused( + await attempt( + recording({ + recipients: { + replace: () => { + throw new Error('listing failed'); + }, + }, + }), + ), + 'RECIPIENTS', + ); +}); + +test('the work directory must hold exactly the expected assets, the ciphertext within its bound', async () => { + const output = (args) => valueOf(args, '--output'); + for (const change of [ + (args) => writeFileSync(join(dirname(output(args)), 'extra'), 'planted'), + (args) => rmSync(output(args)), + (args) => appendFileSync(output(args), Buffer.alloc(container.length + 64 * 1024)), + ]) { + refused( + await attempt( + recording({ + encrypt: { + after: (result, args) => { + change(args); + return result; + }, + }, + }), + ), + 'ASSET', + ); + } +}); + +test('a GnuPG home that gains configuration, keyrings or secrets refuses', async () => { + for (const [kind, name] of [ + ['encrypt', 'common.conf'], + ['encrypt', 'pubring.kbx'], + ['list', 'gpg.conf'], + ['recipients', 'trustdb.gpg'], + ]) { + refused( + await attempt( + recording({ + [kind]: { + after: (result, args) => { + writeFileSync(join(valueOf(args, '--homedir'), name), 'planted'); + return result; + }, + }, + }), + ), + 'HOME_STATE', + ); + } +}); + +test('a failing GnuPG call or an unreadable recipient refuses with a fixed code', async () => { + const fail = () => { + throw new Error('child failed'); + }; + refused(await attempt(recording({ encrypt: { replace: fail } })), 'ENCRYPTION_FAILED'); + refused(await attempt(recording({ list: { replace: fail } })), 'RECIPIENT_UNREADABLE'); + refused(await attempt(recording({ version: { replace: fail } })), 'GPG_UNAVAILABLE'); + refused( + await attempt({ recipient: { file: join(fresh('none'), 'key.asc'), pins: PINS.good } }), + 'RECIPIENT_UNREADABLE', + ); + const empty = join(fresh('empty-recipient'), 'key.asc'); + mkdirSync(dirname(empty)); + writeFileSync(empty, ''); + const outcome = await attempt({ recipient: { file: empty, pins: PINS.good } }); + assert.ok( + ['RECIPIENT_UNREADABLE', 'RECIPIENT_MISSING'].includes(outcome.error?.code), + String(outcome.error), + ); + refused(outcome, outcome.error.code); +}); + +test('the checked recipient copy is the one encrypted to, read within its bound', async () => { + // The caller's file changes after it was read: the checked copy is still the only key used. + const work = fresh('recipient-copy'); + mkdirSync(work); + const original = join(work, 'recipient.asc'); + writeFileSync(original, readFileSync(fixture('good'))); + const seen = []; + const swap = recording({ + list: { + before: (args) => { + seen.push(args.at(-1)); + writeFileSync(original, readFileSync(fixture('other'))); + }, + }, + encrypt: { + before: (args) => seen.push(args[args.indexOf('--recipient-file') + 1]), + }, + }); + const outcome = await attempt({ run: swap.run, recipient: { file: original, pins: PINS.good } }); + assert.equal(outcome.error, undefined, String(outcome.error)); + assert.deepEqual(outcome.record.recipient, PINS.good); + assert.equal(seen.length, 2); + for (const path of seen) { + assert.notEqual(path, original); + assert.equal(path.split(/[\\/]/u).at(-1), 'recipient.asc'); + } + const large = join(work, 'large.asc'); + writeFileSync(large, Buffer.concat([readFileSync(fixture('good')), Buffer.alloc(256 * 1024)])); + refused(await attempt({ recipient: { file: large, pins: PINS.good } }), 'RECIPIENT_UNREADABLE'); +}); + +test("a global GnuPG configuration in the installation's sysconfdir refuses before any key is handled", async () => { + const sysconf = fresh('sysconfdir'); + mkdirSync(sysconf); + const config = (stdout) => ({ config: { replace: () => ({ stdout, stderr: '' }) } }); + // A configuration directory with neither file admits: Homebrew ships only `scdaemon.conf` there, + // and Debian and Ubuntu ship no such directory. + const admitted = recording(config(`${sysconf}\n`)); + const outcome = await attempt(admitted); + assert.equal(outcome.error, undefined, String(outcome.error)); + for (const name of ['gpg.conf', 'common.conf']) { + const directory = fresh(`sysconf-${name}`); + mkdirSync(directory); + writeFileSync(join(directory, name), 'log-file /tmp/leak.log\ndebug-all\n'); + const { run, calls } = recording(config(`${directory}\n`)); + refused(await attempt({ run }), 'GPG_CONFIG'); + assert.deepEqual(calls, ['version', 'config'], name); + } + // gpgconf prints a named directory raw: a space or a literal `%41` is part of the path. + const raw = fresh('sys conf%41'); + mkdirSync(raw); + writeFileSync(join(raw, 'gpg.conf'), 'throw-keyids\n'); + refused(await attempt(recording(config(`${raw}\n`))), 'GPG_CONFIG'); + refused(await attempt(recording(config('relative/dir\n'))), 'GPG_CONFIG'); + refused( + await attempt( + recording({ + config: { + replace: () => { + throw new Error('gpgconf failed'); + }, + }, + }), + ), + 'GPG_UNAVAILABLE', + ); +}); + +test('the published bytes are the listed bytes, and the default clock counts seconds', async () => { + // A ciphertext file replaced after it was read — here by another valid ciphertext for the same + // recipient, so its listing still passes — is refused, never published. + const earlier = await attempt(); + assert.equal(earlier.error, undefined, String(earlier.error)); + const other = readFileSync(join(earlier.destination, NAME())); + const swapped = recording({ + recipients: { before: (args) => writeFileSync(args.at(-1), other) }, + }); + refused(await attempt(swapped), 'ASSET'); + // One that can no longer be read after its listing refuses the same way. + const vanished = recording({ + recipients: { + after: (result, args) => { + rmSync(args.at(-1)); + return result; + }, + }, + }); + refused(await attempt(vanished), 'ASSET'); + // A recipient whose subkey expires tomorrow is valid under the default clock (seconds since the + // epoch); a clock in milliseconds would call it expired. + const pins = PINS.good; + const tomorrow = String(Math.floor(Date.now() / 1000) + 86_400); + const listing = [key('pub'), fpr(pins.primary), key('sub', { 6: tomorrow }), fpr(pins.subkey)] + .map((fields) => fields.join(':')) + .join('\n'); + const { root, destination } = places(); + const forged = recording({ list: { replace: () => ({ stdout: listing, stderr: '' }) } }); + const record = await encryptOnce({ + container, + recipient: { file: fixture('good'), pins }, + destination, + privateRoot: root, + run: forged.run, + }); + assert.equal(record.name, NAME()); +}); + +test('a ciphertext is published once: an existing one refuses before any GnuPG call', async () => { + const { root, destination } = places(); + writeFileSync(join(destination, NAME()), 'earlier ciphertext'); + const { run, calls } = recording(); + await assert.rejects( + encryptOnce({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + destination, + privateRoot: root, + run, + }), + refuses('ALREADY_ENCRYPTED'), + ); + assert.deepEqual(calls, []); + assert.equal(readFileSync(join(destination, NAME()), 'utf8'), 'earlier ciphertext'); + // One that appears while encrypting is never overwritten. + const raced = places(); + const late = recording({ + recipients: { + before: () => writeFileSync(join(raced.destination, NAME()), 'raced ciphertext'), + }, + }); + await assert.rejects( + encryptOnce({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + destination: raced.destination, + privateRoot: raced.root, + run: late.run, + }), + refuses('ALREADY_ENCRYPTED'), + ); + assert.equal(readFileSync(join(raced.destination, NAME()), 'utf8'), 'raced ciphertext'); + assert.deepEqual(readdirSync(raced.destination), [NAME()]); + assert.deepEqual(readdirSync(raced.root), []); +}); + +test('the destination and the private root are separate directories, the root owner-only', async () => { + const { run, calls } = recording(); + const { root, destination } = places(); + const nested = join(root, 'destination'); + mkdirSync(nested); + const options = (overrides) => ({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + run, + ...overrides, + }); + await assert.rejects( + encryptOnce(options({ destination: nested, privateRoot: root })), + refuses('PLACEMENT'), + ); + const inner = join(destination, 'root'); + mkdirSync(inner); + chmodSync(inner, 0o700); + await assert.rejects( + encryptOnce(options({ destination, privateRoot: inner })), + refuses('PLACEMENT'), + ); + if (posixModes) { + const linked = fresh('linked-destination'); + symlinkSync(destination, linked); + await assert.rejects( + encryptOnce(options({ destination: linked, privateRoot: root })), + refuses('PLACEMENT'), + ); + for (const mode of [0o755, 0o750, 0o770]) { + const open = fresh('open-root'); + mkdirSync(open); + chmodSync(open, mode); + await assert.rejects( + encryptOnce(options({ destination, privateRoot: open })), + refuses('PRIVATE_ROOT'), + mode.toString(8), + ); + } + // A link to an owner-only directory is not one. + const target = fresh('linked-root-target'); + mkdirSync(target); + chmodSync(target, 0o700); + const linkedRoot = fresh('linked-root'); + symlinkSync(target, linkedRoot); + await assert.rejects( + encryptOnce(options({ destination, privateRoot: linkedRoot })), + refuses('PRIVATE_ROOT'), + ); + } + await assert.rejects( + encryptOnce( + options({ container: Buffer.from('not a container'), destination, privateRoot: root }), + ), + (error) => error instanceof CustodyError && error.code.startsWith('CONTAINER_'), + ); + assert.deepEqual(calls, []); +}); + +test('a private root of another user, or any on Windows, refuses before any GnuPG call', async () => { + const { root, destination } = places(); + const { run, calls } = recording(); + const attemptHere = () => + encryptOnce({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + destination, + privateRoot: root, + run, + }); + // Windows modes say nothing about access, and no ACL check exists. + const platform = Object.getOwnPropertyDescriptor(process, 'platform'); + Object.defineProperty(process, 'platform', { ...platform, value: 'win32' }); + try { + await assert.rejects(attemptHere(), refuses('PRIVATE_ROOT')); + } finally { + Object.defineProperty(process, 'platform', platform); + } + if (posixModes) { + // An owner-only directory that belongs to someone else is not this user's. + const getuid = process.getuid; + process.getuid = () => getuid.call(process) + 1; + try { + await assert.rejects(attemptHere(), refuses('PRIVATE_ROOT')); + } finally { + process.getuid = getuid; + } + } + assert.deepEqual(calls, []); +}); + +test( + 'the work directory, its homes and its plaintext files are owner-only', + { skip: !posixModes }, + async () => { + /** @type {Record} */ + const modes = {}; + const mode = (path) => statSync(path).mode & 0o777; + const { run } = recording({ + encrypt: { + before: (args) => { + const work = dirname(valueOf(args, '--output')); + modes.work = mode(work); + modes.homes = ['listing', 'encryption', 'recipients'].map((name) => + mode(join(work, name)), + ); + modes.container = mode(args.at(-1)); + modes.recipient = mode(valueOf(args, '--recipient-file')); + }, + }, + }); + const outcome = await attempt({ run }); + assert.equal(outcome.error, undefined, String(outcome.error)); + assert.deepEqual(modes, { + work: 0o700, + homes: [0o700, 0o700, 0o700], + container: 0o600, + recipient: 0o600, + }); + }, +); + +test('an overlap spelled differently on a case-insensitive volume still refuses', async (context) => { + const base = fresh('case'); + mkdirSync(join(base, 'Public'), { recursive: true }); + if (!existsSync(join(base, 'PUBLIC'))) { + context.skip('this volume is case-sensitive'); + return; + } + const root = join(base, 'PUBLIC', 'root'); + mkdirSync(root); + chmodSync(root, 0o700); + const { run, calls } = recording(); + await assert.rejects( + encryptOnce({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + destination: join(base, 'Public'), + privateRoot: root, + run, + }), + refuses('PLACEMENT'), + ); + assert.deepEqual(calls, []); +}); + +test('a missing destination or private root refuses with a fixed code and no path', async () => { + const { root, destination } = places(); + const missing = join(fresh('missing'), 'nowhere'); + // An owner-only regular file is not a private root. + const file = join(fresh('file-root'), 'nowhere'); + mkdirSync(dirname(file)); + writeFileSync(file, '', { mode: 0o600 }); + for (const [code, overrides] of [ + ['PLACEMENT', { destination: missing, privateRoot: root }], + ['PRIVATE_ROOT', { destination, privateRoot: missing }], + ['PRIVATE_ROOT', { destination, privateRoot: file }], + ]) { + await assert.rejects( + encryptOnce({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + ...overrides, + }), + (error) => + error instanceof CustodyError && + error.code === code && + !JSON.stringify({ message: error.message, ...error }).includes('nowhere'), + ); + } +}); + +test( + 'a work directory that cannot be removed refuses with a fixed code', + { skip: !posixModes || process.getuid?.() === 0 }, + async () => { + const { root, destination } = places(); + const locked = recording({ recipients: { before: () => chmodSync(root, 0o500) } }); + try { + await assert.rejects( + encryptOnce({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + destination, + privateRoot: root, + run: locked.run, + }), + (error) => + error instanceof CustodyError && + error.code === 'CLEANUP' && + !JSON.stringify({ message: error.message, ...error }).includes(root), + ); + // The plaintext is removed before anything is published. + assert.deepEqual(readdirSync(destination), []); + } finally { + chmodSync(root, 0o700); + } + // A refusal whose work directory then cannot be removed reports CLEANUP, never its own code. + const failed = places(); + const refusing = recording({ + encrypt: { + after: (result) => { + chmodSync(failed.root, 0o500); + return { ...result, stdout: `${result.stdout}[GNUPG:] FAILURE encrypt 53\n` }; + }, + }, + }); + try { + await assert.rejects( + encryptOnce({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + destination: failed.destination, + privateRoot: failed.root, + run: refusing.run, + }), + (error) => + error instanceof CustodyError && + error.code === 'CLEANUP' && + !JSON.stringify({ message: error.message, ...error }).includes(failed.root), + ); + assert.deepEqual(refusing.calls, ['version', 'config', 'list', 'encrypt']); + assert.equal(readdirSync(failed.root).length, 1, 'the work directory was removed'); + assert.deepEqual(readdirSync(failed.destination), []); + } finally { + chmodSync(failed.root, 0o700); + } + }, +); + +test('no GnuPG output, path or host error text reaches a refusal', async () => { + const marker = join(fresh('marker'), 'private', 'path'); + const leaked = (error) => { + const text = JSON.stringify({ message: error.message, ...error, cause: error.cause }); + assert.equal(text.includes(marker), false, text); + assert.equal(text.includes('fixture.invalid'), false, text); + assert.match(error.message, /^PACKED_CLI_CUSTODY_[A-Z_]+$/u); + }; + const failing = recording({ + encrypt: { + replace: () => { + const error = new Error(`gpg failed at ${marker}`); + Object.assign(error, { + result: { stdout: marker, stderr: `good ${marker}` }, + }); + throw error; + }, + }, + }); + const first = await attempt(failing); + refused(first, 'ENCRYPTION_FAILED'); + leaked(first.error); + // A host failure after encryption is UNEXPECTED, never its message. + const lost = places(); + const vanished = recording({ + recipients: { before: () => rmSync(lost.destination, { recursive: true }) }, + }); + await assert.rejects( + encryptOnce({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + destination: lost.destination, + privateRoot: lost.root, + run: vanished.run, + }), + (error) => { + leaked(error); + return error instanceof CustodyError && error.code === 'UNEXPECTED'; + }, + ); + assert.deepEqual(vanished.calls, ['version', 'config', 'list', 'encrypt', 'recipients']); + assert.deepEqual(readdirSync(lost.root), []); +}); + +test( + 'an interrupt during encryption stops GnuPG, removes the plaintext and publishes nothing', + { skip: !posixModes }, + async () => { + const { root, destination } = places(); + const work = fresh('interrupt'); + mkdirSync(work); + const listing = join(work, 'listing.txt'); + const pins = PINS.good; + writeFileSync( + listing, + [key('pub'), fpr(pins.primary), key('sub'), fpr(pins.subkey)] + .map((fields) => fields.join(':')) + .join('\n'), + ); + const started = join(work, 'started'); + const pidFile = join(work, 'gpg.pid'); + const fakeGpg = join(work, 'gpg.mjs'); + writeFileSync( + fakeGpg, + [ + "import { readFileSync, writeFileSync } from 'node:fs';", + 'const args = process.argv.slice(2);', + `if (args.includes('--list-dirs')) process.stdout.write(${JSON.stringify(work)} + '\\n');`, + "else if (args.includes('--version')) process.stdout.write('gpg (GnuPG) 2.4.4\\n');", + `else if (args.includes('--show-keys')) process.stdout.write(readFileSync(${JSON.stringify(listing)}, 'utf8'));`, + `else { writeFileSync(${JSON.stringify(pidFile)}, String(process.pid)); writeFileSync(${JSON.stringify(started)}, ''); setInterval(() => {}, 1000); }`, + ].join('\n'), + ); + const driver = join(work, 'driver.mjs'); + const here = (name) => JSON.stringify(new URL(`./${name}`, import.meta.url).href); + writeFileSync( + driver, + [ + "import { readFileSync, rmSync } from 'node:fs';", + `import { encryptOnce } from ${here('custody.mjs')};`, + `import { exitOnInterrupt } from ${here('process.mjs')};`, + `const root = ${JSON.stringify(root)};`, + `await exitOnInterrupt(() => rmSync(root, { recursive: true, force: true }), () =>`, + ` encryptOnce({ container: readFileSync(${JSON.stringify(join(work, 'container.tar'))}),`, + ` recipient: { file: ${JSON.stringify(fixture('good'))}, pins: ${JSON.stringify(pins)} },`, + ` destination: ${JSON.stringify(destination)}, privateRoot: root,`, + ` gpg: { command: process.execPath, prefix: [${JSON.stringify(fakeGpg)}],`, + ` gpgconf: { command: process.execPath, prefix: [${JSON.stringify(fakeGpg)}] } } }));`, + ].join('\n'), + ); + writeFileSync(join(work, 'container.tar'), container); + const child = spawn(process.execPath, [driver], { stdio: 'ignore' }); + const closed = new Promise((resolve) => + child.once('close', (code, signal) => resolve({ code, signal })), + ); + for (let index = 0; index < 400 && !existsSync(started); index += 1) { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + assert.ok(existsSync(started), 'the stand-in never started encrypting'); + const gpgPid = Number(readFileSync(pidFile, 'utf8')); + child.kill('SIGINT'); + assert.deepEqual(await closed, { code: 130, signal: null }); + assert.equal(existsSync(root), false, 'the private root and its plaintext survived'); + assert.deepEqual(readdirSync(destination), []); + // The group got SIGKILL before the driver exited; its member is reaped shortly after. + const alive = () => { + try { + process.kill(gpgPid, 0); + return true; + } catch (error) { + assert.equal(error.code, 'ESRCH'); + return false; + } + }; + for (let index = 0; index < 500 && alive(); index += 1) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.equal(alive(), false, 'the stopped GnuPG stand-in outlived its group kill'); + }, +); diff --git a/tools/cli-package/files.mjs b/tools/cli-package/files.mjs index e63455ef..6d44f2dd 100644 --- a/tools/cli-package/files.mjs +++ b/tools/cli-package/files.mjs @@ -14,6 +14,18 @@ import { join } from 'node:path'; export const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex'); +/** + * Code-unit order for a string sort: deterministic and locale-independent, exactly what `Array#sort` does + * without a comparator. Manifests, inventories and digests depend on that order, so `localeCompare` would + * be wrong here; the explicit comparator only states it. + * @param {string} left + * @param {string} right + */ +export function byCodeUnit(left, right) { + if (left < right) return -1; + return left > right ? 1 : 0; +} + export function readOrdinaryFile(path, maxBytes) { assert.ok(Number.isSafeInteger(maxBytes) && maxBytes > 0); const info = lstatSync(path, { bigint: true }); diff --git a/tools/cli-package/finalize.mjs b/tools/cli-package/finalize.mjs new file mode 100644 index 00000000..37ae64ad --- /dev/null +++ b/tools/cli-package/finalize.mjs @@ -0,0 +1,563 @@ +/** + * Candidate finalizer (ADR-0111, Steps 9a.1 and 9a.2a). It runs inside the CLI's prepack after the + * sole candidate tsup build and before pnpm packs. It accounts for every emitted output, validates + * every consumed upstream TypeScript map and every final map — decoding their mappings strictly and + * keeping every position inside its generated and original text (source-map.mjs) — rewrites final + * map sources to repository-relative original TypeScript, removes only tsup's exact terminal map-link + * trailers, stages the maps and the build metadata owner-only outside the package, re-checks the + * finite closure and the public file/mode inventory, and only then hashes. Nothing writes JS + * afterwards. + * + * Refusals name a fixed code and at most a fixed label or a repository-relative path that has passed + * validation, never map or source contents and never a host path: a metafile string that escapes + * the repository or is not a plain path is left out. A successful bundle is no evidence by itself: a + * missing upstream map makes the bundler silently map engine frames to compiled JS, so every source + * set is checked explicitly. + */ +import { Buffer } from 'node:buffer'; +import { + chmodSync, + closeSync, + fchmodSync, + fsyncSync, + lstatSync, + mkdirSync, + openSync, + readdirSync, + realpathSync, + renameSync, + unlinkSync, + writeSync, +} from 'node:fs'; +import { basename, isAbsolute, join, posix } from 'node:path'; +import { TextDecoder, isDeepStrictEqual } from 'node:util'; +import { checkBundleClosure } from '../bundle-closure/closure.mjs'; +import { MODE_EVIDENCE, emittedInventory } from '../bundle-closure/inventory.mjs'; +import { flatFiles, readOrdinaryFile, sha256, byCodeUnit } from './files.mjs'; +import { within } from './paths.mjs'; +import { + SourceMapError, + assertGeneratedRange, + assertOriginalRange, + decodeMappings, + lineLengths, +} from './source-map.mjs'; + +/** + * Names the private staging directory of a trusted candidate build. Only the package producer sets + * it; its presence alone selects candidate maps in the CLI build and the finalizer in prepack. + */ +export const CANDIDATE_STAGING_ENV = 'RELAVIUM_CLI_CANDIDATE_STAGING'; + +/** The inlined engine packages, in their dependency build order. */ +export const ENGINE_PACKAGES = Object.freeze(['shared', 'llm', 'core', 'db', 'mcp']); +const LIMITS = Object.freeze({ + metafileBytes: 4 * 1024 * 1024, + jsBytes: 8 * 1024 * 1024, + upstreamMapBytes: 4 * 1024 * 1024, + finalMapBytes: 16 * 1024 * 1024, + outputs: 256, + sourcesPerMap: 4096, + namesPerMap: 1_000_000, +}); +const SOURCE_TAIL = /^(?:[A-Za-z0-9_-][A-Za-z0-9_.-]*\/)*[A-Za-z0-9_-][A-Za-z0-9_.-]*\.tsx?$/u; +const MAPPINGS = /^[A-Za-z0-9+/,;]*$/u; +const PLAIN_PATH = /^(?:[A-Za-z0-9_-][A-Za-z0-9_.-]*\/)*[A-Za-z0-9_-][A-Za-z0-9_.-]*$/u; + +export class CandidateFinalizationError extends Error { + /** @param {string} code @param {string} [subject] a fixed label or a validated repository-relative path */ + constructor(code, subject) { + super( + `CLI candidate finalization refused: ${code}` + + (subject === undefined ? '' : ` (${subject})`), + ); + this.name = 'CandidateFinalizationError'; + this.code = code; + } +} + +/** @param {string} code @param {string} [subject] @returns {never} */ +function refuse(code, subject) { + throw new CandidateFinalizationError(code, subject); +} + +/** @param {unknown} value @returns {value is Record} */ +function isRecord(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +/** The map-link trailer tsc and tsup write for an external map beside its JS file. */ +export const mapTrailer = (/** @type {string} */ jsName) => `//# sourceMappingURL=${jsName}.map`; + +/** @param {string} root @param {string} path repository-relative POSIX path */ +function onDisk(root, path) { + return join(root, ...path.split('/')); +} + +/** @param {string} path @param {number} limit @param {string} subject */ +function readBytes(path, limit, subject) { + try { + return readOrdinaryFile(path, limit); + } catch { + refuse('UNREADABLE_FILE', subject); + } +} + +/** @param {Uint8Array} bytes @param {string} subject */ +function parseJson(bytes, subject) { + try { + return JSON.parse(new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(bytes)); + } catch { + refuse('MALFORMED_JSON', subject); + } +} + +/** @param {string} path @param {number} limit @param {string} subject */ +function readJson(path, limit, subject) { + return parseJson(readBytes(path, limit, subject), subject); +} + +/** @param {string} path @param {string} subject */ +function readText(path, subject) { + try { + return new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode( + readOrdinaryFile(path, LIMITS.jsBytes), + ); + } catch { + refuse('UNREADABLE_FILE', subject); + } +} + +/** + * Resolve one map source against its map's repository-relative directory and admit only original + * TypeScript inside `allowedRoots`. URLs, absolute or escaping paths and anything but a `.ts`/`.tsx` + * source refuse — a JS source means the bundler fell back to compiled output. + * @param {string} repository + * @param {string} mapDirectory repository-relative POSIX directory of the map file + * @param {unknown} source + * @param {readonly string[]} allowedRoots repository-relative POSIX source roots + * @param {string} subject + */ +function originalSource(repository, mapDirectory, source, allowedRoots, subject) { + if ( + typeof source !== 'string' || + source.length === 0 || + source.length > 1024 || + /[\\%?#]/u.test(source) || + [...source].some((character) => (character.codePointAt(0) ?? 0) < 32) || + source.startsWith('/') || + /^[A-Za-z][A-Za-z0-9+.-]*:/u.test(source) + ) { + refuse('MAP_SOURCE_PATH', subject); + } + const resolved = posix.normalize(posix.join(mapDirectory, source)); + const root = allowedRoots.find((candidate) => resolved.startsWith(`${candidate}/`)); + if ( + root === undefined || + resolved.endsWith('.d.ts') || + !SOURCE_TAIL.test(resolved.slice(root.length + 1)) + ) { + refuse('MAP_SOURCE_NOT_ORIGINAL', subject); + } + let info; + try { + info = lstatSync(onDisk(repository, resolved)); + } catch { + refuse('MAP_SOURCE_MISSING', subject); + } + if (!info.isFile()) refuse('MAP_SOURCE_MISSING', subject); + return resolved; +} + +/** + * Decode a map's mappings strictly and require every generated position inside the text it was + * generated for and every original position inside its source: a well-formed charset alone admits + * truncated values, out-of-range indices and positions past a file's end, which the bundler would + * otherwise drop or chain silently. + * @param {string} repository + * @param {{mappings: string, sources: unknown[], names: unknown[]}} map + * @param {string} generated the generated text + * @param {readonly string[]} sources repository-relative original sources, by index + * @param {Map} texts source line lengths, shared by every map of one run + * @param {'UPSTREAM_MAP' | 'MAP'} kind @param {string} subject + */ +function checkMappings(repository, map, generated, sources, texts, kind, subject) { + /** @param {() => void} check @param {string} code */ + const strictly = (check, code) => { + try { + check(); + } catch (error) { + if (error instanceof SourceMapError) refuse(code, subject); + throw error; + } + }; + let lines = []; + strictly(() => { + lines = decodeMappings(map.mappings, { sources: map.sources.length, names: map.names.length }); + }, `${kind}_MAPPINGS`); + const originals = sources.map((source) => { + let lengths = texts.get(source); + if (lengths === undefined) { + lengths = lineLengths(readText(onDisk(repository, source), source)); + texts.set(source, lengths); + } + return lengths; + }); + strictly(() => { + assertGeneratedRange(lines, lineLengths(generated)); + assertOriginalRange(lines, originals); + }, `${kind}_RANGE`); +} + +/** + * Validate one consumed upstream (tsc) map. The JS must end with its exact trailer and the map must + * name only this package's original TypeScript, without embedded sources or a source root. Its + * mappings may be empty only for a module whose types all erase, which tsc emits as exactly + * `export {};`. + * @param {string} repository + * @param {string} packageName + * @param {string} jsPath repository-relative POSIX path of the consumed compiled JS + * @param {Map} texts source line lengths, shared by every map of one run + */ +function upstreamSources(repository, packageName, jsPath, texts) { + const name = posix.basename(jsPath); + const js = readText(onDisk(repository, jsPath), jsPath); + if (!js.endsWith(mapTrailer(name)) || js.split('sourceMappingURL').length !== 2) { + refuse('UPSTREAM_MAP_LINK', jsPath); + } + const mapPath = `${jsPath}.map`; + let exists = false; + try { + exists = lstatSync(onDisk(repository, mapPath)).isFile(); + } catch { + exists = false; + } + if (!exists) refuse('UPSTREAM_MAP_MISSING', jsPath); + const map = readJson(onDisk(repository, mapPath), LIMITS.upstreamMapBytes, mapPath); + if ( + !isRecord(map) || + !isDeepStrictEqual(Object.keys(map).sort(byCodeUnit), [ + 'file', + 'mappings', + 'names', + 'sourceRoot', + 'sources', + 'version', + ]) || + map.version !== 3 || + map.file !== name || + map.sourceRoot !== '' || + typeof map.mappings !== 'string' || + !MAPPINGS.test(map.mappings) || + !Array.isArray(map.names) || + map.names.some((item) => typeof item !== 'string') || + !Array.isArray(map.sources) || + map.sources.length === 0 || + map.sources.length > LIMITS.sourcesPerMap + ) { + refuse('UPSTREAM_MAP_SHAPE', mapPath); + } + if (map.mappings === '' && js !== `export {};\n${mapTrailer(name)}`) { + refuse('UPSTREAM_MAP_UNMAPPED', mapPath); + } + const directory = posix.dirname(jsPath); + const sources = map.sources.map((source) => + originalSource(repository, directory, source, [`packages/${packageName}/src`], mapPath), + ); + checkMappings(repository, map, js, sources, texts, 'UPSTREAM_MAP', mapPath); + return sources; +} + +/** + * A metafile input (relative to apps/cli) as a refusal subject: its repository-relative path when + * that is a plain path inside the repository, otherwise nothing. + * @param {string} input + */ +function inputSubject(input) { + if (input.startsWith('/')) return undefined; + const path = posix.normalize(posix.join('apps/cli', input)); + return path.length <= 200 && PLAIN_PATH.test(path) ? path : undefined; +} + +/** + * Original TypeScript sources behind one metafile input: the CLI's own source directly, an engine + * package's compiled JS through its tsc map. Any other input kind refuses. + * @param {string} repository @param {string} input metafile input relative to apps/cli + * @param {Map} cache + * @param {Map} texts source line lengths, shared by every map of one run + */ +function inputSources(repository, input, cache, texts) { + const cached = cache.get(input); + if (cached) return cached; + let sources; + const engine = /^\.\.\/\.\.\/packages\/([a-z]+)\/dist\/(.+\.js)$/u.exec(input); + if (input.startsWith('src/') && SOURCE_TAIL.test(input.slice(4)) && !input.endsWith('.d.ts')) { + sources = [ + originalSource(repository, 'apps/cli', input, ['apps/cli/src'], `apps/cli/${input}`), + ]; + } else if ( + engine && + ENGINE_PACKAGES.includes(engine[1]) && + /^(?:[A-Za-z0-9_-][A-Za-z0-9_.-]*\/)*[A-Za-z0-9_-][A-Za-z0-9_.-]*\.js$/u.test(engine[2]) + ) { + sources = upstreamSources( + repository, + engine[1], + `packages/${engine[1]}/dist/${engine[2]}`, + texts, + ); + } else { + refuse('UNSUPPORTED_INPUT', inputSubject(input)); + } + cache.set(input, sources); + return sources; +} + +/** + * Validate one final map against the original sources of the inputs that contributed bytes to its + * runtime output, and return it with repository-relative sources. Mappings and names are kept. + * @param {string} repository @param {string} jsName @param {unknown} map @param {Set} expected + * @param {string} code the runtime output without its map-link trailer + * @param {Map} texts source line lengths, shared by every map of one run + */ +function finalMap(repository, jsName, map, expected, code, texts) { + const subject = `apps/cli/dist/${jsName}.map`; + const keys = isRecord(map) ? Object.keys(map) : []; + if ( + !isRecord(map) || + !['mappings', 'names', 'sources', 'version'].every((key) => keys.includes(key)) || + keys.some( + (key) => !['file', 'mappings', 'names', 'sourceRoot', 'sources', 'version'].includes(key), + ) + ) { + // sourcesContent, an index map's sections or any unreviewed field refuses. + refuse(isRecord(map) && 'sourcesContent' in map ? 'EMBEDDED_SOURCES' : 'MAP_SHAPE', subject); + } + if ( + map.version !== 3 || + (map.file !== undefined && map.file !== jsName) || + (map.sourceRoot !== undefined && map.sourceRoot !== '') || + typeof map.mappings !== 'string' || + !MAPPINGS.test(map.mappings) || + !Array.isArray(map.names) || + map.names.length > LIMITS.namesPerMap || + map.names.some((item) => typeof item !== 'string') || + !Array.isArray(map.sources) || + map.sources.length > LIMITS.sourcesPerMap || + // A code-splitting facade that only re-exports other chunks has no contributing input and an + // empty map; any output with contributing inputs must map them. + (expected.size === 0 + ? map.sources.length !== 0 || map.mappings !== '' || map.names.length !== 0 + : map.mappings.length === 0) + ) { + refuse('MAP_SHAPE', subject); + } + const sources = map.sources.map((source) => + originalSource( + repository, + 'apps/cli/dist', + source, + ['apps/cli/src', ...ENGINE_PACKAGES.map((name) => `packages/${name}/src`)], + subject, + ), + ); + if (new Set(sources).size !== sources.length) refuse('MAP_SOURCE_DUPLICATE', subject); + if (!isDeepStrictEqual([...sources].sort(byCodeUnit), [...expected].sort(byCodeUnit))) { + refuse('MAP_SOURCE_COVERAGE', subject); + } + checkMappings(repository, map, code, sources, texts, 'MAP', subject); + return { ...map, sources }; +} + +/** + * A private staging directory must be absolute and canonical (no link in its path and the letter case + * the filesystem stores, `realpathSync.native`), empty, an ordinary directory outside the candidate + * repository and, where POSIX modes are observable, owner-only and owned by this process. The + * repository is compared by its native real path too, so no alias of either side (a link or, on a + * case-insensitive filesystem, another letter case) lets staging sit inside the candidate. + * @param {string} staging @param {string} repository + */ +export function assertPrivateStaging(staging, repository) { + if (typeof staging !== 'string' || !isAbsolute(staging)) refuse('STAGING_PATH'); + let info; + let real; + let root; + try { + info = lstatSync(staging); + real = realpathSync.native(staging); + root = realpathSync.native(repository); + } catch { + refuse('STAGING_PATH'); + } + if ( + !info.isDirectory() || + real !== staging || + within(root, real) || + within(real, root) || + within(repository, staging) + ) { + refuse('STAGING_PATH'); + } + if (readdirSync(staging).length !== 0) refuse('STAGING_NOT_EMPTY'); + if ( + MODE_EVIDENCE === 'posix' && + ((info.mode & 0o777) !== 0o700 || + (typeof process.getuid === 'function' && info.uid !== process.getuid())) + ) { + refuse('STAGING_NOT_PRIVATE'); + } +} + +/** Create a new file with an exact mode (independent of the umask) and flush its bytes. */ +function writeNew(path, bytes, mode) { + const descriptor = openSync(path, 'wx', mode); + try { + if (MODE_EVIDENCE === 'posix') fchmodSync(descriptor, mode); + let offset = 0; + while (offset < bytes.length) offset += writeSync(descriptor, bytes, offset); + fsyncSync(descriptor); + } finally { + closeSync(descriptor); + } +} + +/** Write bytes to a new owner-only staging file; an existing file refuses. */ +export function writePrivateFile(path, bytes) { + writeNew(path, bytes, 0o600); +} + +/** Replace one runtime output in place without moving any executable line or column. */ +function replaceOutput(path, bytes, mode) { + const temporary = `${path}.finalizing`; + writeNew(temporary, bytes, mode); + renameSync(temporary, path); +} + +/** + * Finalize the candidate CLI build in `/apps/cli` into `staging`. + * @param {{repository: string, staging: string}} options absolute candidate root and private staging + * @returns {{outputs: {path: string, bytes: number, sha256: string}[], maps: {path: string, bytes: number, sha256: string}[], upstreamMaps: number}} + * @throws {CandidateFinalizationError} for any unaccounted, unchained or malformed output. + */ +export function finalizeCandidate({ repository, staging }) { + assertPrivateStaging(staging, repository); + const cli = join(repository, 'apps', 'cli'); + const dist = join(cli, 'dist'); + const metafileBytes = readBytes( + join(dist, 'metafile-esm.json'), + LIMITS.metafileBytes, + 'metafile', + ); + const metafile = parseJson(metafileBytes, 'metafile'); + if (!isRecord(metafile) || !isRecord(metafile.outputs) || !isRecord(metafile.inputs)) { + refuse('METAFILE_SHAPE'); + } + + // 1. Every emitted file is a paired JS output and map, or the metafile. Nothing else. + const runtime = Object.keys(metafile.outputs).filter((path) => !path.endsWith('.map')); + if (runtime.length === 0 || runtime.length > LIMITS.outputs) refuse('OUTPUT_INVENTORY'); + const expectedEntries = ['metafile-esm.json']; + for (const path of runtime) { + if (!/^dist\/[A-Za-z0-9_-]+\.js$/u.test(path)) refuse('OUTPUT_INVENTORY'); + if (!Object.hasOwn(metafile.outputs, `${path}.map`)) refuse('MAP_PAIRING', `apps/cli/${path}`); + expectedEntries.push(basename(path), `${basename(path)}.map`); + } + if (Object.keys(metafile.outputs).length !== runtime.length * 2) refuse('MAP_PAIRING'); + const entries = readdirSync(dist, { withFileTypes: true }); + if ( + entries.some((entry) => !entry.isFile()) || + !isDeepStrictEqual( + entries.map((entry) => entry.name).sort(byCodeUnit), + [...expectedEntries].sort(byCodeUnit), + ) + ) { + refuse('OUTPUT_INVENTORY'); + } + + // 2. Validate every consumed upstream map and every final map, and strip each exact trailer. + // No file is written until every output has been accepted. + const cache = new Map(); + const texts = new Map(); + for (const input of Object.keys(metafile.inputs)) inputSources(repository, input, cache, texts); + const prepared = runtime.map((path) => { + const jsName = basename(path); + const output = metafile.outputs[path]; + if (!isRecord(output) || !isRecord(output.inputs) || !Number.isSafeInteger(output.bytes)) { + refuse('METAFILE_SHAPE', `apps/cli/${path}`); + } + const js = readText(join(dist, jsName), `apps/cli/${path}`); + const trailer = mapTrailer(jsName); + if (!js.endsWith(trailer) || js.split('sourceMappingURL').length !== 2) { + refuse('FINAL_MAP_LINK', `apps/cli/${path}`); + } + const code = Buffer.from(js.slice(0, -trailer.length), 'utf8'); + // tsup appends the trailer after esbuild records the output size, so only the trailer may differ. + if (code.length !== output.bytes) refuse('TRAILER_BYTES', `apps/cli/${path}`); + const expected = new Set(); + for (const [input, contribution] of Object.entries(output.inputs)) { + if (!isRecord(contribution) || !Number.isSafeInteger(contribution.bytesInOutput)) { + refuse('METAFILE_SHAPE', `apps/cli/${path}`); + } + if (contribution.bytesInOutput === 0) continue; + const sources = cache.get(input); + if (!sources) refuse('UNSUPPORTED_INPUT', inputSubject(input)); + for (const source of sources) expected.add(source); + } + const map = finalMap( + repository, + jsName, + readJson(join(dist, `${jsName}.map`), LIMITS.finalMapBytes, `apps/cli/${path}.map`), + expected, + js.slice(0, -trailer.length), + texts, + ); + const mode = lstatSync(join(dist, jsName)).mode & 0o777; + return { path, jsName, code, map: Buffer.from(JSON.stringify(map), 'utf8'), mode }; + }); + + // 3. Stage maps and build metadata owner-only outside the package, then finalize the JS. + mkdirSync(join(staging, 'maps'), { mode: 0o700 }); + // The creation mode passes through the umask; the staged directory keeps the exact owner-only mode. + if (MODE_EVIDENCE === 'posix') chmodSync(join(staging, 'maps'), 0o700); + for (const item of prepared) + writePrivateFile(join(staging, 'maps', `${item.jsName}.map`), item.map); + writePrivateFile(join(staging, 'metafile-esm.json'), metafileBytes); + for (const item of prepared) replaceOutput(join(dist, item.jsName), item.code, item.mode); + for (const item of prepared) unlinkSync(join(dist, `${item.jsName}.map`)); + unlinkSync(join(dist, 'metafile-esm.json')); + + // 4. Re-check the finite closure and the public inventory on the finalized files, then hash. + const manifest = readJson(join(cli, 'package.json'), 128 * 1024, 'apps/cli/package.json'); + checkBundleClosure({ + metafile, + manifest, + files: emittedInventory(cli), + modeEvidence: MODE_EVIDENCE, + maps: 'staged', + }); + if ( + !isDeepStrictEqual( + [...flatFiles(join(cli, 'drizzle'), 'package/drizzle')], + [...flatFiles(join(repository, 'packages', 'db', 'drizzle'), 'package/drizzle')], + ) + ) { + refuse('MIGRATION_COPY'); + } + const outputs = prepared.map((item) => { + const bytes = readOrdinaryFile(join(dist, item.jsName), LIMITS.jsBytes); + if (!bytes.equals(item.code)) refuse('OUTPUT_CHANGED', `apps/cli/${item.path}`); + return { path: item.path, bytes: bytes.length, sha256: sha256(bytes) }; + }); + const maps = prepared.map((item) => ({ + path: `${item.path}.map`, + bytes: item.map.length, + sha256: sha256(item.map), + })); + const upstreamMaps = [...cache.keys()].filter((input) => !input.startsWith('src/')).length; + const record = { version: 1, outputs, maps, upstreamMaps }; + writePrivateFile( + join(staging, 'finalized.json'), + Buffer.from(JSON.stringify(record, null, 2) + '\n'), + ); + return record; +} diff --git a/tools/cli-package/finalize.test.mjs b/tools/cli-package/finalize.test.mjs new file mode 100644 index 00000000..bd6e71ec --- /dev/null +++ b/tools/cli-package/finalize.test.mjs @@ -0,0 +1,1235 @@ +import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join, relative, sep } from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import ts from 'typescript'; +import { BundleClosureError } from '../bundle-closure/closure.mjs'; +import { CandidateFinalizationError, finalizeCandidate, mapTrailer } from './finalize.mjs'; + +// The exact esbuild behind the installed tsup and the installed TypeScript compiler produce real +// upstream and final maps from tiny fixtures; tsup's writer is reproduced exactly (trailer appended +// to the chunk code, map JSON re-serialized beside it). No product build runs here. +const cliRequire = createRequire(new URL('../../apps/cli/package.json', import.meta.url)); +const { build, transformSync } = createRequire(cliRequire.resolve('tsup/package.json'))('esbuild'); +const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-candidate-finalizer-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); +const posixModes = process.platform !== 'win32'; +let count = 0; + +function write(path, contents, mode) { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, contents); + if (mode !== undefined) chmodSync(path, mode); +} + +/** Every file below a directory, with its bytes, for unchanged-state assertions. */ +function snapshot(directory) { + const result = {}; + for (const entry of readdirSync(directory, { withFileTypes: true, recursive: true })) { + if (!entry.isFile()) continue; + const path = join(entry.parentPath ?? entry.path, entry.name); + result[relative(directory, path).split(sep).join('/')] = readFileSync(path).toString('base64'); + } + return result; +} + +/** + * A candidate repository whose CLI imports an engine package through its compiled JS, as the + * product does, built with splitting so it has shared chunks and both kinds of code-splitting + * facade: a statically and dynamically imported module (no inputs) and a pure re-export module + * (a zero-byte input). + */ +async function fixture({ upstreamMapAtBundle = true, sourcesContent = false, tsx = false } = {}) { + count += 1; + const root = join(scratch, `repository-${count}`); + const core = join(root, 'packages', 'core'); + const cli = join(root, 'apps', 'cli'); + write( + join(core, 'src', 'index.ts'), + "export * from './types.js';\nexport const greet = (name: string): string => `hello ${name}`;\n", + ); + write(join(core, 'src', 'types.ts'), 'export interface Shape {\n size: number;\n}\n'); + const program = ts.createProgram([join(core, 'src', 'index.ts'), join(core, 'src', 'types.ts')], { + rootDir: join(core, 'src'), + outDir: join(core, 'dist'), + sourceMap: true, + inlineSources: false, + module: ts.ModuleKind.ESNext, + target: ts.ScriptTarget.ES2022, + moduleResolution: ts.ModuleResolutionKind.Bundler, + strict: true, + skipLibCheck: true, + types: [], + }); + assert.equal(program.emit().emitSkipped, false); + const upstreamMap = join(core, 'dist', 'index.js.map'); + const upstreamMapBytes = readFileSync(upstreamMap); + if (!upstreamMapAtBundle) rmSync(upstreamMap); + write(join(root, 'packages', 'db', 'drizzle', '0000_init.sql'), 'create table t (id integer);\n'); + write(join(cli, 'drizzle', '0000_init.sql'), 'create table t (id integer);\n'); + write( + join(cli, 'package.json'), + JSON.stringify({ name: 'relavium', version: '0.0.0', type: 'module', dependencies: {} }), + ); + write( + join(cli, 'src', 'index.ts'), + "#!/usr/bin/env node\nimport { greet } from '../../../packages/core/dist/index.js';\n" + + "import { later } from './lazy.js';\nconsole.log(greet('cli'), later());\n" + + "export const lazy = () => import('./lazy.js');\nexport const facade = () => import('./facade.js');\n", + ); + write( + join(cli, 'src', 'lazy.ts'), + "import { greet } from '../../../packages/core/dist/index.js';\n" + + "export const later = (): string => greet('lazy');\n", + ); + write( + join(cli, 'src', 'facade.ts'), + "export { greet } from '../../../packages/core/dist/index.js';\n", + ); + if (tsx) { + write(join(cli, 'src', 'view.tsx'), "export const view = (): string => 'view';\n"); + write( + join(cli, 'src', 'index.ts'), + readFileSync(join(cli, 'src', 'index.ts'), 'utf8') + + "import { view } from './view.js';\nconsole.log(view());\n", + ); + } + const built = await build({ + absWorkingDir: cli, + entryPoints: { index: 'src/index.ts' }, + outdir: 'dist', + format: 'esm', + platform: 'node', + target: 'node22', + bundle: true, + splitting: true, + minify: true, + metafile: true, + write: false, + sourcemap: 'external', + sourcesContent, + logLevel: 'silent', + }); + const emitted = {}; + for (const output of built.outputFiles) { + const name = basename(output.path); + if (name.endsWith('.map')) { + write(join(cli, 'dist', name), JSON.stringify(JSON.parse(output.text))); + } else { + emitted[name] = output.text; + write( + join(cli, 'dist', name), + output.text + mapTrailer(name), + name === 'index.js' ? 0o755 : 0o644, + ); + } + } + write(join(cli, 'dist', 'metafile-esm.json'), JSON.stringify(built.metafile)); + if (!upstreamMapAtBundle) writeFileSync(upstreamMap, upstreamMapBytes); + const staging = join(scratch, `staging-${count}`); + mkdirSync(staging); + chmodSync(staging, 0o700); + return { root, core, cli, dist: join(cli, 'dist'), staging, metafile: built.metafile, emitted }; +} + +/** Refuses with `code` before writing anything: the package and staging stay exactly as they were. */ +function refusesBeforeWriting(candidate, code) { + const before = snapshot(candidate.root); + assert.throws( + () => finalizeCandidate({ repository: candidate.root, staging: candidate.staging }), + (error) => error instanceof CandidateFinalizationError && error.code === code, + ); + assert.deepEqual(snapshot(candidate.root), before); + assert.deepEqual(readdirSync(candidate.staging), []); +} + +/** The finalizer's refusal of `candidate`; accepting it fails the test. */ +function refusal(candidate) { + try { + finalizeCandidate({ repository: candidate.root, staging: candidate.staging }); + } catch (error) { + return error; + } + assert.fail('the finalizer accepted the candidate'); +} + +const digest = (bytes) => createHash('sha256').update(bytes).digest('hex'); + +function editJson(path, change) { + const value = JSON.parse(readFileSync(path, 'utf8')); + change(value); + writeFileSync(path, JSON.stringify(value)); +} + +/** Runtime outputs to which no input contributes a byte: code-splitting facades. */ +function facadeNames(candidate) { + return Object.entries(candidate.metafile.outputs) + .filter( + ([key, output]) => + !key.endsWith('.map') && + Object.values(output.inputs).every((contribution) => contribution.bytesInOutput === 0), + ) + .map(([key]) => basename(key)) + .sort(); +} + +test('finalizes a chained candidate into private repository-relative maps and public JS only', async () => { + const candidate = await fixture(); + const original = Object.fromEntries( + readdirSync(candidate.dist) + .filter((name) => name.endsWith('.map')) + .map((name) => [name, JSON.parse(readFileSync(join(candidate.dist, name), 'utf8'))]), + ); + const metafileBytes = readFileSync(join(candidate.dist, 'metafile-esm.json')); + const record = finalizeCandidate({ repository: candidate.root, staging: candidate.staging }); + + const names = Object.keys(candidate.emitted).sort(); + assert.ok(names.length >= 4, 'fixture has an entry, a shared chunk, a lazy chunk and a facade'); + assert.deepEqual(readdirSync(candidate.dist).sort(), names); + for (const name of names) { + // Only the terminal trailer is gone; every executable byte, line and column is unchanged. + assert.equal(readFileSync(join(candidate.dist, name), 'utf8'), candidate.emitted[name]); + } + assert.deepEqual(readdirSync(candidate.staging).sort(), [ + 'finalized.json', + 'maps', + 'metafile-esm.json', + ]); + assert.ok(readFileSync(join(candidate.staging, 'metafile-esm.json')).equals(metafileBytes)); + assert.deepEqual( + readdirSync(join(candidate.staging, 'maps')).sort(), + names.map((name) => `${name}.map`), + ); + const sources = new Set(); + for (const name of names) { + const map = JSON.parse(readFileSync(join(candidate.staging, 'maps', `${name}.map`), 'utf8')); + assert.equal('sourcesContent' in map, false); + assert.equal(map.mappings, original[`${name}.map`].mappings); + assert.deepEqual(map.names, original[`${name}.map`].names); + for (const source of map.sources) sources.add(source); + } + assert.deepEqual([...sources].sort(), [ + 'apps/cli/src/index.ts', + 'apps/cli/src/lazy.ts', + 'packages/core/src/index.ts', + ]); + assert.equal(record.upstreamMaps, 2); + assert.deepEqual( + JSON.parse(readFileSync(join(candidate.staging, 'finalized.json'), 'utf8')), + record, + ); + // The private record hashes exactly the finalized JS and the staged maps. + assert.equal(record.version, 1); + assert.deepEqual( + record.outputs.map((output) => output.path).sort(), + names.map((name) => `dist/${name}`), + ); + for (const output of record.outputs) { + const bytes = readFileSync(join(candidate.cli, output.path)); + assert.deepEqual(output, { path: output.path, bytes: bytes.length, sha256: digest(bytes) }); + } + assert.deepEqual( + record.maps.map((map) => map.path).sort(), + names.map((name) => `dist/${name}.map`), + ); + for (const map of record.maps) { + const bytes = readFileSync(join(candidate.staging, 'maps', basename(map.path))); + assert.deepEqual(map, { path: map.path, bytes: bytes.length, sha256: digest(bytes) }); + } + const facades = facadeNames(candidate); + assert.equal(facades.length, 2, 'both facade kinds are exercised'); + const inputs = (name) => Object.keys(candidate.metafile.outputs[`dist/${name}`].inputs).length; + assert.deepEqual(facades.map(inputs).sort(), [0, 1]); + if (posixModes) { + assert.equal(statSync(candidate.staging).mode & 0o777, 0o700); + assert.equal(statSync(join(candidate.staging, 'maps')).mode & 0o777, 0o700); + for (const name of readdirSync(join(candidate.staging, 'maps'))) { + assert.equal(statSync(join(candidate.staging, 'maps', name)).mode & 0o777, 0o600); + } + for (const name of names) { + assert.equal( + statSync(join(candidate.dist, name)).mode & 0o777, + name === 'index.js' ? 0o755 : 0o644, + ); + } + } +}); + +test('refuses a missing upstream map before the bundler can fall back to compiled JS', async () => { + const candidate = await fixture(); + rmSync(join(candidate.core, 'dist', 'index.js.map')); + refusesBeforeWriting(candidate, 'UPSTREAM_MAP_MISSING'); +}); + +test('refuses a final map that the bundler silently chained to compiled JS', async () => { + // Bundled while the upstream map was absent, then restored: only the final source set shows it. + const candidate = await fixture({ upstreamMapAtBundle: false }); + refusesBeforeWriting(candidate, 'MAP_SOURCE_NOT_ORIGINAL'); +}); + +for (const [label, change, code] of [ + ['embedded sources', (map) => (map.sourcesContent = ['x']), 'UPSTREAM_MAP_SHAPE'], + ['a source root', (map) => (map.sourceRoot = 'src'), 'UPSTREAM_MAP_SHAPE'], + ['another version', (map) => (map.version = 2), 'UPSTREAM_MAP_SHAPE'], + ['another file', (map) => (map.file = 'other.js'), 'UPSTREAM_MAP_SHAPE'], + ['no sources', (map) => (map.sources = []), 'UPSTREAM_MAP_SHAPE'], + ['malformed mappings', (map) => (map.mappings = 'AAAA!'), 'UPSTREAM_MAP_SHAPE'], + ['a non-string name', (map) => (map.names = [1]), 'UPSTREAM_MAP_SHAPE'], + // Only a module whose types all erase (`export {};`, as types.js is) may map nothing. + ['no mappings for executable code', (map) => (map.mappings = ''), 'UPSTREAM_MAP_UNMAPPED'], + [ + 'a URL source', + (map) => (map.sources = ['https://example.invalid/index.ts']), + 'MAP_SOURCE_PATH', + ], + ['an absolute source', (map) => (map.sources = ['/index.ts']), 'MAP_SOURCE_PATH'], + ['a backslash source', (map) => (map.sources = ['..\\src\\index.ts']), 'MAP_SOURCE_PATH'], + [ + 'an escaping source', + (map) => (map.sources = ['../../../../outside.ts']), + 'MAP_SOURCE_NOT_ORIGINAL', + ], + [ + 'another package', + (map) => (map.sources = ['../../shared/src/index.ts']), + 'MAP_SOURCE_NOT_ORIGINAL', + ], + ['a declaration', (map) => (map.sources = ['../src/index.d.ts']), 'MAP_SOURCE_NOT_ORIGINAL'], + ['compiled JS', (map) => (map.sources = ['index.js']), 'MAP_SOURCE_NOT_ORIGINAL'], + [ + 'a JS file under the source root', + (map) => (map.sources = ['../src/index.js']), + 'MAP_SOURCE_NOT_ORIGINAL', + ], + ['a missing source', (map) => (map.sources = ['../src/missing.ts']), 'MAP_SOURCE_MISSING'], + [ + 'more sources than the bound', + (map) => (map.sources = Array.from({ length: 4097 }, (_, i) => `../src/s${i}.ts`)), + 'UPSTREAM_MAP_SHAPE', + ], +]) { + test(`refuses an upstream map with ${label}`, async () => { + const candidate = await fixture(); + editJson(join(candidate.core, 'dist', 'index.js.map'), change); + refusesBeforeWriting(candidate, code); + }); +} + +test('refuses an upstream map source that exists but is not a file', async () => { + const candidate = await fixture(); + mkdirSync(join(candidate.core, 'src', 'dir.ts')); + editJson( + join(candidate.core, 'dist', 'index.js.map'), + (map) => (map.sources = ['../src/dir.ts']), + ); + refusesBeforeWriting(candidate, 'MAP_SOURCE_MISSING'); +}); + +test('refuses an unparsable upstream map and an inexact upstream map link', async () => { + const unparsable = await fixture(); + writeFileSync(join(unparsable.core, 'dist', 'index.js.map'), '{'); + refusesBeforeWriting(unparsable, 'MALFORMED_JSON'); + for (const change of [ + (js) => js.slice(0, -mapTrailer('index.js').length), + (js) => `${js}\n`, + (js) => js.replace(mapTrailer('index.js'), mapTrailer('other.js')), + (js) => `// sourceMappingURL in a comment\n${js}`, + ]) { + const candidate = await fixture(); + const path = join(candidate.core, 'dist', 'index.js'); + writeFileSync(path, change(readFileSync(path, 'utf8'))); + refusesBeforeWriting(candidate, 'UPSTREAM_MAP_LINK'); + } +}); + +test('accepts empty upstream mappings only for exactly the JS tsc emits for erased types', async () => { + // `export {};` followed by executable code is not a module whose types all erase. + const candidate = await fixture(); + writeFileSync( + join(candidate.core, 'dist', 'index.js'), + `export {};\nconsole.log(1);\n${mapTrailer('index.js')}`, + ); + editJson(join(candidate.core, 'dist', 'index.js.map'), (map) => (map.mappings = '')); + refusesBeforeWriting(candidate, 'UPSTREAM_MAP_UNMAPPED'); +}); + +test('refuses final maps that embed sources, lose coverage or carry unreviewed fields', async () => { + refusesBeforeWriting(await fixture({ sourcesContent: true }), 'EMBEDDED_SOURCES'); + for (const [change, code] of [ + [(map) => map.sources.pop(), 'MAP_SOURCE_COVERAGE'], + [(map) => map.sources.push('../src/lazy.ts'), 'MAP_SOURCE_COVERAGE'], + [(map) => map.sources.push(map.sources[0]), 'MAP_SOURCE_DUPLICATE'], + [(map) => (map.x_google_ignoreList = []), 'MAP_SHAPE'], + [(map) => (map.sections = []), 'MAP_SHAPE'], + [(map) => (map.sourceRoot = 'src'), 'MAP_SHAPE'], + [(map) => (map.file = 'other.js'), 'MAP_SHAPE'], + [(map) => (map.mappings = ''), 'MAP_SHAPE'], + [(map) => (map.version = 2), 'MAP_SHAPE'], + [(map) => (map.mappings = 'AAAA!'), 'MAP_SHAPE'], + [(map) => (map.names = [1]), 'MAP_SHAPE'], + [ + (map) => (map.sources = Array.from({ length: 4097 }, (_, i) => `../src/s${i}.ts`)), + 'MAP_SHAPE', + ], + [(map) => (map.names = Array.from({ length: 1_000_001 }, () => '')), 'MAP_SHAPE'], + [(map) => (map.sources = ['https://example.invalid/a.ts']), 'MAP_SOURCE_PATH'], + ]) { + const candidate = await fixture(); + editJson(join(candidate.dist, 'index.js.map'), change); + refusesBeforeWriting(candidate, code); + } + // A facade's map stays empty in every field: sources, mappings and names each refuse alone. + for (const change of [ + (map) => (map.sources = ['../src/facade.ts']), + (map) => (map.mappings = 'AAAA'), + (map) => (map.names = ['x']), + ]) { + const candidate = await fixture(); + editJson(join(candidate.dist, `${facadeNames(candidate)[0]}.map`), change); + refusesBeforeWriting(candidate, 'MAP_SHAPE'); + } +}); + +test('removes only the exact terminal trailer that tsup appended after esbuild sized the output', async () => { + for (const change of [ + (js) => js.slice(0, -mapTrailer('index.js').length), + (js) => `${js}\n`, + (js) => js.replace(mapTrailer('index.js'), mapTrailer('chunk.js')), + (js) => `${js}${mapTrailer('index.js')}`, + ]) { + const candidate = await fixture(); + const path = join(candidate.dist, 'index.js'); + writeFileSync(path, change(readFileSync(path, 'utf8'))); + refusesBeforeWriting(candidate, 'FINAL_MAP_LINK'); + } + const candidate = await fixture(); + const path = join(candidate.dist, 'index.js'); + writeFileSync( + path, + readFileSync(path, 'utf8').replace(mapTrailer('index.js'), `;${mapTrailer('index.js')}`), + ); + refusesBeforeWriting(candidate, 'TRAILER_BYTES'); +}); + +test('accounts for every emitted file and every consumed input', async () => { + const extra = await fixture(); + writeFileSync(join(extra.dist, 'notes.txt'), 'x'); + refusesBeforeWriting(extra, 'OUTPUT_INVENTORY'); + const directory = await fixture(); + mkdirSync(join(directory.dist, 'nested')); + refusesBeforeWriting(directory, 'OUTPUT_INVENTORY'); + const missing = await fixture(); + rmSync(join(missing.dist, 'index.js.map')); + refusesBeforeWriting(missing, 'OUTPUT_INVENTORY'); + const unpaired = await fixture(); + editJson(join(unpaired.dist, 'metafile-esm.json'), (metafile) => { + delete metafile.outputs['dist/index.js.map']; + }); + refusesBeforeWriting(unpaired, 'MAP_PAIRING'); + // Exactly 256 runtime outputs pass the bound (and then lack the last one's map entry); 257 refuse. + for (const [total, code] of [ + [257, 'OUTPUT_INVENTORY'], + [256, 'MAP_PAIRING'], + ]) { + const crowded = await fixture(); + editJson(join(crowded.dist, 'metafile-esm.json'), (metafile) => { + const runtime = Object.keys(metafile.outputs).filter((key) => !key.endsWith('.map')); + const extra = { imports: [], exports: [], inputs: {}, bytes: 1 }; + for (let i = 0; i < total - runtime.length; i += 1) { + metafile.outputs[`dist/extra${i}.js`] = extra; + if (i < total - runtime.length - 1) metafile.outputs[`dist/extra${i}.js.map`] = extra; + } + }); + refusesBeforeWriting(crowded, code); + } + const unsized = await fixture(); + editJson(join(unsized.dist, 'metafile-esm.json'), (metafile) => { + const contribution = Object.values(metafile.outputs['dist/index.js'].inputs)[0]; + delete contribution.bytesInOutput; + }); + refusesBeforeWriting(unsized, 'METAFILE_SHAPE'); + const foreign = await fixture(); + editJson(join(foreign.dist, 'metafile-esm.json'), (metafile) => { + metafile.inputs['../../node_modules/left-pad/index.js'] = { + bytes: 1, + imports: [], + format: 'esm', + }; + }); + refusesBeforeWriting(foreign, 'UNSUPPORTED_INPUT'); + // Only the five inlined engine packages' canonical compiled JS and the CLI's own TypeScript are + // supported inputs, even where a non-canonical spelling names a real compiled file. + for (const input of [ + '../../packages/ui/dist/x.js', + 'src/types.d.ts', + 'src/x.js', + '../../packages/core/dist/../dist/index.js', + 'src/../../../packages/core/dist/index.js', + 'x/../../packages/core/dist/index.js', + ]) { + const unsupported = await fixture(); + write(join(unsupported.cli, 'src', 'x.js'), 'export {};\n'); + editJson(join(unsupported.dist, 'metafile-esm.json'), (metafile) => { + metafile.inputs[input] = { bytes: 1, imports: [], format: 'esm' }; + }); + refusesBeforeWriting(unsupported, 'UNSUPPORTED_INPUT'); + } + // An input contributing bytes to an output must be one of the metafile's inputs. + const ghost = await fixture(); + editJson(join(ghost.dist, 'metafile-esm.json'), (metafile) => { + metafile.outputs['dist/index.js'].inputs['src/ghost.ts'] = { bytesInOutput: 1 }; + }); + refusesBeforeWriting(ghost, 'UNSUPPORTED_INPUT'); + // A build with no runtime output refuses before anything is written. + const empty = await fixture(); + for (const name of readdirSync(empty.dist)) { + if (name !== 'metafile-esm.json') rmSync(join(empty.dist, name)); + } + editJson(join(empty.dist, 'metafile-esm.json'), (metafile) => (metafile.outputs = {})); + refusesBeforeWriting(empty, 'OUTPUT_INVENTORY'); + if (posixModes) { + const linked = await fixture(); + symlinkSync(join(linked.dist, 'index.js'), join(linked.dist, 'alias.js')); + refusesBeforeWriting(linked, 'OUTPUT_INVENTORY'); + // A link standing in for an expected output has the right name but is not an ordinary file. + const replaced = await fixture(); + const map = join(replaced.dist, 'index.js.map'); + writeFileSync(join(replaced.root, 'elsewhere.map'), readFileSync(map)); + rmSync(map); + symlinkSync(join(replaced.root, 'elsewhere.map'), map); + refusesBeforeWriting(replaced, 'OUTPUT_INVENTORY'); + } +}); + +test('refuses an orphan map entry and validates the upstream map of a zero-byte input', async () => { + const orphan = await fixture(); + editJson(join(orphan.dist, 'metafile-esm.json'), (metafile) => { + metafile.outputs['dist/orphan.js.map'] = { imports: [], exports: [], inputs: {}, bytes: 1 }; + }); + refusesBeforeWriting(orphan, 'MAP_PAIRING'); + // A runtime output whose own map entry was swapped for another keeps the entry count even. + const swapped = await fixture(); + editJson(join(swapped.dist, 'metafile-esm.json'), (metafile) => { + metafile.outputs['dist/orphan.js.map'] = metafile.outputs['dist/index.js.map']; + delete metafile.outputs['dist/index.js.map']; + }); + refusesBeforeWriting(swapped, 'MAP_PAIRING'); + // A runtime output must be a flat dist/.js, even when its basename matches the inventory. + const nested = await fixture(); + editJson(join(nested.dist, 'metafile-esm.json'), (metafile) => { + const path = Object.keys(metafile.outputs).find( + (key) => key.endsWith('.js') && key !== 'dist/index.js', + ); + const moved = `dist/x/../${basename(path)}`; + metafile.outputs[moved] = metafile.outputs[path]; + metafile.outputs[`${moved}.map`] = metafile.outputs[`${path}.map`]; + delete metafile.outputs[path]; + delete metafile.outputs[`${path}.map`]; + }); + refusesBeforeWriting(nested, 'OUTPUT_INVENTORY'); + // types.js contributes no byte to any output, yet its map is a consumed upstream map. + const zero = await fixture(); + editJson(join(zero.core, 'dist', 'types.js.map'), (map) => (map.sourceRoot = 'src')); + refusesBeforeWriting(zero, 'UPSTREAM_MAP_SHAPE'); +}); + +test('a source root is a whole directory: a sibling that only shares its prefix is not original', async () => { + // An upstream map naming packages/core/srcxx and a final map naming apps/cli/srcxx, both existing. + const upstream = await fixture(); + write(join(upstream.core, 'srcxx', 'index.ts'), 'export const x = 1;\n'); + editJson( + join(upstream.core, 'dist', 'index.js.map'), + (map) => (map.sources = ['../srcxx/index.ts']), + ); + refusesBeforeWriting(upstream, 'MAP_SOURCE_NOT_ORIGINAL'); + const final = await fixture(); + write(join(final.cli, 'srcxx', 'x.ts'), 'export const x = 1;\n'); + editJson(join(final.dist, 'index.js.map'), (map) => (map.sources = ['../srcxx/x.ts'])); + refusesBeforeWriting(final, 'MAP_SOURCE_NOT_ORIGINAL'); +}); + +test('refuses a metafile whose records have the wrong type', async () => { + for (const change of [ + (metafile) => delete metafile.outputs, + (metafile) => delete metafile.inputs, + (metafile) => (metafile.outputs = []), + (metafile) => (metafile.inputs = []), + (metafile) => (metafile.outputs['dist/index.js'] = null), + (metafile) => delete metafile.outputs['dist/index.js'].inputs, + (metafile) => (metafile.outputs['dist/index.js'].inputs = []), + (metafile) => (metafile.outputs['dist/index.js'].bytes = '12'), + (metafile) => { + const inputs = metafile.outputs['dist/index.js'].inputs; + inputs[Object.keys(inputs)[0]] = null; + }, + ]) { + const candidate = await fixture(); + editJson(join(candidate.dist, 'metafile-esm.json'), change); + refusesBeforeWriting(candidate, 'METAFILE_SHAPE'); + } + const absent = await fixture(); + writeFileSync(join(absent.dist, 'metafile-esm.json'), 'null'); + refusesBeforeWriting(absent, 'METAFILE_SHAPE'); +}); + +test('reads a metafile, an upstream map or a final map of exactly its byte bound, and no more', async () => { + // Valid JSON padded to each bound, so only the bound decides. + for (const [path, bound] of [ + [(candidate) => join(candidate.dist, 'metafile-esm.json'), 4 * 1024 * 1024], + [(candidate) => join(candidate.core, 'dist', 'index.js.map'), 4 * 1024 * 1024], + [(candidate) => join(candidate.dist, 'index.js.map'), 16 * 1024 * 1024], + ]) { + for (const extra of [0, 1]) { + const candidate = await fixture(); + const file = path(candidate); + const text = readFileSync(file, 'utf8'); + writeFileSync(file, text + ' '.repeat(bound - Buffer.byteLength(text) + extra)); + if (extra === 0) + finalizeCandidate({ repository: candidate.root, staging: candidate.staging }); + else refusesBeforeWriting(candidate, 'UNREADABLE_FILE'); + } + } + // Final and upstream JS over the JS bound. + for (const path of [ + (candidate) => join(candidate.dist, 'index.js'), + (candidate) => join(candidate.core, 'dist', 'index.js'), + ]) { + const candidate = await fixture(); + const file = path(candidate); + writeFileSync(file, ' '.repeat(8 * 1024 * 1024) + readFileSync(file, 'utf8')); + refusesBeforeWriting(candidate, 'UNREADABLE_FILE'); + } +}); + +test('decodes JS and maps strictly: no byte-order mark is removed or admitted, no invalid UTF-8', async () => { + const bom = Buffer.from([0xef, 0xbb, 0xbf]); + // The leading mark would change the public bytes, so the recorded size no longer matches. + const js = await fixture(); + writeFileSync( + join(js.dist, 'index.js'), + Buffer.concat([bom, readFileSync(join(js.dist, 'index.js'))]), + ); + refusesBeforeWriting(js, 'TRAILER_BYTES'); + const map = await fixture(); + writeFileSync( + join(map.dist, 'index.js.map'), + Buffer.concat([bom, readFileSync(join(map.dist, 'index.js.map'))]), + ); + refusesBeforeWriting(map, 'MALFORMED_JSON'); + const invalidMap = await fixture(); + const text = readFileSync(join(invalidMap.dist, 'index.js.map'), 'utf8'); + writeFileSync( + join(invalidMap.dist, 'index.js.map'), + Buffer.concat([Buffer.from(`${text.slice(0, -1)},"file":"`), Buffer.from([0xff, 0x22, 0x7d])]), + ); + refusesBeforeWriting(invalidMap, 'MALFORMED_JSON'); + const invalidUpstream = await fixture(); + const upstream = join(invalidUpstream.core, 'dist', 'index.js'); + writeFileSync( + upstream, + Buffer.concat([Buffer.from([0x2f, 0x2f, 0xff, 0x0a]), readFileSync(upstream)]), + ); + refusesBeforeWriting(invalidUpstream, 'UNREADABLE_FILE'); +}); + +test('refuses upstream and final maps whose fields have the wrong type', async () => { + for (const change of [ + (map) => (map.mappings = 1234), + (map) => (map.names = 'x'), + (map) => (map.names = {}), + (map) => (map.sources = 'x'), + (map) => (map.sources = {}), + ]) { + const upstream = await fixture(); + editJson(join(upstream.core, 'dist', 'index.js.map'), change); + refusesBeforeWriting(upstream, 'UPSTREAM_MAP_SHAPE'); + const final = await fixture(); + editJson(join(final.dist, 'index.js.map'), change); + refusesBeforeWriting(final, 'MAP_SHAPE'); + } + const upstream = await fixture(); + writeFileSync(join(upstream.core, 'dist', 'index.js.map'), 'null'); + refusesBeforeWriting(upstream, 'UPSTREAM_MAP_SHAPE'); + const final = await fixture(); + writeFileSync(join(final.dist, 'index.js.map'), 'null'); + refusesBeforeWriting(final, 'MAP_SHAPE'); + if (posixModes) { + // A link standing in for an upstream map is not one. + const linked = await fixture(); + const map = join(linked.core, 'dist', 'index.js.map'); + writeFileSync(join(linked.root, 'elsewhere.map'), readFileSync(map)); + rmSync(map); + symlinkSync(join(linked.root, 'elsewhere.map'), map); + refusesBeforeWriting(linked, 'UPSTREAM_MAP_MISSING'); + } +}); + +test('a map source must be a plain path of at most 1024 characters to an original file', async () => { + for (const source of [ + '../src/index%2Ets', + '../src/index.ts?x', + '../src/index.ts#x', + '../src/in\u0001dex.ts', + '', + `../src/${'a/'.repeat(510)}x.ts`, + 1, + null, + ['../src/index.ts'], + ]) { + const upstream = await fixture(); + editJson(join(upstream.core, 'dist', 'index.js.map'), (map) => (map.sources = [source])); + refusesBeforeWriting(upstream, 'MAP_SOURCE_PATH'); + if (typeof source !== 'string') { + const final = await fixture(); + editJson(join(final.dist, 'index.js.map'), (map) => (map.sources[0] = source)); + refusesBeforeWriting(final, 'MAP_SOURCE_PATH'); + } + } + // A source of exactly 1024 characters passes the path check and is then looked up. + const bounded = await fixture(); + const longest = `../src/${'a/'.repeat(506)}xy.ts`; + assert.equal(longest.length, 1024); + editJson(join(bounded.core, 'dist', 'index.js.map'), (map) => (map.sources = [longest])); + refusesBeforeWriting(bounded, 'MAP_SOURCE_MISSING'); + // One character more is refused as a path, before any lookup. + const over = await fixture(); + const tooLong = `../src/${'a/'.repeat(507)}x.ts`; + assert.equal(tooLong.length, 1025); + editJson(join(over.core, 'dist', 'index.js.map'), (map) => (map.sources = [tooLong])); + refusesBeforeWriting(over, 'MAP_SOURCE_PATH'); + // A source under a dot directory is not original TypeScript, even when the file exists. + const dotted = await fixture(); + write(join(dotted.core, 'src', '.hidden', 'x.ts'), 'export const x = 1;\n'); + editJson( + join(dotted.core, 'dist', 'index.js.map'), + (map) => (map.sources = ['../src/.hidden/x.ts']), + ); + refusesBeforeWriting(dotted, 'MAP_SOURCE_NOT_ORIGINAL'); + if (posixModes) { + // A link to TypeScript elsewhere is not an original source file. + const linked = await fixture(); + writeFileSync(join(linked.root, 'real.ts'), 'export const x = 1;\n'); + symlinkSync(join(linked.root, 'real.ts'), join(linked.core, 'src', 'linked.ts')); + editJson( + join(linked.core, 'dist', 'index.js.map'), + (map) => (map.sources = ['../src/linked.ts']), + ); + refusesBeforeWriting(linked, 'MAP_SOURCE_MISSING'); + } +}); + +/** Base64 VLQ for one signed value, as a source map writer encodes it. */ +function vlq(value) { + const digits = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; + let rest = value < 0 ? -value * 2 + 1 : value * 2; + let out = ''; + do { + let digit = rest % 32; + rest = Math.floor(rest / 32); + if (rest > 0) digit += 32; + out += digits[digit]; + } while (rest > 0); + return out; +} + +test('refuses mappings that are malformed or point outside their texts, upstream and final', async () => { + // Each mapping string passes the character check; only strict decoding or a range check refuses it. + const pastLastLine = `AAAA;${';'.repeat(5000)}AAAA`; + for (const [label, mappings, code] of [ + ['a truncated value', 'AAAg', 'MAPPINGS'], + ['a two-field segment', 'AA', 'MAPPINGS'], + ['a source index past the sources', `A${vlq(1)}AA`, 'MAPPINGS'], + ['an empty segment', 'AAAA,,CAAC', 'MAPPINGS'], + ['a generated column that goes back', 'EAAA,DAAC', 'MAPPINGS'], + ['an original line past the source', `AA${vlq(100000)}A`, 'RANGE'], + ['an original column past its line', `AAA${vlq(100000)}`, 'RANGE'], + ['a generated line past the text', pastLastLine, 'RANGE'], + ['a generated column past its line', `${vlq(100000)}AAA`, 'RANGE'], + ]) { + const upstream = await fixture(); + editJson(join(upstream.core, 'dist', 'index.js.map'), (map) => (map.mappings = mappings)); + const upstreamError = refusal(upstream); + assert.equal(upstreamError.code, `UPSTREAM_MAP_${code}`, `upstream ${label}`); + assert.equal( + upstreamError.message, + `CLI candidate finalization refused: UPSTREAM_MAP_${code} (packages/core/dist/index.js.map)`, + ); + const final = await fixture(); + editJson(join(final.dist, 'index.js.map'), (map) => (map.mappings = mappings)); + refusesBeforeWriting(final, `MAP_${code}`); + } +}); + +/** Encode absolute segments ([column] or [column, source, line, column, name?]) per generated line. */ +function encode(lines) { + let source = 0; + let line = 0; + let column = 0; + let name = 0; + return lines + .map((segments) => { + let generated = 0; + return segments + .map((segment) => { + let out = vlq(segment[0] - generated); + generated = segment[0]; + if (segment.length >= 4) { + out += vlq(segment[1] - source) + vlq(segment[2] - line) + vlq(segment[3] - column); + [source, line, column] = [segment[1], segment[2], segment[3]]; + } + if (segment.length === 5) { + out += vlq(segment[4] - name); + name = segment[4]; + } + return out; + }) + .join(','); + }) + .join(';'); +} + +test('refuses a name past the names, NaN padding and positions one past their texts', async () => { + const upstreamMap = (candidate) => join(candidate.core, 'dist', 'index.js.map'); + const finalMap = (candidate) => join(candidate.dist, 'index.js.map'); + // A name index one past the names: upstream, which records none, and final. + const unnamed = await fixture(); + editJson(upstreamMap(unnamed), (map) => { + assert.deepEqual(map.names, []); + map.mappings = 'AAAAA'; + }); + assert.equal(refusal(unnamed).code, 'UPSTREAM_MAP_MAPPINGS'); + const named = await fixture(); + let names = 0; + editJson(finalMap(named), (map) => { + names = map.names.length; + map.mappings = `AAAA${vlq(names)}`; + }); + assert.ok(names > 0); + refusesBeforeWriting(named, 'MAP_MAPPINGS'); + // Zero-payload padding that would decode NaN: an upstream name index, and a final generated + // column followed by a segment far past its line, which a NaN column would hide. + const padding = `${'g'.repeat(205)}A`; + const paddedUpstream = await fixture(); + editJson(upstreamMap(paddedUpstream), (map) => (map.mappings = `AAAA${padding}`)); + assert.equal(refusal(paddedUpstream).code, 'UPSTREAM_MAP_MAPPINGS'); + const paddedFinal = await fixture(); + editJson(finalMap(paddedFinal), (map) => (map.mappings = `${padding}AAA,${vlq(100000)}`)); + refusesBeforeWriting(paddedFinal, 'MAP_MAPPINGS'); + // The final text is measured without the stripped trailer: the finalized index.js ends with a + // newline, so column 1 of its empty last line is past it. + const trailer = await fixture(); + editJson(finalMap(trailer), (map) => { + const code = trailer.emitted['index.js']; + assert.ok(code.endsWith('\n')); + map.mappings += `${';'.repeat(code.split('\n').length - map.mappings.split(';').length)}C`; + }); + refusesBeforeWriting(trailer, 'MAP_RANGE'); + // A segment one line past the generated text, final and upstream. + const pastFinal = await fixture(); + editJson(finalMap(pastFinal), (map) => { + const lines = pastFinal.emitted['index.js'].split('\n').length; + map.mappings += `${';'.repeat(lines - map.mappings.split(';').length + 1)}A`; + }); + refusesBeforeWriting(pastFinal, 'MAP_RANGE'); + const pastUpstream = await fixture(); + const js = readFileSync(join(pastUpstream.core, 'dist', 'index.js'), 'utf8'); + editJson(upstreamMap(pastUpstream), (map) => { + map.mappings += `${';'.repeat(js.split('\n').length - map.mappings.split(';').length + 1)}A`; + }); + assert.equal(refusal(pastUpstream).code, 'UPSTREAM_MAP_RANGE'); + // An original line one past its source, and a column one past its line's end. + const source = readFileSync(join(pastUpstream.core, 'src', 'index.ts'), 'utf8').split('\n'); + for (const mappings of [`AA${vlq(source.length)}A`, `AAA${vlq(source[0].length + 1)}`]) { + const candidate = await fixture(); + editJson(upstreamMap(candidate), (map) => (map.mappings = mappings)); + assert.equal(refusal(candidate).code, 'UPSTREAM_MAP_RANGE', mappings); + } +}); + +test('checks each original position against its own source, upstream and final', async () => { + // The .tsx fixture bundles index.ts and the one-line view.tsx into index.js: two sources. + const invalid = await fixture({ tsx: true }); + let view = -1; + editJson(join(invalid.dist, 'index.js.map'), (map) => { + assert.equal(map.sources.length, 2); + view = map.sources.findIndex((source) => source.endsWith('view.tsx')); + // Line 5 exists in index.ts but not in view.tsx. + map.mappings = encode([[], [[0, view, 5, 0]]]); + }); + assert.ok(view >= 0); + refusesBeforeWriting(invalid, 'MAP_RANGE'); + // The last line of index.ts and the caret end of view.tsx's only line are inside their own source. + const valid = await fixture({ tsx: true }); + editJson(join(valid.dist, 'index.js.map'), (map) => { + const index = map.sources.findIndex((source) => source.endsWith('index.ts')); + const indexLines = readFileSync(join(valid.cli, 'src', 'index.ts'), 'utf8').split('\n'); + const viewLines = readFileSync(join(valid.cli, 'src', 'view.tsx'), 'utf8').split('\n'); + assert.ok(indexLines.length > viewLines.length + 2); + map.mappings = encode([ + [], + [ + [0, index, indexLines.length - 2, indexLines[indexLines.length - 2].length], + [1, 1 - index, 0, viewLines[0].length], + ], + ]); + }); + finalizeCandidate({ repository: valid.root, staging: valid.staging }); + // An upstream map with two sources: column 25 of the first line is inside index.ts, not types.ts. + const upstream = await fixture(); + const types = readFileSync(join(upstream.core, 'src', 'types.ts'), 'utf8').split('\n'); + const index = readFileSync(join(upstream.core, 'src', 'index.ts'), 'utf8').split('\n'); + assert.ok(types[0].length < 25 && index[0].length >= 25); + editJson(join(upstream.core, 'dist', 'index.js.map'), (map) => { + map.sources = ['../src/index.ts', '../src/types.ts']; + map.mappings = encode([[[0, 1, 0, 25]]]); + }); + assert.equal(refusal(upstream).code, 'UPSTREAM_MAP_RANGE'); +}); + +test('admits original .tsx CLI sources', async () => { + const candidate = await fixture({ tsx: true }); + finalizeCandidate({ repository: candidate.root, staging: candidate.staging }); + const map = JSON.parse(readFileSync(join(candidate.staging, 'maps', 'index.js.map'), 'utf8')); + assert.ok(map.sources.includes('apps/cli/src/view.tsx'), JSON.stringify(map.sources)); +}); + +test('a refusal names at most a validated repository-relative subject, never a host path', async () => { + const input = (path) => (candidate) => + editJson(join(candidate.dist, 'metafile-esm.json'), (metafile) => { + metafile.inputs[typeof path === 'function' ? path(candidate) : path] = { + bytes: 1, + imports: [], + format: 'esm', + }; + }); + const upstreamMap = (candidate) => join(candidate.core, 'dist', 'index.js.map'); + const finalMap = (candidate) => join(candidate.dist, 'index.js.map'); + for (const [change, code, subject] of [ + [(c) => rmSync(upstreamMap(c)), 'UPSTREAM_MAP_MISSING', 'packages/core/dist/index.js'], + [ + (c) => editJson(upstreamMap(c), (map) => (map.version = 2)), + 'UPSTREAM_MAP_SHAPE', + 'packages/core/dist/index.js.map', + ], + [ + (c) => editJson(upstreamMap(c), (map) => (map.sources = ['../src/missing.ts'])), + 'MAP_SOURCE_MISSING', + 'packages/core/dist/index.js.map', + ], + [ + (c) => editJson(upstreamMap(c), (map) => (map.sources = [`${c.root}/x.ts`])), + 'MAP_SOURCE_PATH', + 'packages/core/dist/index.js.map', + ], + [ + (c) => writeFileSync(upstreamMap(c), '{'), + 'MALFORMED_JSON', + 'packages/core/dist/index.js.map', + ], + [ + (c) => writeFileSync(upstreamMap(c), ' '.repeat(4 * 1024 * 1024 + 1)), + 'UNREADABLE_FILE', + 'packages/core/dist/index.js.map', + ], + [ + (c) => rmSync(join(c.core, 'dist', 'types.js')), + 'UNREADABLE_FILE', + 'packages/core/dist/types.js', + ], + [ + (c) => editJson(finalMap(c), (map) => (map.version = 2)), + 'MAP_SHAPE', + 'apps/cli/dist/index.js.map', + ], + [ + (c) => writeFileSync(finalMap(c), ' '.repeat(16 * 1024 * 1024 + 1)), + 'UNREADABLE_FILE', + 'apps/cli/dist/index.js.map', + ], + [ + (c) => writeFileSync(join(c.dist, 'index.js'), `${c.emitted['index.js']}\n`), + 'FINAL_MAP_LINK', + 'apps/cli/dist/index.js', + ], + [(c) => writeFileSync(join(c.dist, 'metafile-esm.json'), '{'), 'MALFORMED_JSON', 'metafile'], + [input('src/missing.ts'), 'MAP_SOURCE_MISSING', 'apps/cli/src/missing.ts'], + [input('../../node_modules/x/index.js'), 'UNSUPPORTED_INPUT', 'node_modules/x/index.js'], + [ + (c) => + editJson(join(c.dist, 'metafile-esm.json'), (metafile) => { + metafile.outputs['dist/index.js'].inputs['src/ghost.ts'] = { bytesInOutput: 1 }; + }), + 'UNSUPPORTED_INPUT', + 'apps/cli/src/ghost.ts', + ], + // Metafile strings that escape the repository, are absolute or are not plain paths: no subject. + [input(`${'../'.repeat(12)}private-notes.js`), 'UNSUPPORTED_INPUT', undefined], + [input((c) => `../../..${c.root}/private-notes.js`), 'UNSUPPORTED_INPUT', undefined], + [input((c) => `${c.root}/private-notes.js`), 'UNSUPPORTED_INPUT', undefined], + [input('../../node_modules/x y/index.js'), 'UNSUPPORTED_INPUT', undefined], + [input(`../../${'x'.repeat(200)}.js`), 'UNSUPPORTED_INPUT', undefined], + [ + (c) => + editJson(join(c.dist, 'metafile-esm.json'), (metafile) => { + const extra = { imports: [], exports: [], inputs: {}, bytes: 1 }; + metafile.outputs[`${c.root}/x.js`] = extra; + metafile.outputs[`${c.root}/x.js.map`] = extra; + }), + 'OUTPUT_INVENTORY', + undefined, + ], + ]) { + const candidate = await fixture(); + change(candidate); + const error = refusal(candidate); + assert.ok(error instanceof CandidateFinalizationError, `${code}: ${error}`); + assert.equal( + error.message, + `CLI candidate finalization refused: ${code}` + + (subject === undefined ? '' : ` (${subject})`), + ); + assert.equal(error.message.includes(scratch), false); + } +}); + +test('requires an empty private staging directory outside the candidate', async (t) => { + const candidate = await fixture(); + for (const [staging, code] of [ + ['relative-staging', 'STAGING_PATH'], + [join(scratch, 'absent-staging'), 'STAGING_PATH'], + [join(candidate.root, 'apps'), 'STAGING_PATH'], + [scratch, 'STAGING_PATH'], + [undefined, 'STAGING_PATH'], + [join(candidate.root, 'apps', 'cli', 'package.json'), 'STAGING_PATH'], + ]) { + assert.throws( + () => finalizeCandidate({ repository: candidate.root, staging }), + (error) => error instanceof CandidateFinalizationError && error.code === code, + ); + } + writeFileSync(join(candidate.staging, 'left-over'), 'x'); + refusesBeforeWritingIgnoringStaging(candidate, 'STAGING_NOT_EMPTY'); + rmSync(join(candidate.staging, 'left-over')); + if (posixModes) { + // A file outside the candidate is not a directory, however private its mode. + const file = join(scratch, `staging-file-${count}`); + writeFileSync(file, ''); + chmodSync(file, 0o700); + assert.throws( + () => finalizeCandidate({ repository: candidate.root, staging: file }), + (error) => error instanceof CandidateFinalizationError && error.code === 'STAGING_PATH', + ); + for (const mode of [0o755, 0o500]) { + chmodSync(candidate.staging, mode); + try { + refusesBeforeWriting(candidate, 'STAGING_NOT_PRIVATE'); + } finally { + chmodSync(candidate.staging, 0o700); + } + } + // An owner-only directory another account owns refuses too, before anything is written. + const realUid = process.getuid(); + const uid = t.mock.method(process, 'getuid', () => realUid + 1); + try { + refusesBeforeWriting(candidate, 'STAGING_NOT_PRIVATE'); + } finally { + uid.mock.restore(); + } + const link = join(scratch, `staging-link-${count}`); + symlinkSync(candidate.staging, link); + assert.throws( + () => finalizeCandidate({ repository: candidate.root, staging: link }), + (error) => error instanceof CandidateFinalizationError && error.code === 'STAGING_PATH', + ); + // A real owner-only directory inside the candidate, named through a linked parent outside it. + const inside = join(candidate.cli, 'private'); + mkdirSync(inside, { mode: 0o700 }); + chmodSync(inside, 0o700); + const alias = join(scratch, `alias-parent-${count}`); + symlinkSync(candidate.cli, alias); + assert.throws( + () => finalizeCandidate({ repository: candidate.root, staging: join(alias, 'private') }), + (error) => error instanceof CandidateFinalizationError && error.code === 'STAGING_PATH', + ); + // The same real directory refuses when the repository itself is named through a link. + const aliasRoot = join(scratch, `alias-root-${count}`); + symlinkSync(candidate.root, aliasRoot); + assert.throws( + () => finalizeCandidate({ repository: aliasRoot, staging: inside }), + (error) => error instanceof CandidateFinalizationError && error.code === 'STAGING_PATH', + ); + rmSync(inside, { recursive: true }); + // A child whose name merely starts with two dots is inside the candidate. + const dotted = join(candidate.root, '..private'); + mkdirSync(dotted, { mode: 0o700 }); + chmodSync(dotted, 0o700); + assert.throws( + () => finalizeCandidate({ repository: candidate.root, staging: dotted }), + (error) => error instanceof CandidateFinalizationError && error.code === 'STAGING_PATH', + ); + rmSync(dotted, { recursive: true }); + // Staging must be named canonically even outside the candidate. + const outside = join(scratch, `outside-staging-${count}`); + mkdirSync(outside, { mode: 0o700 }); + chmodSync(outside, 0o700); + const outsideAlias = join(scratch, `outside-alias-${count}`); + symlinkSync(scratch, outsideAlias); + assert.throws( + () => + finalizeCandidate({ + repository: candidate.root, + staging: join(outsideAlias, `outside-staging-${count}`), + }), + (error) => error instanceof CandidateFinalizationError && error.code === 'STAGING_PATH', + ); + } +}); + +function refusesBeforeWritingIgnoringStaging(candidate, code) { + const before = snapshot(candidate.root); + assert.throws( + () => finalizeCandidate({ repository: candidate.root, staging: candidate.staging }), + (error) => error instanceof CandidateFinalizationError && error.code === code, + ); + assert.deepEqual(snapshot(candidate.root), before); +} + +test('re-checks the finite closure and the migration copy on the finalized files', async () => { + if (posixModes) { + const mode = await fixture(); + chmodSync(join(mode.dist, 'index.js'), 0o644); + assert.throws( + () => finalizeCandidate({ repository: mode.root, staging: mode.staging }), + (error) => error instanceof BundleClosureError && error.code === 'OUTPUT_MODE', + ); + } + const migration = await fixture(); + writeFileSync(join(migration.cli, 'drizzle', '0000_init.sql'), 'drifted\n'); + assert.throws( + () => finalizeCandidate({ repository: migration.root, staging: migration.staging }), + (error) => error instanceof CandidateFinalizationError && error.code === 'MIGRATION_COPY', + ); +}); + +test('ordinary engine package builds stay map-free', () => { + const root = new URL('../../', import.meta.url); + for (const name of ['shared', 'llm', 'core', 'db', 'mcp']) { + const path = fileURLToPath(new URL(`packages/${name}/tsconfig.build.json`, root)); + const read = ts.readConfigFile(path, ts.sys.readFile); + assert.equal(read.error, undefined); + const { options } = ts.parseJsonConfigFileContent(read.config, ts.sys, dirname(path)); + assert.equal(options.sourceMap, false, name); + assert.equal(options.declarationMap, false, name); + assert.notEqual(options.inlineSourceMap, true, name); + assert.notEqual(options.inlineSources, true, name); + } +}); + +test('only a named candidate CLI build emits maps, and those maps embed no sources', () => { + // The real tsup config, compiled by tsup's own esbuild and evaluated beside a copy of the CLI + // manifest and installation, with and without the candidate variable. + const cli = fileURLToPath(new URL('../../apps/cli/', import.meta.url)); + const directory = join(scratch, 'tsup-config'); + mkdirSync(directory); + const source = readFileSync(join(cli, 'tsup.config.ts'), 'utf8'); + writeFileSync( + join(directory, 'config.mjs'), + transformSync(source, { loader: 'ts', format: 'esm' }).code, + ); + copyFileSync(join(cli, 'package.json'), join(directory, 'package.json')); + symlinkSync(join(cli, 'node_modules'), join(directory, 'node_modules'), 'junction'); + const evaluate = (staging) => { + const env = { ...process.env }; + delete env.RELAVIUM_CLI_CANDIDATE_STAGING; + if (staging !== undefined) env.RELAVIUM_CLI_CANDIDATE_STAGING = staging; + const script = `const { default: config } = await import(${JSON.stringify( + pathToFileURL(join(directory, 'config.mjs')).href, + )}); +const options = {}; +config.esbuildOptions?.(options, { format: 'esm' }); +process.stdout.write(JSON.stringify({ sourcemap: config.sourcemap, options }));`; + const result = spawnSync(process.execPath, ['--input-type=module', '--eval', script], { + env, + encoding: 'utf8', + timeout: 30_000, + }); + assert.equal(result.status, 0, result.stderr); + return JSON.parse(result.stdout); + }; + assert.deepEqual(evaluate(join(scratch, 'private-staging')), { + sourcemap: true, + options: { sourcesContent: false }, + }); + assert.deepEqual(evaluate(undefined), { sourcemap: false, options: {} }); +}); + +test('keeps exact staged and public modes whatever the umask', { skip: !posixModes }, async () => { + const candidate = await fixture(); + // A umask that removes the owner's write bit would leave every file read-only without exact modes. + const previous = process.umask(0o277); + try { + finalizeCandidate({ repository: candidate.root, staging: candidate.staging }); + } finally { + process.umask(previous); + } + assert.equal(statSync(join(candidate.staging, 'maps')).mode & 0o777, 0o700); + for (const name of ['metafile-esm.json', 'finalized.json']) { + assert.equal(statSync(join(candidate.staging, name)).mode & 0o777, 0o600, name); + } + for (const name of readdirSync(join(candidate.staging, 'maps'))) { + assert.equal(statSync(join(candidate.staging, 'maps', name)).mode & 0o777, 0o600, name); + } + for (const name of Object.keys(candidate.emitted)) { + assert.equal( + statSync(join(candidate.dist, name)).mode & 0o777, + name === 'index.js' ? 0o755 : 0o644, + name, + ); + } +}); + +test('compares staging and the candidate by the letter case the filesystem stores', async (t) => { + const candidate = await fixture(); + const parent = dirname(candidate.root); + const variant = join(parent, basename(candidate.root).toUpperCase()); + if (!existsSync(variant)) { + t.skip('case-sensitive filesystem'); + return; + } + const inside = join(candidate.cli, 'private'); + mkdirSync(inside, { mode: 0o700 }); + chmodSync(inside, 0o700); + // Another letter case of a path inside the candidate is not its canonical name. + assert.throws( + () => + finalizeCandidate({ + repository: candidate.root, + staging: join(variant, 'apps', 'cli', 'private'), + }), + (error) => error instanceof CandidateFinalizationError && error.code === 'STAGING_PATH', + ); + // Nor does another letter case of the repository hide the canonical staging inside it. + assert.throws( + () => finalizeCandidate({ repository: variant, staging: inside }), + (error) => error instanceof CandidateFinalizationError && error.code === 'STAGING_PATH', + ); +}); diff --git a/tools/cli-package/frame-capture-hooks.mjs b/tools/cli-package/frame-capture-hooks.mjs new file mode 100644 index 00000000..79d86484 --- /dev/null +++ b/tools/cli-package/frame-capture-hooks.mjs @@ -0,0 +1,31 @@ +// Resolution hook of the symbolication preload (frame-capture.mjs), registered in-thread with +// `module.registerHooks` so that it sees `require` as well as `import`. The packed CLI's keyring +// binding is replaced by a module that refuses every use, so the run loads no native module and reads +// no secret. And no module may resolve outside the candidate: an optional `require` such as ws's +// `bufferutil` would otherwise fall through to a `node_modules` directory above the candidate, which +// another local user can create in a shared temporary directory. (The preload's own modules are +// loaded before the hook is registered, and the stand-in is returned without resolving.) +import { realpathSync } from 'node:fs'; +import { sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const KEYRING = new URL('./frame-capture-keyring.mjs', import.meta.url).href; + +/** + * A synchronous resolve hook confined to the candidate `root` (a real path). + * @param {string} root + */ +export function confinedResolve(root) { + return function resolve(specifier, context, nextResolve) { + if (specifier === '@napi-rs/keyring') return { url: KEYRING, shortCircuit: true }; + if (specifier.startsWith('@napi-rs/keyring-')) throw new Error('Native keyring import refused'); + const resolved = nextResolve(specifier, context); + if (resolved.url.startsWith('file:')) { + const path = realpathSync.native(fileURLToPath(resolved.url)); + if (path !== root && !path.startsWith(`${root}${sep}`)) { + throw new Error('The symbolication run refuses a module outside the candidate'); + } + } + return resolved; + }; +} diff --git a/tools/cli-package/frame-capture-keyring.mjs b/tools/cli-package/frame-capture-keyring.mjs new file mode 100644 index 00000000..e183798d --- /dev/null +++ b/tools/cli-package/frame-capture-keyring.mjs @@ -0,0 +1,28 @@ +// The keyring binding seen by the symbolication run (frame-capture-hooks.mjs): every export of the +// real package is present, and every use refuses, so no native binding loads and no secret is read. +const refused = () => { + throw new Error('The symbolication run refuses the keyring'); +}; +// A class, not a function, so the exports below extend it as the real package's classes do; its methods +// mirror the real `Entry`'s and refuse too, although the constructor already has. +export class Entry { + constructor() { + refused(); + } + getPassword() { + return refused(); + } + setPassword() { + return refused(); + } + deletePassword() { + return refused(); + } +} +export class AsyncEntry extends Entry {} +export class EntryTask extends Entry {} +export class SecretTask extends Entry {} +export class PasswordTask extends Entry {} +export class FindCredentials extends Entry {} +export const findCredentials = refused; +export const findCredentialsAsync = refused; diff --git a/tools/cli-package/frame-capture.mjs b/tools/cli-package/frame-capture.mjs new file mode 100644 index 00000000..2cd7a54d --- /dev/null +++ b/tools/cli-package/frame-capture.mjs @@ -0,0 +1,74 @@ +/** + * Tooling-only preload for symbolication (ADR-0111, Step 9a.2a; the 2026-10-24 frame ruling). It runs + * with `node --import` around the packed CLI while it imports a canary workflow, and adds no product + * surface: the packed bytes are untouched. It wraps the external `yaml` package's `parse` — the one + * instance the CLI's import and require both reach — and records V8's structured call sites for the + * canary document alone, so one synchronous stack crosses the CLI and the inlined `@relavium/core`. It + * writes one owner-only record when the process exits; the producer refuses unless exactly one stack + * was captured. The keyring binding is replaced by a refusing module, no module may resolve outside + * the candidate (frame-capture-hooks.mjs), and any native module load refuses. These hold in this + * process: a thread or subprocess the candidate starts without the preload is not covered. + */ +import { closeSync, openSync, realpathSync, writeSync } from 'node:fs'; +import nodeModule, { createRequire } from 'node:module'; +import { dirname, resolve } from 'node:path'; +import { confinedResolve } from './frame-capture-hooks.mjs'; + +const log = process.env.RELAVIUM_FRAME_CAPTURE_LOG; +const manifest = process.env.RELAVIUM_FRAME_CAPTURE_PACKAGE; +const canary = process.env.RELAVIUM_FRAME_CAPTURE_CANARY; +if (!log || !manifest || !canary) throw new Error('The symbolication preload is not configured'); +if (typeof nodeModule.registerHooks !== 'function') { + throw new TypeError('The symbolication preload needs module.registerHooks'); +} + +// The candidate root holds `apps/cli/package.json`; pnpm installs the CLI's dependencies below it. +nodeModule.registerHooks({ + resolve: confinedResolve(realpathSync.native(resolve(dirname(manifest), '..', '..'))), +}); +// The import route needs no native module; any addon load refuses instead of running native code +// beside the private staging. +process.dlopen = () => { + throw new Error('The symbolication run refuses native modules'); +}; + +/** @type {{file: string | null, line: number | null, column: number | null, async: boolean}[][]} */ +const captures = []; + +/** Structured call sites above `skip`, with the stack trace limit and formatter restored after. */ +function callSites(skip) { + const prepare = Error.prepareStackTrace; + const limit = Error.stackTraceLimit; + try { + Error.stackTraceLimit = 64; + Error.prepareStackTrace = (_error, sites) => sites; + const holder = {}; + Error.captureStackTrace(holder, skip); + return holder.stack.map((site) => ({ + file: site.getFileName() ?? null, + line: site.getLineNumber(), + column: site.getColumnNumber(), + async: site.isAsync(), + })); + } finally { + Error.prepareStackTrace = prepare; + Error.stackTraceLimit = limit; + } +} + +const yaml = createRequire(manifest)('yaml'); +const parse = yaml.parse; +if (typeof parse !== 'function') throw new Error('The yaml package has no parse export'); +yaml.parse = function capturedParse(text, ...rest) { + if (typeof text === 'string' && text.includes(canary)) captures.push(callSites(capturedParse)); + return Reflect.apply(parse, this, [text, ...rest]); +}; + +process.on('exit', () => { + const descriptor = openSync(log, 'wx', 0o600); + try { + writeSync(descriptor, JSON.stringify({ version: 1, captures })); + } finally { + closeSync(descriptor); + } +}); diff --git a/tools/cli-package/gnupg.mjs b/tools/cli-package/gnupg.mjs new file mode 100644 index 00000000..8f01ae7e --- /dev/null +++ b/tools/cli-package/gnupg.mjs @@ -0,0 +1,236 @@ +/** + * The isolated GnuPG boundary of the custody tools (Step 9b.1; packaging.md, "Recipient custody"). + * GnuPG is release tooling, never a runtime dependency. Every call runs a resolved absolute binary + * through the shell-free, bounded `runCommand` in the isolated environment. Every `gpg` call runs in a + * fresh owner-only home the tools create themselves, with no per-home options file, keyring, agent or + * key discovery; the one `gpgconf` query takes no home. GnuPG still reads its installation's + * system-wide `gpg.conf` under `--no-options`, so a global configuration refuses before any call that + * handles a key. Only status lines, colon records, the version line and the configuration directory + * are read; other human-readable output and stderr are never read or surfaced. + */ +import { existsSync, lstatSync, mkdirSync, readdirSync, realpathSync, statSync } from 'node:fs'; +import { delimiter, dirname, isAbsolute, join } from 'node:path'; +import { refuseCustody } from './custody-error.mjs'; +import { isolatedEnvironment } from './process.mjs'; + +/** GnuPG 2.2.8 has `--show-keys`, the CVE-2018-12020 fix and refuses missing-MDC decryption. */ +export const GNUPG_FLOOR = Object.freeze([2, 2, 8]); +/** Bounds for one GnuPG call: its status and colon output are small. */ +export const GNUPG_CALL = Object.freeze({ timeoutMs: 60_000, maxBytes: 256 * 1024 }); +/** The literal-packet name inside the ciphertext, so no host file name enters it. */ +export const LITERAL_NAME = 'relavium-private-maps.tar'; + +/** + * @typedef {{command: string, prefix: string[]}} Command + * @typedef {Command & {gpgconf: Command}} GnupgInstallation + */ + +/** + * Resolve `gpg` from absolute PATH entries only, without a shell, and the `gpgconf` of the same + * installation beside it. + * @returns {GnupgInstallation} + */ +export function resolveGpg(path = process.env.PATH ?? '', platform = process.platform) { + const executable = (name) => (platform === 'win32' ? `${name}.exe` : name); + for (const directory of path.split(delimiter).filter(Boolean)) { + if (!isAbsolute(directory)) continue; + const candidate = join(directory, executable('gpg')); + if (existsSync(candidate) && statSync(candidate).isFile()) { + const command = realpathSync(candidate); + return { + command, + prefix: [], + gpgconf: { command: join(dirname(command), executable('gpgconf')), prefix: [] }, + }; + } + } + return refuseCustody('GPG_UNAVAILABLE'); +} + +/** + * A fresh owner-only GnuPG home. Created here, so no `gpg.conf`, `common.conf` or keyring can already + * be in it. + * @param {string} parent @param {string} name + */ +export function createGnupgHome(parent, name) { + const home = join(parent, name); + mkdirSync(home, { mode: 0o700 }); + return home; +} + +/** The isolated environment for GnuPG, in the C locale, with every home and temporary path in `work`. */ +export function gnupgEnvironment(work) { + return { ...isolatedEnvironment(work), LANG: 'C', LC_ALL: 'C' }; +} + +/** @param {string} home */ +const isolated = (home) => [ + '--homedir', + home, + '--batch', + '--no-tty', + '--no-options', + '--no-autostart', + '--no-keyring', +]; + +/** + * List the keys in one recipient file as colon records, importing nothing. The default trust model + * runs here: GnuPG 2.4 cannot list an expired or revoked key under `--trust-model always` without a + * trust database, so this home may gain an empty one; encryption runs in a separate home. + * @param {string} home @param {string} recipient + */ +export function listArgs(home, recipient) { + return [ + ...isolated(home), + '--with-colons', + '--fixed-list-mode', + '--with-subkey-fingerprint', + '--show-keys', + recipient, + ]; +} + +/** + * Encrypt `input` to the one key in `recipient`, its status on stdout. `--trust-model always`: the + * assurance is the pinned fingerprints and the colon pre-check, and GnuPG treats a `--recipient-file` + * key as fully valid. No compression keeps the ciphertext's size bound. + * @param {string} home @param {string} recipient @param {string} input @param {string} output + */ +export function encryptArgs(home, recipient, input, output) { + return [ + ...isolated(home), + '--no-encrypt-to', + '--no-auto-key-locate', + '--no-auto-key-retrieve', + '--trust-model', + 'always', + '--personal-cipher-preferences', + 'AES256', + '--compress-algo', + 'none', + '--status-fd', + '1', + '--set-filename', + LITERAL_NAME, + '--output', + output, + '--encrypt', + '--recipient-file', + recipient, + input, + ]; +} + +/** + * The recipients of a ciphertext, from its public-key packets: needs no secret key. + * @param {string} home @param {string} ciphertext + */ +export function recipientsArgs(home, ciphertext) { + return [ + ...isolated(home), + '--trust-model', + 'always', + '--status-fd', + '1', + '--list-only', + '--decrypt', + ciphertext, + ]; +} + +/** + * Status lines only: any other line refuses. Unknown keywords and extra arguments are kept for the + * caller to ignore, as GnuPG's DETAILS requires. + * @param {string} text + * @returns {{keyword: string, args: string[]}[]} + */ +export function parseStatus(text) { + return text + .split('\n') + .filter((line, index, lines) => !(index === lines.length - 1 && line === '')) + .map((line) => { + const match = /^\[GNUPG:\] ([A-Z_]+)(?: (.*))?$/u.exec(line); + if (!match) return refuseCustody('STATUS'); + return { keyword: match[1], args: match[2] === undefined ? [] : match[2].split(' ') }; + }); +} + +/** + * Colon records, one array of fields per line. + * @param {string} text + * @returns {string[][]} + */ +export function parseColons(text) { + return text + .split('\n') + .filter((line) => line !== '') + .map((line) => line.split(':')); +} + +/** + * The GnuPG version, refused below the floor. + * @param {{command: string, prefix: string[]}} gpg + * @param {{run: Function, env: Record, home: string}} options + */ +export async function gnupgVersion(gpg, { run, env, home }) { + let stdout; + try { + ({ stdout } = await run(gpg.command, [...gpg.prefix, ...isolated(home), '--version'], { + ...GNUPG_CALL, + env, + })); + } catch { + return refuseCustody('GPG_UNAVAILABLE'); + } + const match = /^gpg \(GnuPG(?:\/[^)]+)?\) (\d+)\.(\d+)\.(\d+)\s/u.exec(String(stdout)); + if (!match) return refuseCustody('GPG_VERSION'); + const version = match.slice(1, 4).map(Number); + for (const [index, floor] of GNUPG_FLOOR.entries()) { + if (version[index] > floor) break; + if (version[index] < floor) refuseCustody('GPG_VERSION'); + } + return version.join('.'); +} + +/** + * GnuPG reads `/gpg.conf` even under `--no-options`, and `[force]` options there override + * the command line: a global configuration could add or hide a recipient, redirect the output or log + * the session key. The installation's `gpgconf` names that directory; a `gpg.conf` or `common.conf` + * in it refuses. + * @param {GnupgInstallation} gpg + * @param {{run: Function, env: Record}} options + */ +export async function assertNoGlobalConfig(gpg, { run, env }) { + let stdout; + try { + ({ stdout } = await run( + gpg.gpgconf.command, + [...gpg.gpgconf.prefix, '--list-dirs', 'sysconfdir'], + { ...GNUPG_CALL, env }, + )); + } catch { + return refuseCustody('GPG_UNAVAILABLE'); + } + // A named directory is printed raw and ended by one newline; only the unnamed listing is + // percent-escaped. Anything that is not exactly one absolute path refuses. + const text = String(stdout); + const directory = text.endsWith('\n') ? text.slice(0, -1) : ''; + if (!isAbsolute(directory) || /[\r\n\0]/u.test(directory)) refuseCustody('GPG_CONFIG'); + for (const name of ['gpg.conf', 'common.conf']) { + if (existsSync(join(directory, name))) refuseCustody('GPG_CONFIG'); + } +} + +/** + * The home may hold only regular files GnuPG itself creates for these calls: never a socket, a + * configuration, a keyring or private-key material. + * @param {string} home @param {readonly string[]} allowed + */ +export function assertHomeState(home, allowed) { + for (const name of readdirSync(home)) { + if (!allowed.includes(name) || !lstatSync(join(home, name)).isFile()) { + refuseCustody('HOME_STATE'); + } + } +} diff --git a/tools/cli-package/gnupg.test.mjs b/tools/cli-package/gnupg.test.mjs new file mode 100644 index 00000000..14633b9e --- /dev/null +++ b/tools/cli-package/gnupg.test.mjs @@ -0,0 +1,286 @@ +import assert from 'node:assert/strict'; +import { + mkdirSync, + mkdtempSync, + realpathSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { delimiter, dirname, isAbsolute, join, relative } from 'node:path'; +import { after, test } from 'node:test'; +import { CustodyError } from './custody-error.mjs'; +import { + GNUPG_FLOOR, + assertHomeState, + assertNoGlobalConfig, + createGnupgHome, + encryptArgs, + gnupgEnvironment, + gnupgVersion, + listArgs, + parseColons, + parseStatus, + recipientsArgs, + resolveGpg, +} from './gnupg.mjs'; +import { runCommand } from './process.mjs'; + +const scratch = mkdtempSync(join(tmpdir(), 'relavium-gnupg-')); +after(() => rmSync(scratch, { recursive: true, force: true })); +const refuses = (code) => (error) => error instanceof CustodyError && error.code === code; + +test('gpg resolves from absolute PATH entries only and is required, never skipped', async () => { + const gpg = resolveGpg(); + assert.ok(gpg.command.length > 0 && gpg.prefix.length === 0); + // The installation's own gpgconf sits beside the resolved gpg. + assert.equal(dirname(gpg.gpgconf.command), dirname(gpg.command)); + assert.match(gpg.gpgconf.command, /gpgconf(?:\.exe)?$/u); + // The real binary meets the floor; this is the prerequisite local `pnpm run ci` and CI rely on. + const home = createGnupgHome(scratch, 'version'); + const version = await gnupgVersion(gpg, { + run: runCommand, + env: gnupgEnvironment(scratch), + home, + }); + assert.match(version, /^\d+\.\d+\.\d+$/u); + assert.throws(() => resolveGpg(join(scratch, 'none')), refuses('GPG_UNAVAILABLE')); + // A relative entry is never searched, even when it resolves to a directory holding a gpg. + mkdirSync(join(scratch, 'relative')); + writeFileSync(join(scratch, 'relative', process.platform === 'win32' ? 'gpg.exe' : 'gpg'), ''); + const entry = relative(process.cwd(), join(scratch, 'relative')); + assert.equal(isAbsolute(entry), false); + assert.throws( + () => resolveGpg([entry, join(scratch, 'none')].join(delimiter)), + refuses('GPG_UNAVAILABLE'), + ); + assert.deepEqual(resolveGpg(join(scratch, 'relative')).prefix, []); + // A directory named gpg is skipped, never chosen. + const directoryEntry = createGnupgHome(scratch, 'gpg-directory'); + mkdirSync(join(directoryEntry, 'gpg')); + assert.equal( + resolveGpg([directoryEntry, dirname(gpg.command)].join(delimiter)).command, + gpg.command, + ); +}); + +test("gpgconf is the one beside the linked gpg's real binary, never a neighbour of the link", () => { + const real = resolveGpg(); + const farm = createGnupgHome(scratch, 'link-farm'); + symlinkSync(real.command, join(farm, 'gpg')); + writeFileSync(join(farm, 'gpgconf'), '#!/bin/sh\necho decoy\n', { mode: 0o700 }); + const linked = resolveGpg(farm); + assert.equal(linked.command, realpathSync(join(farm, 'gpg'))); + assert.equal(linked.gpgconf.command, join(dirname(linked.command), 'gpgconf')); + assert.notEqual(dirname(linked.gpgconf.command), farm); +}); + +test('the version floor is GnuPG 2.2.8, read from the version line only', async () => { + assert.deepEqual(GNUPG_FLOOR, [2, 2, 8]); + const gpg = { command: 'gpg', prefix: [] }; + const version = (stdout) => + gnupgVersion(gpg, { run: async () => ({ stdout, stderr: '' }), env: {}, home: scratch }); + for (const line of [ + 'gpg (GnuPG) 2.2.8\n', + 'gpg (GnuPG/MacGPG2) 2.2.41\n', + 'gpg (GnuPG) 2.4.4\n', + ]) { + assert.match(await version(line), /^2\.\d+\.\d+$/u); + } + for (const line of ['gpg (GnuPG) 2.2.7\n', 'gpg (GnuPG) 2.1.23\n', 'gpg (GnuPG) 1.4.23\n']) { + await assert.rejects(version(line), refuses('GPG_VERSION')); + } + await assert.rejects(version('gpg 2.4.4\n'), refuses('GPG_VERSION')); + // The version call is isolated like every other call. + let argv = []; + await gnupgVersion(gpg, { + run: async (_command, args) => { + argv = args; + return { stdout: 'gpg (GnuPG) 2.4.4\n', stderr: '' }; + }, + env: {}, + home: 'H', + }); + for (const flag of [ + '--homedir', + '--batch', + '--no-tty', + '--no-options', + '--no-autostart', + '--no-keyring', + ]) { + assert.ok(argv.includes(flag), flag); + } + assert.equal(argv.at(-1), '--version'); + await assert.rejects( + gnupgVersion(gpg, { + run: async () => { + throw new Error('spawn failed'); + }, + env: {}, + home: scratch, + }), + refuses('GPG_UNAVAILABLE'), + ); +}); + +test('every call is isolated, and encryption never passes a flag that widens it', () => { + const required = [ + '--homedir', + '--batch', + '--no-tty', + '--no-options', + '--no-autostart', + '--no-keyring', + ]; + const forbidden = [ + '--yes', + '-R', + '-F', + '-r', + '--recipient', + '--hidden-recipient', + '--throw-keyids', + '--faked-system-time', + '--ignore-time-conflict', + '--ignore-valid-from', + '--keyring', + '--import', + '--encrypt-to', + ]; + const calls = [listArgs('H', 'K'), encryptArgs('H', 'K', 'I', 'O'), recipientsArgs('H', 'C')]; + for (const args of calls) { + for (const flag of required) assert.ok(args.includes(flag), flag); + assert.equal(args[args.indexOf('--homedir') + 1], 'H'); + for (const flag of forbidden) assert.ok(!args.includes(flag), flag); + assert.ok(!args.some((arg) => arg.startsWith('--allow-') || arg.startsWith('--ignore-'))); + } + const encrypt = encryptArgs('H', 'K', 'I', 'O'); + for (const [flag, value] of [ + ['--trust-model', 'always'], + ['--personal-cipher-preferences', 'AES256'], + ['--compress-algo', 'none'], + ['--status-fd', '1'], + ['--set-filename', 'relavium-private-maps.tar'], + ['--output', 'O'], + ['--recipient-file', 'K'], + ]) { + assert.equal(encrypt[encrypt.indexOf(flag) + 1], value, flag); + } + for (const flag of ['--no-encrypt-to', '--no-auto-key-locate', '--no-auto-key-retrieve']) { + assert.ok(encrypt.includes(flag), flag); + } + assert.equal(encrypt.at(-1), 'I'); +}); + +test('the environment is isolated in the C locale and every home is owner-only', () => { + // Planted in this process's environment, so a builder that leaked it would carry them. + const planted = { + GNUPGHOME: '/planted', + GPG_AGENT_INFO: '/planted', + GPG_TTY: '/planted', + GNUPG_HOME: '/planted', + LD_PRELOAD: '/planted', + DYLD_INSERT_LIBRARIES: '/planted', + LANG: 'tr_TR.UTF-8', + LC_ALL: 'tr_TR.UTF-8', + }; + const saved = Object.keys(planted).map((name) => [name, process.env[name]]); + Object.assign(process.env, planted); + let env; + try { + env = gnupgEnvironment(scratch); + } finally { + for (const [name, value] of saved) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + } + assert.equal(env.LANG, 'C'); + assert.equal(env.LC_ALL, 'C'); + for (const name of ['HOME', 'TMPDIR', 'XDG_CONFIG_HOME']) assert.equal(env[name], scratch, name); + for (const name of Object.keys(planted).filter((key) => !['LANG', 'LC_ALL'].includes(key))) { + assert.equal(env[name], undefined, name); + } + assert.ok(!Object.values(env).includes('/planted')); + const home = createGnupgHome(scratch, 'owner-only'); + if (process.platform !== 'win32') assert.equal(statSync(home).mode & 0o777, 0o700); +}); + +test('a global configuration in the sysconfdir gpgconf names refuses', async () => { + const gpg = { command: 'gpg', prefix: [], gpgconf: { command: 'gpgconf', prefix: [] } }; + const check = (stdout) => + assertNoGlobalConfig(gpg, { run: async () => ({ stdout, stderr: '' }), env: {} }); + const empty = createGnupgHome(scratch, 'sysconf-empty'); + await check(`${empty}\n`); + const planted = createGnupgHome(scratch, 'sysconf-planted'); + writeFileSync(join(planted, 'gpg.conf'), 'encrypt-to 0123456789ABCDEF\n'); + await assert.rejects(check(`${planted}\n`), refuses('GPG_CONFIG')); + // Only the configuration files GnuPG reads count; another daemon's file is not one. + const other = createGnupgHome(scratch, 'sysconf-other'); + writeFileSync(join(other, 'scdaemon.conf'), 'disable-ccid\n'); + await check(`${other}\n`); + // Debian and Ubuntu name a directory they do not ship: nothing there to read. + await check(`${join(scratch, 'sysconf-missing')}\n`); + // A named directory is printed raw: a literal `%41` is part of the path, never an escape. + const raw = createGnupgHome(scratch, 'sysconf %41'); + writeFileSync(join(raw, 'gpg.conf'), 'encrypt-to 0123456789ABCDEF\n'); + await assert.rejects(check(`${raw}\n`), refuses('GPG_CONFIG')); + // Anything but exactly one absolute path ended by one newline refuses. + for (const answer of [ + 'relative/dir\n', + '\n', + '', + empty, + `${empty}\n${empty}\n`, + `${empty}\r\n`, + `${empty}\0\n`, + ]) { + await assert.rejects(check(answer), refuses('GPG_CONFIG'), JSON.stringify(answer)); + } + // The real installation has none, so every encryption may proceed. + const real = resolveGpg(); + await assertNoGlobalConfig(real, { run: runCommand, env: gnupgEnvironment(scratch) }); +}); + +test('status parsing admits only status lines and keeps unknown keywords for the caller', () => { + assert.deepEqual(parseStatus('[GNUPG:] BEGIN_ENCRYPTION 2 9\n[GNUPG:] NEWSIG\n'), [ + { keyword: 'BEGIN_ENCRYPTION', args: ['2', '9'] }, + { keyword: 'NEWSIG', args: [] }, + ]); + assert.deepEqual(parseStatus(''), []); + for (const text of [ + 'gpg: encrypted\n', + '[GNUPG:] begin\n', + '[GNUPG:] BEGIN_ENCRYPTION 2 9\n\n', + '[GNUPG:]BEGIN_ENCRYPTION\n', + 'gpg: noise [GNUPG:] NEWSIG\n', + ]) { + assert.throws(() => parseStatus(text), refuses('STATUS'), text); + } + assert.deepEqual(parseColons('pub:-:255\nfpr:::\n'), [ + ['pub', '-', '255'], + ['fpr', '', '', ''], + ]); +}); + +test('a home may hold only the regular files GnuPG creates for these calls', () => { + const home = createGnupgHome(scratch, 'state'); + assertHomeState(home, ['random_seed']); + writeFileSync(join(home, 'random_seed'), ''); + assertHomeState(home, ['random_seed']); + for (const [index, plant] of [ + (path) => writeFileSync(join(path, 'common.conf'), 'use-keyboxd\n'), + (path) => mkdirSync(join(path, 'private-keys-v1.d')), + (path) => symlinkSync(join(scratch, 'elsewhere'), join(path, 'S.gpg-agent')), + ].entries()) { + const planted = createGnupgHome(scratch, `planted-${index}`); + plant(planted); + assert.throws(() => assertHomeState(planted, ['random_seed']), refuses('HOME_STATE')); + } + const linkedSeed = createGnupgHome(scratch, 'linked-seed'); + symlinkSync(join(scratch, 'elsewhere'), join(linkedSeed, 'random_seed')); + assert.throws(() => assertHomeState(linkedSeed, ['random_seed']), refuses('HOME_STATE')); +}); diff --git a/tools/cli-package/gpg-fixture.mjs b/tools/cli-package/gpg-fixture.mjs new file mode 100644 index 00000000..d654f2a5 --- /dev/null +++ b/tools/cli-package/gpg-fixture.mjs @@ -0,0 +1,108 @@ +/** + * Test-only GnuPG fixtures (Step 9b.1): the committed synthetic public keys and their pins, and an + * ephemeral decryption key generated per run in a short owner-only home. Its public export and, for + * the secret-material refusal, its armored secret export go to a second owner-only temporary + * directory. Both directories are removed and the home's agent stopped when the test file ends; no + * secret key is ever committed. + */ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { after } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { resolveGpg } from './gnupg.mjs'; + +export const FIXTURES = fileURLToPath(new URL('./custody-fixtures/', import.meta.url)); +/** @type {Record} */ +export const PINS = JSON.parse(readFileSync(join(FIXTURES, 'pins.json'), 'utf8')); +/** @param {string} name */ +export const fixture = (name) => join(FIXTURES, `${name}.asc`); + +const NO_AEAD = 'AES256 AES192 AES SHA512 SHA384 SHA256 ZLIB ZIP Uncompressed'; +/** The longest agent socket name GnuPG creates in a home, against macOS's 104-byte socket path limit. */ +const SOCKET = '/S.gpg-agent.browser'; + +/** Every process whose command line names `home`. */ +export function processesNaming(home) { + // An absolute path, not a PATH lookup; `/bin/ps` exists on macOS and on Linux (merged-/usr or not). + const listing = spawnSync( + '/bin/ps', + process.platform === 'darwin' ? ['-axo', 'command'] : ['-eo', 'args'], + { encoding: 'utf8' }, + ); + return listing.stdout.split('\n').filter((line) => line.includes(home)); +} + +/** + * A fresh synthetic key pair whose secret key stays in its own short home: the public key (only the + * pinned encryption subkey) in a file, its pins, the secret key's export for refusal tests, and + * decryption. The home's agent is stopped and the home removed after the test file. + */ +export function ephemeralKey() { + const gpg = resolveGpg(); + // Checked before anything is created: `mkdtemp` appends six characters to the prefix. + const prefix = join(tmpdir(), 'rg-'); + assert.ok(prefix.length + 6 + SOCKET.length < 100, 'the ephemeral key home path is too long'); + const home = mkdtempSync(prefix); + const exportDirectory = mkdtempSync(join(tmpdir(), 'rge-')); + const env = { PATH: process.env.PATH ?? '', LANG: 'C', LC_ALL: 'C' }; + /** @param {string[]} args @param {string} [input] */ + const call = (args, input) => { + const run = spawnSync( + gpg.command, + [ + ...gpg.prefix, + '--homedir', + home, + '--batch', + '--no-tty', + '--no-options', + '--pinentry-mode', + 'loopback', + '--passphrase', + '', + ...args, + ], + { encoding: 'utf8', input, env, timeout: 60_000 }, + ); + assert.equal(run.status, 0, `ephemeral gpg ${args[0]} failed`); + return run.stdout; + }; + const dispose = () => { + // The `gpgconf` of the same installation, not a PATH lookup. + spawnSync(gpg.gpgconf.command, ['--homedir', home, '--kill', 'all'], { stdio: 'ignore', env }); + spawnSync(gpg.gpgconf.command, ['--homedir', home, '--remove-socketdir'], { + stdio: 'ignore', + env, + }); + rmSync(home, { recursive: true, force: true }); + rmSync(exportDirectory, { recursive: true, force: true }); + }; + after(dispose); + const uid = 'ephemeral '; + call(['--quick-gen-key', uid, 'ed25519', 'cert', 'never']); + const listing = () => + call(['--with-colons', '--fixed-list-mode', '--with-subkey-fingerprint', '--list-keys', uid]) + .split('\n') + .filter((line) => line.startsWith('fpr:')) + .map((line) => line.split(':')[9]); + const [primary] = listing(); + call(['--quick-add-key', primary, 'cv25519', 'encr', 'never']); + call(['--command-fd', '0', '--edit-key', primary], `setpref ${NO_AEAD}\ny\nsave\n`); + const [, subkey] = listing(); + const file = join(exportDirectory, 'public.asc'); + writeFileSync(file, call(['--armor', '--export', `${subkey}!`])); + const secret = join(exportDirectory, 'secret.asc'); + writeFileSync(secret, call(['--armor', '--export-secret-keys', primary])); + return { + home, + file, + secret, + pins: { primary, subkey }, + /** @param {string} ciphertext @param {string} output */ + decrypt: (ciphertext, output) => call(['--output', output, '--decrypt', ciphertext]), + dispose, + }; +} diff --git a/tools/cli-package/manifest.mjs b/tools/cli-package/manifest.mjs new file mode 100644 index 00000000..f094b099 --- /dev/null +++ b/tools/cli-package/manifest.mjs @@ -0,0 +1,350 @@ +/** + * The private map manifest (ADR-0111; packaging.md, "private map manifest"; Step 9a.2a). Sealed into + * private staging only after the archive inspection, it binds the source commit and its tree, the + * package version, the toolchain and target, the lock and build-configuration digests, every final JS + * file as `dist/.js` with its map, a digest of every original source a final map names, and the + * tarball digest. It holds no host path, credential or source content, and it is removed with the + * staging; Step 9b carries it into ciphertext. + */ +import { Buffer } from 'node:buffer'; +import { readdirSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { basename, join } from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; +import { isPackageVersion } from './binding.mjs'; +import { ENGINE_PACKAGES, writePrivateFile } from './finalize.mjs'; +import { readOrdinaryFile, sha256, byCodeUnit } from './files.mjs'; +import { isolatedGitEnvironment, runCommand } from './process.mjs'; + +export const MANIFEST_FILE = 'manifest.json'; + +/** + * The candidate files whose bytes decide the build: the lock, every package manifest, and every build + * configuration with the configurations it extends. + */ +export const MANIFEST_INPUTS = Object.freeze([ + '.npmrc', + 'package.json', + 'pnpm-lock.yaml', + 'pnpm-workspace.yaml', + 'tsconfig.base.json', + 'apps/cli/package.json', + 'apps/cli/tsconfig.json', + 'apps/cli/tsup.config.ts', + ...ENGINE_PACKAGES.flatMap((name) => [ + `packages/${name}/package.json`, + `packages/${name}/tsconfig.json`, + `packages/${name}/tsconfig.build.json`, + ]), + 'packages/core/tsconfig.purity.json', +]); + +const SHA256 = /^[0-9a-f]{64}$/u; +const GIT_OBJECT = /^[0-9a-f]{40}$/u; +const OUTPUT = /^dist\/[A-Za-z0-9_-]+\.js$/u; +const SEGMENT = '[A-Za-z0-9_-][A-Za-z0-9_.-]*'; +/** A repository-relative original source the finalizer admits: CLI or engine TypeScript. */ +const SOURCE = new RegExp( + String.raw`^(?:apps/cli|packages/(?:${ENGINE_PACKAGES.join('|')}))/src/(?:${SEGMENT}/)*${SEGMENT}\.tsx?$`, + 'u', +); +const VERSION = /^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]{1,64})?$/u; +const LIMITS = Object.freeze({ outputs: 256, sources: 8192, recordBytes: 1024 * 1024 }); +/** + * The size bound each staged record's reader applies, by file name: no record in the custody container + * exceeds the size its reader accepts. + */ +export const RECORD_BYTES = Object.freeze({ + 'finalized.json': LIMITS.recordBytes, + [MANIFEST_FILE]: LIMITS.recordBytes * 4, + 'metafile-esm.json': 4 * 1024 * 1024, +}); + +export class ManifestError extends Error { + /** @param {string} code */ + constructor(code) { + super(`PACKED_CLI_MANIFEST_${code}`); + this.name = 'ManifestError'; + this.code = code; + } +} + +/** @param {string} code @returns {never} */ +function refuse(code) { + throw new ManifestError(code); +} + +/** @param {string} root @param {string} path repository-relative POSIX path @param {number} limit */ +function read(root, path, limit) { + try { + return readOrdinaryFile(join(root, ...path.split('/')), limit); + } catch { + refuse('UNREADABLE_INPUT'); + } +} + +/** @param {unknown} value @returns {value is Record} */ +function isRecord(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +/** + * @typedef {{version: 1, source: {commit: string, tree: string, dirty: boolean}, + * packageVersion: string, target: string, + * toolchain: {node: string, pnpm: string, typescript: string, tsup: string, esbuild: string}, + * inputs: Record, + * outputs: {path: string, sha256: string, map: {path: string, sha256: string}}[], + * sources: Record, tarballSha256: string}} PrivateManifest + */ + +/** + * Validate a manifest's exact shape. Every key is fixed, every digest is SHA-256 hex, every path is + * repository- or package-relative, and the outputs and sources are sorted and unique. + * @param {unknown} value + * @returns {PrivateManifest} + */ +export function validateManifest(value) { + const exact = (record, keys) => + isRecord(record) && + isDeepStrictEqual(Object.keys(record).sort(byCodeUnit), [...keys].sort(byCodeUnit)); + if ( + !exact(value, [ + 'version', + 'source', + 'packageVersion', + 'target', + 'toolchain', + 'inputs', + 'outputs', + 'sources', + 'tarballSha256', + ]) || + value.version !== 1 + ) { + refuse('SHAPE'); + } + const { source, toolchain, inputs, outputs, sources } = value; + if ( + !exact(source, ['commit', 'tree', 'dirty']) || + !GIT_OBJECT.test(String(source.commit)) || + !GIT_OBJECT.test(String(source.tree)) || + typeof source.dirty !== 'boolean' || + !isPackageVersion(value.packageVersion) || + typeof value.target !== 'string' || + !/^node\d{2,3}$/u.test(value.target) || + !exact(toolchain, ['node', 'pnpm', 'typescript', 'tsup', 'esbuild']) || + !/^v\d+\.\d+\.\d+$/u.test(String(toolchain.node)) || + toolchain.pnpm !== '9.12.3' || + !['typescript', 'tsup', 'esbuild'].every((name) => VERSION.test(String(toolchain[name]))) || + !exact(inputs, MANIFEST_INPUTS) || + !Object.values(inputs).every((digest) => SHA256.test(String(digest))) || + typeof value.tarballSha256 !== 'string' || + !SHA256.test(value.tarballSha256) + ) { + refuse('SHAPE'); + } + if ( + !Array.isArray(outputs) || + outputs.length === 0 || + outputs.length > LIMITS.outputs || + !outputs.every( + (output, index) => + exact(output, ['path', 'sha256', 'map']) && + OUTPUT.test(String(output.path)) && + SHA256.test(String(output.sha256)) && + exact(output.map, ['path', 'sha256']) && + output.map.path === `${output.path}.map` && + SHA256.test(String(output.map.sha256)) && + (index === 0 || outputs[index - 1].path < output.path), + ) + ) { + refuse('SHAPE'); + } + const names = isRecord(sources) ? Object.keys(sources) : []; + if ( + !isRecord(sources) || + names.length === 0 || + names.length > LIMITS.sources || + !names.every( + (name, index) => + SOURCE.test(name) && + SHA256.test(String(sources[name])) && + (index === 0 || names[index - 1] < name), + ) + ) { + refuse('SHAPE'); + } + return /** @type {PrivateManifest} */ (value); +} + +/** + * Seal the private manifest into `staging`, after the archive inspection has proven the tarball's + * JS byte-equal to the finalized files. The staged maps must still be the bytes the finalizer + * recorded, and each recorded JS digest must be the tarball entry's. An existing manifest refuses. + * @param {{candidate: string, staging: string, checkout: string, source: string, dirty: boolean, + * archive: {sha256: string, files: {path: string, sha256: string}[], package: {version: string}}, + * run?: typeof runCommand}} options + * @returns {Promise} + */ +export async function sealPrivateManifest({ + candidate, + staging, + checkout, + source, + dirty, + archive, + run = runCommand, +}) { + let record; + try { + record = JSON.parse( + readOrdinaryFile(join(staging, 'finalized.json'), RECORD_BYTES['finalized.json']).toString( + 'utf8', + ), + ); + } catch { + refuse('RECORD'); + } + if (!isRecord(record) || !Array.isArray(record.outputs) || !Array.isArray(record.maps)) { + refuse('RECORD'); + } + const maps = new Map(record.maps.map((map) => [isRecord(map) ? map.path : undefined, map])); + const packed = new Map(archive.files.map((file) => [file.path, file.sha256])); + const named = new Set(); + const outputs = record.outputs.map((output) => { + const map = isRecord(output) ? maps.get(`${output.path}.map`) : undefined; + if (!isRecord(output) || !isRecord(map) || !OUTPUT.test(String(output.path))) refuse('RECORD'); + const bytes = read(staging, `maps/${basename(output.path)}.map`, 16 * 1024 * 1024); + if (bytes.length !== map.bytes || sha256(bytes) !== map.sha256) refuse('MAP_DRIFT'); + if (packed.get(`package/${output.path}`) !== output.sha256) refuse('TARBALL_DRIFT'); + let parsed; + try { + parsed = JSON.parse(bytes.toString('utf8')); + } catch { + refuse('MAP_DRIFT'); + } + if (!isRecord(parsed) || !Array.isArray(parsed.sources)) refuse('MAP_DRIFT'); + for (const name of parsed.sources) named.add(name); + return { + path: output.path, + sha256: output.sha256, + map: { path: map.path, sha256: map.sha256 }, + }; + }); + if (maps.size !== outputs.length) refuse('RECORD'); + outputs.sort((left, right) => (left.path < right.path ? -1 : 1)); + const sources = Object.fromEntries( + [...named] + .map(String) + .sort(byCodeUnit) + .map((name) => { + if (!SOURCE.test(name)) refuse('SOURCE_PATH'); + return [name, sha256(read(candidate, name, 8 * 1024 * 1024))]; + }), + ); + let tree; + try { + tree = ( + await run('git', ['--no-replace-objects', 'rev-parse', '--verify', `${source}^{tree}`], { + cwd: checkout, + env: isolatedGitEnvironment(), + timeoutMs: 60_000, + }) + ).stdout.trim(); + } catch { + refuse('SOURCE_TREE'); + } + if (!GIT_OBJECT.test(tree)) refuse('SOURCE_TREE'); + const cli = createRequire(join(candidate, 'apps', 'cli', 'package.json')); + const version = (require, name) => { + try { + return String(require(`${name}/package.json`).version); + } catch { + refuse('TOOLCHAIN'); + } + }; + const toolchain = { + node: process.version, + pnpm: '9.12.3', + typescript: version(cli, 'typescript'), + tsup: version(cli, 'tsup'), + esbuild: version(createRequire(cli.resolve('tsup/package.json')), 'esbuild'), + }; + const targets = [ + ...read(candidate, 'apps/cli/tsup.config.ts', 128 * 1024) + .toString('utf8') + .matchAll(/^ {2}target: '([^']+)',$/gmu), + ]; + if (targets.length !== 1) refuse('TARGET'); + const manifest = validateManifest({ + version: 1, + source: { commit: source, tree, dirty }, + packageVersion: archive.package.version, + target: targets[0][1], + toolchain, + inputs: Object.fromEntries( + MANIFEST_INPUTS.map((path) => [path, sha256(read(candidate, path, 16 * 1024 * 1024))]), + ), + outputs, + sources, + tarballSha256: archive.sha256, + }); + writePrivateFile( + join(staging, MANIFEST_FILE), + Buffer.from(JSON.stringify(manifest, null, 2) + '\n', 'utf8'), + ); + return manifest; +} + +/** + * Verify that the finalized `dist` and the tarball are still the ones `manifest` records: exactly its + * outputs, each with its digest, and the tarball's digest (ADR-0111's post-receipt rebuild refusal; + * Step 9a.2b). The seal runs it with the manifest it has just sealed, after symbolication and + * immediately before the receipt is written, so a `dist` rebuilt or changed after the inspection — in + * a chunk no frame lands in as well — refuses. + * @param {{candidate: string, manifest: PrivateManifest, tarball: string}} options + */ +export function verifySealedDist({ candidate, manifest, tarball }) { + const cli = join(candidate, 'apps', 'cli'); + let names; + try { + names = readdirSync(join(cli, 'dist')).sort(); + } catch { + refuse('DIST_DRIFT'); + } + const expected = manifest.outputs.map((output) => basename(output.path)).sort(byCodeUnit); + if (!isDeepStrictEqual(names, expected)) refuse('DIST_DRIFT'); + for (const output of manifest.outputs) { + let bytes; + try { + bytes = readOrdinaryFile(join(cli, ...output.path.split('/')), 8 * 1024 * 1024); + } catch { + refuse('DIST_DRIFT'); + } + if (sha256(bytes) !== output.sha256) refuse('DIST_DRIFT'); + } + let packed; + try { + packed = readOrdinaryFile(tarball, 16 * 1024 * 1024); + } catch { + refuse('TARBALL_CHANGED'); + } + if (sha256(packed) !== manifest.tarballSha256) refuse('TARBALL_CHANGED'); +} + +/** + * Read and validate the sealed manifest from `staging`. + * @param {string} staging + * @returns {PrivateManifest} + */ +export function readPrivateManifest(staging) { + let value; + try { + value = JSON.parse( + readOrdinaryFile(join(staging, MANIFEST_FILE), RECORD_BYTES[MANIFEST_FILE]).toString('utf8'), + ); + } catch { + refuse('UNREADABLE'); + } + return validateManifest(value); +} diff --git a/tools/cli-package/manifest.test.mjs b/tools/cli-package/manifest.test.mjs new file mode 100644 index 00000000..93a42cef --- /dev/null +++ b/tools/cli-package/manifest.test.mjs @@ -0,0 +1,645 @@ +import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; +import { createRequire } from 'node:module'; +import { + existsSync, + mkdirSync, + readFileSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { join, posix } from 'node:path'; +import { test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { + COMMIT, + SOURCES, + TREE, + candidate, + cliRequire, + coreChunk, + digest, + fresh, + git, + posixModes, + rerecordMaps, + scratch, + seal, + ts, +} from './candidate-fixture.mjs'; +import { ENGINE_PACKAGES } from './finalize.mjs'; +import { + MANIFEST_INPUTS, + ManifestError, + readPrivateManifest, + sealPrivateManifest, + verifySealedDist, +} from './manifest.mjs'; + +const refuses = (code) => (error) => error instanceof ManifestError && error.code === code; +const repository = fileURLToPath(new URL('../../', import.meta.url)); + +test('the manifest inputs are every build configuration the real repository extends', () => { + // Each configuration a build reads, followed through its `extends` chain, is a digested input. + const roots = [ + 'apps/cli/tsconfig.json', + ...ENGINE_PACKAGES.map((name) => `packages/${name}/tsconfig.build.json`), + ]; + const reached = new Set(); + const follow = (path) => { + reached.add(path); + const { config, error } = ts.readConfigFile( + join(repository, ...path.split('/')), + ts.sys.readFile, + ); + assert.equal(error, undefined, path); + if (config.extends === undefined) return; + assert.equal(typeof config.extends, 'string', path); + assert.match(config.extends, /^\.\.?\//u, `${path} extends a relative configuration`); + follow(posix.normalize(posix.join(posix.dirname(path), config.extends))); + }; + for (const root of roots) follow(root); + // With the lock, the workspace and npm configuration, the tsup configuration and every package + // manifest the build installs, that is exactly the input list. + const expected = new Set([ + ...reached, + '.npmrc', + 'package.json', + 'pnpm-lock.yaml', + 'pnpm-workspace.yaml', + 'apps/cli/package.json', + 'apps/cli/tsup.config.ts', + ...ENGINE_PACKAGES.map((name) => `packages/${name}/package.json`), + ]); + assert.equal(MANIFEST_INPUTS.length, expected.size); + assert.deepEqual(new Set(MANIFEST_INPUTS), expected); + for (const path of MANIFEST_INPUTS) + assert.ok(existsSync(join(repository, ...path.split('/'))), path); +}); + +test('the manifest binds the commit, toolchain, inputs, finalized pairs, sources and tarball', async () => { + const fixture = await candidate(); + const manifest = await seal(fixture); + assert.deepEqual(manifest.source, { commit: COMMIT, tree: TREE, dirty: true }); + assert.equal(manifest.packageVersion, '0.1.1'); + assert.equal(manifest.target, 'node22'); + assert.equal(manifest.tarballSha256, 'a'.repeat(64)); + assert.equal(manifest.toolchain.node, process.version); + assert.equal(manifest.toolchain.pnpm, '9.12.3'); + assert.equal(manifest.toolchain.typescript, ts.version); + // The installed tsup, and the esbuild tsup itself resolves. + assert.equal(manifest.toolchain.tsup, cliRequire('tsup/package.json').version); + assert.equal( + manifest.toolchain.esbuild, + createRequire(cliRequire.resolve('tsup/package.json'))('esbuild/package.json').version, + ); + assert.deepEqual(Object.keys(manifest.inputs), [...MANIFEST_INPUTS]); + for (const path of MANIFEST_INPUTS) { + assert.equal( + manifest.inputs[path], + digest(readFileSync(join(fixture.root, ...path.split('/')))), + path, + ); + } + assert.deepEqual( + manifest.outputs.map((output) => output.path), + fixture.outputs.map((output) => output.path).sort(), + ); + for (const output of manifest.outputs) { + assert.equal(output.sha256, digest(readFileSync(join(fixture.cli, output.path)))); + assert.deepEqual(output.map, { + path: `${output.path}.map`, + sha256: digest( + readFileSync(join(fixture.staging, 'maps', `${posix.basename(output.path)}.map`)), + ), + }); + } + assert.deepEqual(Object.keys(manifest.sources), Object.keys(SOURCES).sort()); + for (const [path, value] of Object.entries(manifest.sources)) { + assert.equal(value, digest(readFileSync(join(fixture.root, ...path.split('/'))))); + } + // Written once, owner-only, holding no absolute path, and read back exactly. + const text = readFileSync(join(fixture.staging, 'manifest.json'), 'utf8'); + assert.deepEqual(JSON.parse(text), manifest); + assert.deepEqual(readPrivateManifest(fixture.staging), manifest); + assert.equal(text.includes(scratch), false); + if (posixModes) + assert.equal(statSync(join(fixture.staging, 'manifest.json')).mode & 0o777, 0o600); + await assert.rejects(seal(fixture), (error) => error.code === 'EEXIST'); +}); + +test('the package version and tarball digest are the inspected archive’s', async () => { + const fixture = await candidate(); + fixture.archive.package.version = '9.8.7'; + fixture.archive.sha256 = 'e'.repeat(64); + const manifest = await seal(fixture); + assert.equal(manifest.packageVersion, '9.8.7'); + assert.equal(manifest.tarballSha256, 'e'.repeat(64)); + // Every version a receipt admits: a prerelease with build metadata, and exactly 100 characters. + for (const version of ['1.0.0-rc.1+build.5', `0.1.1-${'a'.repeat(94)}`]) { + const admitted = await candidate(); + admitted.archive.package.version = version; + assert.equal((await seal(admitted)).packageVersion, version); + } +}); + +test('the tree is asked of the original checkout through isolated Git', async () => { + const fixture = await candidate(); + // Sources and toolchain come from the candidate, so an absent checkout serves only Git. + const checkout = join(scratch, 'absent-checkout'); + let seen; + await sealPrivateManifest({ + candidate: fixture.root, + staging: fixture.staging, + checkout, + source: COMMIT, + dirty: false, + archive: fixture.archive, + run: async (command, args, options) => { + seen = options; + return git(command, args); + }, + }); + assert.equal(seen.cwd, checkout); + assert.equal(seen.env.GIT_CONFIG_NOSYSTEM, '1'); + assert.equal(seen.env.GIT_NO_REPLACE_OBJECTS, '1'); + assert.equal(seen.env.GIT_DIR, undefined); + assert.ok(Number.isFinite(seen.timeoutMs) && seen.timeoutMs >= 10_000); +}); + +test('the manifest refuses drifted maps and tarball entries, unpaired records and a missing tree', async () => { + for (const [label, change, code] of [ + [ + 'a staged map changed after the finalizer recorded it', + (fixture) => + writeFileSync(join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`), '{"sources":[]}'), + 'MAP_DRIFT', + ], + [ + 'a staged map changed to bytes of the same length', + (fixture) => { + const path = join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`); + const text = readFileSync(path, 'utf8'); + assert.equal(text.startsWith('{"version":3,'), true); + writeFileSync(path, text.replace('{"version":3,', '{"version":4,')); + }, + 'MAP_DRIFT', + ], + [ + 'a recorded map size that disagrees with its bytes', + (fixture) => { + const name = coreChunk(fixture); + rerecordMaps(fixture, (record) => { + record.maps.find((map) => map.path === `dist/${name}.map`).bytes += 1; + }); + }, + 'MAP_DRIFT', + ], + [ + 'a recorded map that is not JSON', + (fixture) => { + writeFileSync(join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`), 'not json'); + rerecordMaps(fixture); + }, + 'MAP_DRIFT', + ], + [ + 'a recorded map without sources', + (fixture) => { + writeFileSync(join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`), '{"version":3}'); + rerecordMaps(fixture); + }, + 'MAP_DRIFT', + ], + [ + 'a tarball entry that is not the finalized JS', + (fixture) => (fixture.archive.files[0].sha256 = 'd'.repeat(64)), + 'TARBALL_DRIFT', + ], + [ + 'an output without its map in the record', + (fixture) => rerecordMaps(fixture, (record) => record.maps.pop()), + 'RECORD', + ], + [ + 'a map that pairs no output', + (fixture) => + rerecordMaps(fixture, (record) => + record.maps.push({ path: 'dist/unpaired.js.map', bytes: 2, sha256: digest('{}') }), + ), + 'RECORD', + ], + [ + 'a null map entry', + (fixture) => rerecordMaps(fixture, (record) => record.maps.push(null)), + 'RECORD', + ], + [ + 'an output outside dist', + (fixture) => { + const name = coreChunk(fixture); + rerecordMaps(fixture, (record) => { + record.outputs.find((output) => output.path === `dist/${name}`).path = `dist/../${name}`; + record.maps.find((map) => map.path === `dist/${name}.map`).path = `dist/../${name}.map`; + }); + }, + 'RECORD', + ], + [ + 'outputs that are not a list', + (fixture) => + writeFileSync( + join(fixture.staging, 'finalized.json'), + JSON.stringify({ outputs: {}, maps: [] }), + ), + 'RECORD', + ], + ['a missing record', (fixture) => rmSync(join(fixture.staging, 'finalized.json')), 'RECORD'], + [ + 'a build configuration without its target', + (fixture) => writeFileSync(join(fixture.cli, 'tsup.config.ts'), 'export default {};\n'), + 'TARGET', + ], + [ + 'a build configuration with two top-level targets', + (fixture) => + writeFileSync( + join(fixture.cli, 'tsup.config.ts'), + "export default {\n target: 'node20',\n target: 'node22',\n};\n", + ), + 'TARGET', + ], + [ + 'a target that is not a Node target (the seal validates what it writes)', + (fixture) => + writeFileSync( + join(fixture.cli, 'tsup.config.ts'), + "export default {\n target: 'es2022',\n};\n", + ), + 'SHAPE', + ], + [ + 'a missing manifest input', + (fixture) => rmSync(join(fixture.root, 'pnpm-lock.yaml')), + 'UNREADABLE_INPUT', + ], + [ + 'an uninstalled toolchain', + (fixture) => + rmSync(join(fixture.root, 'apps', 'cli', 'node_modules'), { recursive: true, force: true }), + 'TOOLCHAIN', + ], + ]) { + const fixture = await candidate(); + change(fixture); + await assert.rejects(seal(fixture), refuses(code), label); + assert.equal(existsSync(join(fixture.staging, 'manifest.json')), false, label); + } + const fixture = await candidate(); + for (const run of [ + async () => ({ stdout: 'not a tree\n', stderr: '' }), + async () => { + throw new Error('git failed'); + }, + ]) { + await assert.rejects( + sealPrivateManifest({ + candidate: fixture.root, + staging: fixture.staging, + checkout: fixture.root, + source: COMMIT, + dirty: false, + archive: fixture.archive, + run, + }), + refuses('SOURCE_TREE'), + ); + } +}); + +test('only the top-level target line of the build configuration counts', async () => { + const fixture = await candidate(); + writeFileSync( + join(fixture.cli, 'tsup.config.ts'), + "// target: 'node18',\nexport default {\n // target: 'node20',\n esbuildOptions: {\n target: 'node20',\n },\n target: 'node22',\n};\n", + ); + assert.equal((await seal(fixture)).target, 'node22'); +}); + +test('a final map naming a source outside the admitted roots refuses before it is read', async () => { + writeFileSync(join(scratch, 'outside.ts'), 'export {};\n'); + for (const name of ['apps/cli/src/../../../../outside.ts', 'node_modules/yaml/dist/index.ts']) { + const fixture = await candidate(); + const path = join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`); + const map = JSON.parse(readFileSync(path, 'utf8')); + map.sources.push(name); + writeFileSync(path, JSON.stringify(map)); + rerecordMaps(fixture); + await assert.rejects(seal(fixture), refuses('SOURCE_PATH'), name); + assert.equal(existsSync(join(fixture.staging, 'manifest.json')), false, name); + } +}); + +test('every manifest read is bounded', async () => { + const pad = (text, size) => text + ' '.repeat(Math.max(0, size - Buffer.byteLength(text))); + for (const [label, change, code] of [ + [ + 'build configuration', + (fixture) => { + const path = join(fixture.cli, 'tsup.config.ts'); + writeFileSync(path, pad(readFileSync(path, 'utf8'), 128 * 1024 + 1)); + }, + 'UNREADABLE_INPUT', + ], + [ + 'lockfile', + (fixture) => + writeFileSync(join(fixture.root, 'pnpm-lock.yaml'), pad('# lock\n', 16 * 1024 * 1024 + 1)), + 'UNREADABLE_INPUT', + ], + [ + 'source', + (fixture) => { + const path = join(fixture.root, 'packages', 'core', 'src', 'parser.ts'); + writeFileSync(path, pad(readFileSync(path, 'utf8'), 8 * 1024 * 1024 + 1)); + }, + 'UNREADABLE_INPUT', + ], + [ + 'map', + (fixture) => { + const path = join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`); + writeFileSync(path, pad(readFileSync(path, 'utf8'), 16 * 1024 * 1024 + 1)); + rerecordMaps(fixture); + }, + 'UNREADABLE_INPUT', + ], + [ + 'record', + (fixture) => { + const path = join(fixture.staging, 'finalized.json'); + writeFileSync(path, pad(readFileSync(path, 'utf8'), 1024 * 1024 + 1)); + }, + 'RECORD', + ], + ]) { + const fixture = await candidate(); + change(fixture); + await assert.rejects(seal(fixture), refuses(code), label); + } + const fixture = await candidate(); + await seal(fixture); + const path = join(fixture.staging, 'manifest.json'); + const text = readFileSync(path, 'utf8'); + rmSync(path); + writeFileSync(path, pad(text, 4 * 1024 * 1024 + 1)); + assert.throws(() => readPrivateManifest(fixture.staging), refuses('UNREADABLE')); + const empty = fresh('no-manifest'); + mkdirSync(empty); + assert.throws(() => readPrivateManifest(empty), refuses('UNREADABLE')); +}); + +/** A sealed fixture whose manifest records the digest of an actual tarball file. */ +async function sealedWithTarball() { + const fixture = await candidate(); + const artifact = fresh('artifact'); + mkdirSync(artifact); + const tarball = join(artifact, 'relavium-0.1.1.tgz'); + writeFileSync(tarball, 'synthetic tarball bytes'); + fixture.archive.sha256 = digest(readFileSync(tarball)); + return { fixture, tarball, manifest: await seal(fixture) }; +} + +test('the dist and the tarball are verified against the manifest just sealed', async () => { + const sealed = await sealedWithTarball(); + verifySealedDist({ + candidate: sealed.fixture.root, + manifest: sealed.manifest, + tarball: sealed.tarball, + }); + for (const [label, change, code] of [ + [ + 'one more byte in a chunk no anchor lies in', + (fixture) => { + const path = join(fixture.cli, 'dist', 'index.js'); + writeFileSync(path, `${readFileSync(path, 'utf8')} `); + }, + 'DIST_DRIFT', + ], + [ + 'an extra file', + (fixture) => writeFileSync(join(fixture.cli, 'dist', 'extra.js'), ''), + 'DIST_DRIFT', + ], + [ + 'a removed chunk', + (fixture) => rmSync(join(fixture.cli, 'dist', coreChunk(fixture))), + 'DIST_DRIFT', + ], + [ + 'a chunk replaced by a link to the same bytes', + (fixture) => { + const path = join(fixture.cli, 'dist', 'index.js'); + const copy = join(fixture.work, 'index.js'); + writeFileSync(copy, readFileSync(path)); + rmSync(path); + symlinkSync(copy, path); + }, + 'DIST_DRIFT', + ], + ['no dist', (fixture) => rmSync(join(fixture.cli, 'dist'), { recursive: true }), 'DIST_DRIFT'], + [ + 'a replaced tarball', + (_fixture, tarball) => writeFileSync(tarball, 'other bytes'), + 'TARBALL_CHANGED', + ], + ['a missing tarball', (_fixture, tarball) => rmSync(tarball), 'TARBALL_CHANGED'], + ]) { + const { fixture, tarball, manifest } = await sealedWithTarball(); + change(fixture, tarball); + assert.throws( + () => verifySealedDist({ candidate: fixture.root, manifest, tarball }), + refuses(code), + label, + ); + } +}); + +test('every output is checked, each read is bounded, and so is the tarball', async () => { + // A same-length change in any output refuses, the first as much as the last. + const sealed = await sealedWithTarball(); + assert.ok(sealed.manifest.outputs.length >= 2); + for (const output of sealed.manifest.outputs) { + const path = join(sealed.fixture.cli, ...output.path.split('/')); + const saved = readFileSync(path); + const changed = Buffer.from(saved); + changed[changed.length - 1] = changed[changed.length - 1] === 0x0a ? 0x20 : 0x0a; + writeFileSync(path, changed); + assert.throws( + () => + verifySealedDist({ + candidate: sealed.fixture.root, + manifest: sealed.manifest, + tarball: sealed.tarball, + }), + refuses('DIST_DRIFT'), + output.path, + ); + writeFileSync(path, saved); + } + // An output over 8 MiB, or a tarball over 16 MiB, refuses even when the manifest records its digest. + const big = Buffer.alloc(8 * 1024 * 1024 + 1, 0x20); + const output = sealed.manifest.outputs[0]; + writeFileSync(join(sealed.fixture.cli, ...output.path.split('/')), big); + assert.throws( + () => + verifySealedDist({ + candidate: sealed.fixture.root, + manifest: { + ...sealed.manifest, + outputs: sealed.manifest.outputs.map((entry) => + entry === output ? { ...entry, sha256: digest(big) } : entry, + ), + }, + tarball: sealed.tarball, + }), + refuses('DIST_DRIFT'), + ); + const fresh = await sealedWithTarball(); + const bigger = Buffer.alloc(16 * 1024 * 1024 + 1, 0x20); + writeFileSync(fresh.tarball, bigger); + assert.throws( + () => + verifySealedDist({ + candidate: fresh.fixture.root, + manifest: { ...fresh.manifest, tarballSha256: digest(bigger) }, + tarball: fresh.tarball, + }), + refuses('TARBALL_CHANGED'), + ); +}); + +test('a refusal is a ManifestError naming only its fixed code', () => { + const empty = fresh('no-manifest'); + mkdirSync(empty); + assert.throws(() => readPrivateManifest(empty), { + name: 'ManifestError', + code: 'UNREADABLE', + message: 'PACKED_CLI_MANIFEST_UNREADABLE', + }); +}); + +test('a sealed manifest that is not exactly the validated shape refuses on read', async () => { + const fixture = await candidate(); + const valid = await seal(fixture); + const hex = (c, n = 64) => c.repeat(n); + const many = (n, make) => Array.from({ length: n }, (_, i) => make(String(i).padStart(5, '0'))); + const output = (name) => ({ + path: `dist/${name}.js`, + sha256: hex('1'), + map: { path: `dist/${name}.js.map`, sha256: hex('2') }, + }); + const cases = [ + ['extra top-level key', (m) => (m.extra = 1)], + ['missing top-level key', (m) => delete m.target], + ['version 2', (m) => (m.version = 2)], + ['source extra key', (m) => (m.source.extra = 1)], + ['commit not hex', (m) => (m.source.commit = 'x')], + ['commit with suffix', (m) => (m.source.commit = `${hex('c', 40)}0`)], + ['commit short', (m) => (m.source.commit = hex('c', 39))], + ['tree not hex', (m) => (m.source.tree = 'x')], + ['dirty as string', (m) => (m.source.dirty = 'true')], + ['packageVersion as array', (m) => (m.packageVersion = ['0.1.1'])], + ['packageVersion not semver', (m) => (m.packageVersion = '0.1')], + ['packageVersion with trailing text', (m) => (m.packageVersion = '0.1.1 x')], + ['packageVersion over 100 characters', (m) => (m.packageVersion = `0.1.1-${'a'.repeat(95)}`)], + ['packageVersion with a plus in its build', (m) => (m.packageVersion = '0.1.1+a+b')], + ['packageVersion with an empty prerelease', (m) => (m.packageVersion = '0.1.1-')], + ['target as array', (m) => (m.target = ['node22'])], + ['target not nodeNN', (m) => (m.target = 'es2022')], + ['toolchain extra key', (m) => (m.toolchain.extra = '1.0.0')], + ['node version malformed', (m) => (m.toolchain.node = 'v22')], + ['pnpm other', (m) => (m.toolchain.pnpm = '9.12.4')], + ['typescript malformed', (m) => (m.toolchain.typescript = 'x')], + ['tsup malformed', (m) => (m.toolchain.tsup = 'x')], + ['esbuild malformed', (m) => (m.toolchain.esbuild = 'x')], + ['inputs missing one', (m) => delete m.inputs['.npmrc']], + ['inputs extra one', (m) => (m.inputs['extra.json'] = hex('3'))], + ['input digest not hex', (m) => (m.inputs['.npmrc'] = 'x')], + ['input digest uppercase', (m) => (m.inputs['.npmrc'] = hex('A'))], + ['input digest short', (m) => (m.inputs['.npmrc'] = hex('3', 63))], + ['input digest with prefix', (m) => (m.inputs['.npmrc'] = `z${hex('3')}`)], + ['tarball digest not hex', (m) => (m.tarballSha256 = 'x')], + ['tarball digest as array', (m) => (m.tarballSha256 = [hex('a')])], + ['outputs not an array', (m) => (m.outputs = {})], + ['outputs empty', (m) => (m.outputs = [])], + ['outputs over the bound', (m) => (m.outputs = many(257, (i) => output(`c${i}`)))], + ['output extra key', (m) => (m.outputs[0].extra = 1)], + // Paired map paths and the order stay valid, so only the path pattern can refuse these. + [ + 'output path escaping', + (m) => { + m.outputs[0].path = 'dist/../x.js'; + m.outputs[0].map.path = 'dist/../x.js.map'; + }, + ], + [ + 'output path not anchored', + (m) => { + m.outputs.at(-1).path = 'x/dist/zz.js'; + m.outputs.at(-1).map.path = 'x/dist/zz.js.map'; + }, + ], + ['output digest not hex', (m) => (m.outputs[0].sha256 = 'x')], + ['output map extra key', (m) => (m.outputs[0].map.extra = 1)], + ['output map path not paired', (m) => (m.outputs[0].map.path = 'dist/other.js.map')], + ['output map digest not hex', (m) => (m.outputs[0].map.sha256 = 'x')], + ['outputs unsorted', (m) => m.outputs.reverse()], + ['outputs duplicated', (m) => m.outputs.push(m.outputs.at(-1))], + ['sources an array', (m) => (m.sources = [])], + ['sources null', (m) => (m.sources = null)], + ['sources empty', (m) => (m.sources = {})], + [ + 'sources over the bound', + (m) => + (m.sources = Object.fromEntries(many(8193, (i) => [`apps/cli/src/s${i}.ts`, hex('4')]))), + ], + [ + 'source outside the admitted roots', + (m) => (m.sources['node_modules/yaml/index.ts'] = hex('4')), + ], + [ + 'source escaping by dot segments', + (m) => (m.sources['apps/cli/src/../../../x.ts'] = hex('4')), + ], + ['source not anchored', (m) => (m.sources['x/apps/cli/src/a.ts'] = hex('4'))], + ['source in another package', (m) => (m.sources['packages/other/src/a.ts'] = hex('4'))], + ['source outside src', (m) => (m.sources['apps/cli/a.ts'] = hex('4'))], + ['source not TypeScript', (m) => (m.sources['apps/cli/src/a.js'] = hex('4'))], + ['source digest not hex', (m) => (m.sources[Object.keys(m.sources)[0]] = 'x')], + [ + 'sources unsorted', + (m) => (m.sources = Object.fromEntries(Object.entries(m.sources).reverse())), + ], + ]; + for (const [label, change] of cases) { + const variant = JSON.parse(JSON.stringify(valid)); + change(variant); + // Keep sources sorted after an addition unless the case is about order. + if ( + label !== 'sources unsorted' && + variant.sources !== null && + !Array.isArray(variant.sources) + ) { + variant.sources = Object.fromEntries( + Object.entries(variant.sources).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)), + ); + } + const staging = fresh('shape'); + mkdirSync(staging); + writeFileSync(join(staging, 'manifest.json'), JSON.stringify(variant)); + assert.throws(() => readPrivateManifest(staging), refuses('SHAPE'), label); + } +}); diff --git a/tools/cli-package/map-controls-core.mjs b/tools/cli-package/map-controls-core.mjs new file mode 100644 index 00000000..84606d0f --- /dev/null +++ b/tools/cli-package/map-controls-core.mjs @@ -0,0 +1,853 @@ +/** + * Product map controls (ADR-0111; packaging.md "Product map controls and the package-test receipt — + * Step 9a.2b"; the 2026-10-24 preflight rulings), run by map-controls.mjs. One real package-test + * production runs at the checked-out commit. Inside its seal, after the baseline receipt and while the + * private staging still exists, every named negative control corrupts one product copy — the + * producer's own owned candidate tree, or a re-recorded copy of its private staging — and requires that + * control's exact refusal; each corrupted file is restored and checked by digest (dist itself is + * replaced by the next build), and a restoration + * production must then seal again. Staging copies stay under the staging, which the producer removes + * first on success, failure and interrupt; a refused control build's `dist`, which holds the bundler's + * maps, is removed at once. The developer's checkout is read, never written (Git may refresh its own + * index's stat cache). The summary carries control ids, expected and observed codes and prepack's fixed + * lines with their repository-relative subjects — never an absolute path, frame, position, map byte or + * raw child output. + */ +import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; +import { + chmodSync, + cpSync, + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readdirSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join } from 'node:path'; +import { originFromEnvironment } from './binding.mjs'; +import { sourceIdentity } from './candidate.mjs'; +import { CANDIDATE_STAGING_ENV } from './finalize.mjs'; +import { readOrdinaryFile, sha256, byCodeUnit } from './files.mjs'; +import { readPrivateManifest, sealPrivateManifest, verifySealedDist } from './manifest.mjs'; +import { emitFreshPackage } from './pipeline.mjs'; +import { producePackage } from './produce.mjs'; +import { productionFailure } from './production-failure.mjs'; +import { sealPackage } from './seal.mjs'; +import { + assertGeneratedRange, + assertOriginalRange, + decodeMappings, + lineLengths, +} from './source-map.mjs'; +import { ANCHORS, symbolicateCandidate } from './symbolicate.mjs'; + +const BASE64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; +const MIB = 1024 * 1024; + +/** Base64 VLQ of one signed value, as a source map writer encodes it. */ +function vlq(value) { + let rest = value < 0 ? -value * 2 + 1 : value * 2; + let out = ''; + do { + let digit = rest % 32; + rest = Math.floor(rest / 32); + if (rest > 0) digit += 32; + out += BASE64[digit]; + } while (rest > 0); + return out; +} + +/** + * Encode decoded segments (as `decodeMappings` returns them) back into `mappings`. + * @param {import('./source-map.mjs').Segment[][]} lines + */ +export function encodeMappings(lines) { + let source = 0; + let originalLine = 0; + let originalColumn = 0; + let name = 0; + return lines + .map((segments) => { + let column = 0; + return segments + .map((segment) => { + let out = vlq(segment.generatedColumn - column); + column = segment.generatedColumn; + if (segment.source !== undefined) { + out += + vlq(segment.source - source) + + vlq(segment.originalLine - originalLine) + + vlq(segment.originalColumn - originalColumn); + source = segment.source; + originalLine = segment.originalLine; + originalColumn = segment.originalColumn; + if (segment.name !== undefined) { + out += vlq(segment.name - name); + name = segment.name; + } + } + return out; + }) + .join(','); + }) + .join(';'); +} + +/** + * A map shifted while staying valid: every original column after a line's start moves one to the left, + * so every position stays inside its text and only symbolication can tell. + * @param {string} mappings @param {{sources: number, names: number}} counts + */ +export function shiftOriginalColumns(mappings, counts) { + const lines = decodeMappings(mappings, counts); + let moved = 0; + for (const segments of lines) { + for (const segment of segments) { + if (segment.originalColumn !== undefined && segment.originalColumn > 0) { + segment.originalColumn -= 1; + moved += 1; + } + } + } + assert.ok(moved > 0, 'PACKED_CLI_MAP_CONTROL_NOTHING_SHIFTED'); + return encodeMappings(lines); +} + +/** Sources rotated by one: with two or more distinct sources, no source keeps its index. */ +export function rotateSources(sources) { + assert.ok( + sources.length >= 2 && new Set(sources).size === sources.length, + 'PACKED_CLI_MAP_CONTROL_SOURCES', + ); + return [...sources.slice(1), sources[0]]; +} + +/** The JS one line down: a line break after the hashbang line, or at the start. */ +export function shiftLines(text) { + if (!text.startsWith('#!')) return `\n${text}`; + const end = text.indexOf('\n'); + assert.ok(end > 0, 'PACKED_CLI_MAP_CONTROL_HASHBANG'); + return `${text.slice(0, end + 1)}\n${text.slice(end + 1)}`; +} + +/** The JS one column right: a space before every line but a hashbang line. */ +export function shiftColumns(text) { + return text + .split('\n') + .map((line, index) => (index === 0 && line.startsWith('#!') ? line : ` ${line}`)) + .join('\n'); +} + +/** `mappings` cut right after the first continuation digit, so the value it starts never ends. */ +export function cutInsideValue(mappings) { + for (let index = 0; index < mappings.length; index += 1) { + if (BASE64.indexOf(mappings[index]) >= 32) return mappings.slice(0, index + 1); + } + assert.fail('PACKED_CLI_MAP_CONTROL_NO_CONTINUATION'); +} + +/** An owner-only directory, created fresh. */ +function privateDirectory(path) { + mkdirSync(path, { recursive: true, mode: 0o700 }); + chmodSync(path, 0o700); + return path; +} + +/** The refusal of `action`: its code, its prepack diagnostics and how many times stdout says each marker. */ +async function refusal(action) { + try { + await action(); + } catch (error) { + const lines = productionFailure(error); + const stdout = typeof error?.result?.stdout === 'string' ? error.result.stdout : ''; + return { + code: /: ([A-Z0-9_]+)$/u.exec(lines.at(-1))?.[1] ?? 'UNEXPECTED', + diagnostics: lines.slice(0, -1), + builds: (stdout.match(/\bESM Build start\b/gu) ?? []).length, + finalized: (stdout.match(/^✓ CLI candidate finalized: /gmu) ?? []).length, + }; + } + return { code: 'ACCEPTED', diagnostics: [], builds: 0, finalized: 0 }; +} + +/** A file's bytes and mode, to restore it exactly. */ +function snapshot(path) { + const bytes = readOrdinaryFile(path, 16 * MIB); + return { path, bytes, sha256: sha256(bytes), mode: lstatSync(path).mode & 0o777 }; +} + +/** Put a snapshot back and prove it is byte for byte the original. */ +function restore(saved) { + rmSync(saved.path, { force: true }); + writeFileSync(saved.path, saved.bytes, { mode: saved.mode, flag: 'wx' }); + chmodSync(saved.path, saved.mode); + assert.equal( + sha256(readOrdinaryFile(saved.path, 16 * MIB)), + saved.sha256, + 'PACKED_CLI_MAP_CONTROL_RESTORE', + ); +} + +/** SHA-256 of every file in a directory, by name; a directory listed as such. */ +function inventory(directory, skip = new Set()) { + return Object.fromEntries( + readdirSync(directory, { withFileTypes: true, recursive: true }) + .map((entry) => join(entry.parentPath ?? entry.path, entry.name)) + .filter((path) => ![...skip].some((name) => path.startsWith(join(directory, name)))) + .sort() + .map((path) => [ + path.slice(directory.length), + lstatSync(path).isFile() ? sha256(readOrdinaryFile(path, 32 * MIB)) : 'directory', + ]), + ); +} + +/** + * Run every control inside the baseline seal, recording into `state` as it goes, so a run that stops + * early still reports what ran. `context` holds the seal's own inputs, its receipt, the baseline + * artifact directory and the emitter's isolated environment and pnpm. + * @param {{results: {id: string, expected: string, observed: string}[], chunks: number}} state + */ +async function runControls(context, state) { + const { candidate, staging, checkout, source, dirty, receipt, artifact, env, pnpm } = context; + const record = (id, expected, observed) => state.results.push({ id, expected, observed }); + const controls = privateDirectory(join(staging, 'controls')); + let count = 0; + const scratch = (id) => { + count += 1; + return privateDirectory(join(controls, `${String(count).padStart(2, '0')}-${id}`)); + }; + const cli = join(candidate, 'apps', 'cli'); + const tarball = join(artifact, receipt.tarball); + const sealed = readPrivateManifest(staging); + // The dist's digests, or null once a broken refusal let a build replace or remove it. + const distDigests = () => { + try { + return inventory(join(cli, 'dist')); + } catch { + return null; + } + }; + + // The chunks the anchors' sources lie in, by their staged maps; the anchor sources' upstream map. + const anchorSources = new Set(ANCHORS.map((anchor) => anchor.source)); + const mapsOf = (name) => + JSON.parse(readOrdinaryFile(join(staging, 'maps', name), 16 * MIB).toString('utf8')); + const chunks = readdirSync(join(staging, 'maps')) + .filter((name) => mapsOf(name).sources.some((path) => anchorSources.has(path))) + .map((name) => name.slice(0, -'.map'.length)) + .sort(); + assert.ok(chunks.length > 0, 'PACKED_CLI_MAP_CONTROL_NO_ANCHOR_CHUNK'); + state.chunks = chunks.length; + const plain = sealed.outputs + .map((output) => basename(output.path)) + .filter((name) => !chunks.includes(name)) + .sort(); + + const sealedRecord = JSON.parse( + readOrdinaryFile(join(staging, 'finalized.json'), MIB).toString('utf8'), + ); + const sealedDigests = new Map( + [...sealedRecord.outputs, ...sealedRecord.maps].map((entry) => [entry.path, entry.sha256]), + ); + + /** + * A copy of the sealed staging re-recorded for the candidate as it now is, then symbolicated. The + * copy must differ from the sealed record in exactly the `changed` outputs and maps, so a control + * corrupts the chunk it names and nothing else. + */ + const resealed = async (id, { forceDirty = dirty, changeMap, changed = [] } = {}) => { + const root = scratch(id); + const copy = privateDirectory(join(root, 'staging')); + privateDirectory(join(copy, 'maps')); + const finalized = JSON.parse( + readOrdinaryFile(join(staging, 'finalized.json'), MIB).toString('utf8'), + ); + finalized.maps = finalized.maps.map((entry) => { + const name = basename(entry.path); + let bytes = readOrdinaryFile(join(staging, 'maps', name), 16 * MIB); + if (changeMap !== undefined && name === `${changeMap.chunk}.map`) { + bytes = Buffer.from(JSON.stringify(changeMap.change(JSON.parse(bytes.toString('utf8'))))); + } + writeFileSync(join(copy, 'maps', name), bytes, { mode: 0o600, flag: 'wx' }); + return { path: entry.path, bytes: bytes.length, sha256: sha256(bytes) }; + }); + finalized.outputs = finalized.outputs.map((entry) => { + const bytes = readOrdinaryFile(join(cli, ...entry.path.split('/')), 8 * MIB); + return { path: entry.path, bytes: bytes.length, sha256: sha256(bytes) }; + }); + assert.deepEqual( + [...finalized.outputs, ...finalized.maps] + .filter((entry) => sealedDigests.get(entry.path) !== entry.sha256) + .map((entry) => entry.path) + .sort(byCodeUnit), + [...changed].sort(byCodeUnit), + 'PACKED_CLI_MAP_CONTROL_TARGET', + ); + writeFileSync(join(copy, 'finalized.json'), JSON.stringify(finalized), { + mode: 0o600, + flag: 'wx', + }); + const digests = new Map( + finalized.outputs.map((output) => [`package/${output.path}`, output.sha256]), + ); + const archive = { + sha256: receipt.tarballSha256, + package: receipt.package, + files: receipt.files.map((file) => + digests.has(file.path) ? { ...file, sha256: digests.get(file.path) } : file, + ), + }; + await sealPrivateManifest({ + candidate, + staging: copy, + checkout, + source, + dirty: forceDirty, + archive, + }); + const work = privateDirectory(join(root, 'work')); + return refusal(() => + symbolicateCandidate({ candidate, staging: copy, checkout, work, archive }), + ); + }; + + /** One pack of the candidate as it now is, into a fresh staging and artifact directory. */ + const pack = async (id, stagingOverride) => { + const root = scratch(id); + const destination = privateDirectory(join(root, 'artifact')); + const fresh = stagingOverride ?? privateDirectory(join(root, 'staging')); + const observed = await refusal(() => + pnpm(['pack', '--pack-destination', destination], { + cwd: cli, + env: { ...env, [CANDIDATE_STAGING_ENV]: fresh }, + }), + ); + // A refused build left the bundler's own maps in dist: none outlives the control. A refusal before + // any build (a second prepack into a sealed staging) leaves the sealed dist as it is. + if (observed.code !== 'ACCEPTED' && observed.builds > 0) { + rmSync(join(cli, 'dist'), { recursive: true, force: true }); + } + return { observed, destination, staging: fresh, root }; + }; + + /** Seal a fresh pack's tarball with fresh work, as the producer would. */ + const seal = (packed) => + refusal(() => + sealPackage(packed.destination, { + repository: candidate, + staging: packed.staging, + source, + dirty, + checkout, + work: privateDirectory(join(packed.root, 'work')), + }), + ); + + // Post-receipt rebuild refusals that need the sealed state come first. + { + // A second prepack into the sealed staging refuses before any build, changing nothing. + const before = { dist: distDigests(), staging: inventory(staging, new Set(['controls'])) }; + const second = await pack('second-prepack', staging); + const unchanged = + JSON.stringify(distDigests()) === JSON.stringify(before.dist) && + JSON.stringify(inventory(staging, new Set(['controls']))) === + JSON.stringify(before.staging) && + readdirSync(second.destination).length === 0 && + second.observed.builds === 0; + record( + 'second-prepack', + 'STAGING_NOT_EMPTY', + unchanged && + second.observed.diagnostics.length === 1 && + second.observed.diagnostics[0] === '✗ CLI candidate finalization refused: STAGING_NOT_EMPTY' + ? 'STAGING_NOT_EMPTY' + : `MARKERS_${second.observed.code}`, + ); + } + { + // A re-seal into the artifact directory, or of the sealed staging beside a copied tarball, refuses + // before any capture runs. + const receiptDigest = sha256( + readOrdinaryFile(join(artifact, 'package-test-receipt.json'), MIB), + ); + const root = scratch('re-seal'); + const works = [privateDirectory(join(root, 'work-1')), privateDirectory(join(root, 'work-2'))]; + // The receipt half alone: a copy of the sealed staging without its manifest. + const unsealed = privateDirectory(join(root, 'unsealed')); + cpSync(join(staging, 'maps'), join(unsealed, 'maps'), { recursive: true }); + for (const name of ['finalized.json', 'metafile-esm.json']) { + cpSync(join(staging, name), join(unsealed, name)); + } + const again = await refusal(() => + sealPackage(artifact, { + repository: candidate, + staging: unsealed, + source, + dirty, + checkout, + work: works[0], + }), + ); + // The staging half alone: the tarball copied without its receipt, beside the sealed staging. + const copied = privateDirectory(join(root, 'artifact')); + cpSync(tarball, join(copied, receipt.tarball)); + const beside = await refusal(() => + sealPackage(copied, { + repository: candidate, + staging, + source, + dirty, + checkout, + work: works[1], + }), + ); + const quiet = + works.every((work) => readdirSync(work).length === 0) && + !existsSync(join(unsealed, 'manifest.json')) && + !existsSync(join(copied, 'package-test-receipt.json')) && + sha256(readOrdinaryFile(join(artifact, 'package-test-receipt.json'), MIB)) === receiptDigest; + record( + 're-seal-receipt', + 'PACKED_CLI_ALREADY_SEALED', + quiet ? again.code : `MARKERS_${again.code}`, + ); + record( + 're-seal-staging', + 'PACKED_CLI_ALREADY_SEALED', + quiet ? beside.code : `MARKERS_${beside.code}`, + ); + } + + // The re-recording technique alone refuses nothing: the unchanged copy reaches every anchor. + record('re-recorded-baseline', 'ACCEPTED', (await resealed('re-recorded-baseline')).code); + + // Wrong JS by line and by column, a wrong map and permuted sources, on every anchor-bearing chunk. + for (const [index, chunk] of chunks.entries()) { + const path = join(cli, 'dist', chunk); + for (const [kind, change] of [ + ['js-line', shiftLines], + ['js-column', shiftColumns], + ]) { + const saved = snapshot(path); + try { + const text = saved.bytes.toString('utf8'); + const changedText = change(text); + // One more line for a line shift, the same lines for a column shift. + assert.ok( + changedText !== text && + changedText.split('\n').length - text.split('\n').length === + (kind === 'js-line' ? 1 : 0), + 'PACKED_CLI_MAP_CONTROL_KIND', + ); + writeFileSync(path, changedText); + record( + `${kind}-${index + 1}`, + 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', + (await resealed(kind, { changed: [`dist/${chunk}`] })).code, + ); + } finally { + restore(saved); + } + } + record( + `map-line-${index + 1}`, + 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', + ( + await resealed('map-line', { + changeMap: { chunk, change: (map) => ({ ...map, mappings: `;${map.mappings}` }) }, + changed: [`dist/${chunk}.map`], + }) + ).code, + ); + record( + `sources-rotated-${index + 1}`, + 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', + ( + await resealed('sources-rotated', { + changeMap: { chunk, change: (map) => ({ ...map, sources: rotateSources(map.sources) }) }, + changed: [`dist/${chunk}.map`], + }) + ).code, + ); + } + + // A wrong source revision: clean, the Git object refuses the copy; dirty, the anchor positions move. + { + const saved = snapshot(join(candidate, ...ANCHORS[1].source.split('/'))); + try { + writeFileSync(saved.path, `// shifted\n${saved.bytes.toString('utf8')}`); + record( + 'source-clean', + 'PACKED_CLI_SYMBOLICATION_SOURCE_REVISION', + (await resealed('source-clean', { forceDirty: false })).code, + ); + record( + 'source-dirty', + 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', + (await resealed('source-dirty', { forceDirty: true })).code, + ); + } finally { + restore(saved); + } + } + + // A rebuilt dist that differs from the manifest: the sealed state verifies; a changed chunk whose map + // names no anchor source, and a real rebuild from a changed source, refuse. + record( + 'sealed-dist', + 'ACCEPTED', + (await refusal(() => verifySealedDist({ candidate, manifest: sealed, tarball }))).code, + ); + if (plain.length === 0) + record('dist-plain-chunk', 'PACKED_CLI_MANIFEST_DIST_DRIFT', 'NO_PLAIN_CHUNK'); + else { + const saved = snapshot(join(cli, 'dist', plain[0])); + try { + writeFileSync(saved.path, Buffer.concat([saved.bytes, Buffer.from(' ')])); + record( + 'dist-plain-chunk', + 'PACKED_CLI_MANIFEST_DIST_DRIFT', + (await refusal(() => verifySealedDist({ candidate, manifest: sealed, tarball }))).code, + ); + } finally { + restore(saved); + } + } + const entry = snapshot(join(cli, 'src', 'index.ts')); + try { + writeFileSync( + entry.path, + `${entry.bytes.toString('utf8')}\nglobalThis.relaviumMapControls = 1;\n`, + ); + const rebuilt = await pack('rebuild'); + const changed = sealed.outputs.some((output) => { + try { + return ( + sha256(readOrdinaryFile(join(cli, ...output.path.split('/')), 8 * MIB)) !== output.sha256 + ); + } catch { + return true; + } + }); + const verified = await refusal(() => + verifySealedDist({ candidate, manifest: sealed, tarball }), + ); + record( + 'dist-rebuilt', + 'PACKED_CLI_MANIFEST_DIST_DRIFT', + rebuilt.observed.code === 'ACCEPTED' && changed + ? verified.code + : `MARKERS_${rebuilt.observed.code}`, + ); + } finally { + restore(entry); + } + + // Upstream maps: a missing, a malformed and a value-truncated map refuse in the finalizer after the + // bundler silently succeeded; a byte-truncated map fails the build; a shifted but valid map is + // accepted there and refused by symbolication. + const upstream = ANCHORS[0].source + .replace(/^packages\/core\/src\//u, 'packages/core/dist/') + .replace(/\.ts$/u, '.js'); + const upstreamMap = join(candidate, ...`${upstream}.map`.split('/')); + const saved = snapshot(upstreamMap); + const original = JSON.parse(saved.bytes.toString('utf8')); + const refusedLine = (code, subject) => + `✗ CLI candidate finalization refused: ${code} (${subject})`; + for (const [id, corrupt, expected] of [ + ['upstream-missing', () => rmSync(upstreamMap), refusedLine('UPSTREAM_MAP_MISSING', upstream)], + [ + 'upstream-shape', + () => writeFileSync(upstreamMap, JSON.stringify({ ...original, sourcesContent: [null] })), + refusedLine('UPSTREAM_MAP_SHAPE', `${upstream}.map`), + ], + [ + 'upstream-index', + () => + writeFileSync( + upstreamMap, + JSON.stringify({ ...original, mappings: `A${vlq(original.sources.length)}AA` }), + ), + refusedLine('UPSTREAM_MAP_MAPPINGS', `${upstream}.map`), + ], + [ + // A map that is no longer JSON fails the bundler's own build, before the finalizer reads it. + 'upstream-cut-bytes', + () => writeFileSync(upstreamMap, saved.bytes.subarray(0, Math.floor(saved.bytes.length / 2))), + '✗ CLI build failed', + ], + [ + 'upstream-cut-value', + () => + writeFileSync( + upstreamMap, + JSON.stringify({ ...original, mappings: cutInsideValue(original.mappings) }), + ), + refusedLine('UPSTREAM_MAP_MAPPINGS', `${upstream}.map`), + ], + ]) { + try { + corrupt(); + const packed = await pack(id); + // The bundler silently succeeded once, nothing was finalized, staged or packed, and the one + // prepack diagnostic is the expected fixed line. + const markers = [ + packed.observed.builds, + packed.observed.finalized, + readdirSync(packed.staging).length, + readdirSync(packed.destination).length, + packed.observed.diagnostics.length, + ]; + const line = packed.observed.diagnostics.at(-1); + record( + id, + expected, + markers.join() === '1,0,0,0,1' && line === expected + ? expected + : `MARKERS(${markers.join()}) ${line ?? packed.observed.code}`, + ); + } finally { + restore(saved); + } + } + try { + const mappings = shiftOriginalColumns(original.mappings, { + sources: original.sources.length, + names: original.names.length, + }); + // The shift is strictly valid: the finalizer's own checks admit it. + const lines = decodeMappings(mappings, { + sources: original.sources.length, + names: original.names.length, + }); + assertGeneratedRange( + lines, + lineLengths( + readOrdinaryFile(join(candidate, ...upstream.split('/')), 16 * MIB).toString('utf8'), + ), + ); + assertOriginalRange( + lines, + original.sources.map((path) => + lineLengths( + readOrdinaryFile(join(dirname(upstreamMap), ...path.split('/')), 16 * MIB).toString( + 'utf8', + ), + ), + ), + ); + writeFileSync(upstreamMap, JSON.stringify({ ...original, mappings })); + const packed = await pack('upstream-shifted'); + const accepted = packed.observed.code === 'ACCEPTED'; + record( + 'upstream-shifted', + 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', + accepted ? (await seal(packed)).code : `MARKERS_${packed.observed.code}`, + ); + } finally { + restore(saved); + } + + // The seal's own check: a dist the run changed after the inspection refuses before the receipt. + { + const packed = await pack('seal-verify'); + let observed = `MARKERS_${packed.observed.code}`; + if (packed.observed.code === 'ACCEPTED' && plain.length > 0) { + const refused = await refusal(() => + sealPackage(packed.destination, { + repository: candidate, + staging: packed.staging, + source, + dirty, + checkout, + work: privateDirectory(join(packed.root, 'work')), + ports: { + symbolicate: async (options) => { + await symbolicateCandidate(options); + const path = join(cli, 'dist', plain[0]); + writeFileSync( + path, + Buffer.concat([readOrdinaryFile(path, 8 * MIB), Buffer.from(' ')]), + ); + }, + }, + }), + ); + observed = existsSync(join(packed.destination, 'package-test-receipt.json')) + ? 'MARKERS_RECEIPT_WRITTEN' + : refused.code; + } + record('seal-verify', 'PACKED_CLI_MANIFEST_DIST_DRIFT', observed); + } + + // Restoration: every corrupted file is back, so a fresh pack seals again. + const restored = await pack('restoration'); + record( + 'restoration', + 'ACCEPTED', + restored.observed.code === 'ACCEPTED' + ? (await seal(restored)).code + : `MARKERS_${restored.observed.code}`, + ); +} + +/** + * Run the controls through one real production of `repository` and return their results with the + * source identity. The developer checkout must be the same before and after. + * @param {{repository: string, environment: Record, + * temporary?: string}} options + */ +export async function runMapControls({ repository, environment, temporary = tmpdir() }) { + // A half-configured CI environment refuses before anything is built. + try { + originFromEnvironment(environment); + } catch { + assert.fail('PACKED_CLI_MAP_CONTROL_ORIGIN'); + } + const before = await sourceIdentity(repository); + const root = realpathSync.native(mkdtempSync(join(temporary, 'relavium-map-controls-'))); + chmodSync(root, 0o700); + const ran = { results: [], chunks: 0 }; + let emitted; + try { + const produced = await producePackage({ + repository, + destination: join(root, 'artifact'), + environment, + temporary, + ports: { + emit: (options) => { + emitted = { env: options.env, pnpm: options.pnpm }; + return emitFreshPackage(options); + }, + seal: async (artifact, inputs) => { + const receipt = await sealPackage(artifact, inputs); + try { + await runControls( + { + candidate: inputs.repository, + staging: inputs.staging, + checkout: inputs.checkout, + source: inputs.source, + dirty: inputs.dirty, + receipt, + artifact, + ...emitted, + }, + ran, + ); + } catch (error) { + // A harness self-check or an unexpected error stops the controls; the summary still names + // it, with only its fixed code, and every control that did not run is reported missing. + const line = productionFailure(error).at(-1); + ran.results.push({ + id: 'aborted', + expected: 'none', + observed: line.slice(line.lastIndexOf(': ') + 2), + }); + } + return receipt; + }, + }, + }); + const after = await sourceIdentity(repository); + assert.deepEqual( + [after.source, after.status, after.unchecked], + [before.source, before.status, before.unchecked], + 'PACKED_CLI_MAP_CONTROL_CHECKOUT_CHANGED', + ); + return { source: produced.receipt.source, dirty: produced.identity.dirty, ...ran }; + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +/** The controls every run records once, besides four for each anchor-bearing chunk. */ +export const FIXED_CONTROLS = Object.freeze([ + 'second-prepack', + 're-seal-receipt', + 're-seal-staging', + 're-recorded-baseline', + 'source-clean', + 'source-dirty', + 'sealed-dist', + 'dist-plain-chunk', + 'dist-rebuilt', + 'upstream-missing', + 'upstream-shape', + 'upstream-index', + 'upstream-cut-bytes', + 'upstream-cut-value', + 'upstream-shifted', + 'seal-verify', + 'restoration', +]); +const PER_CHUNK = Object.freeze(['js-line', 'js-column', 'map-line', 'sources-rotated']); + +/** + * The verdict on a run: at least one anchor-bearing chunk, every expected control recorded exactly + * once, and each with its expected outcome. + * @param {{results: {id: string, expected: string, observed: string}[], chunks: number}} outcome + */ +export function verdict({ results, chunks }) { + const expected = [ + ...FIXED_CONTROLS, + ...Array.from({ length: chunks }, (_, index) => + PER_CHUNK.map((kind) => `${kind}-${index + 1}`), + ).flat(), + ]; + const ids = results.map((result) => result.id); + const missing = expected.filter((id) => !ids.includes(id)); + const unexpected = ids.filter((id, index) => !expected.includes(id) || ids.indexOf(id) !== index); + const failed = results + .filter((result) => result.observed !== result.expected) + .map((result) => result.id); + return { + expected: expected.length, + missing, + unexpected, + failed, + passed: chunks >= 1 && missing.length === 0 && unexpected.length === 0 && failed.length === 0, + }; +} + +/** + * Print a run's summary and return the process exit code: 0 only when the verdict passes. + * @param {{source: string, dirty: boolean, results: {id: string, expected: string, + * observed: string}[], chunks: number}} outcome + * @param {{seconds: number, write?: (line: string) => void, warn?: (line: string) => void}} options + */ +export function report(outcome, { seconds, write = console.log, warn = console.error }) { + const judged = verdict(outcome); + write( + JSON.stringify({ + scope: 'product copies of one real package-test production', + source: outcome.source, + dirty: outcome.dirty, + chunks: outcome.chunks, + controls: outcome.results.length, + expected: judged.expected, + failed: judged.failed, + missing: judged.missing, + unexpected: judged.unexpected, + results: outcome.results, + seconds, + }), + ); + if (judged.passed) return 0; + const named = [ + ...judged.failed, + ...judged.missing.map((id) => `${id} (missing)`), + ...judged.unexpected.map((id) => `${id} (unexpected)`), + ]; + warn( + `✗ CLI product map controls failed: ${named.length > 0 ? named.join(', ') : 'no anchor-bearing chunk'}`, + ); + return 1; +} diff --git a/tools/cli-package/map-controls.mjs b/tools/cli-package/map-controls.mjs new file mode 100644 index 00000000..7c4a112c --- /dev/null +++ b/tools/cli-package/map-controls.mjs @@ -0,0 +1,22 @@ +/** + * Command-line entry point of the product map controls (`map-controls-core.mjs`; `pnpm + * check:map-controls`): one real production at this checkout, every named control, one JSON summary. + * It runs unconditionally, so it can never exit 0 without running the controls, and exits 1 unless + * every expected control ran once with its expected outcome. A failure prints only fixed lines. + */ +import { fileURLToPath } from 'node:url'; +import { report, runMapControls } from './map-controls-core.mjs'; +import { productionFailure } from './production-failure.mjs'; + +const started = Date.now(); +let outcome; +try { + outcome = await runMapControls({ + repository: fileURLToPath(new URL('../../', import.meta.url)), + environment: process.env, + }); +} catch (error) { + for (const line of productionFailure(error)) console.error(line); + process.exit(1); +} +process.exitCode = report(outcome, { seconds: Math.round((Date.now() - started) / 1000) }); diff --git a/tools/cli-package/map-controls.test.mjs b/tools/cli-package/map-controls.test.mjs new file mode 100644 index 00000000..66e0912f --- /dev/null +++ b/tools/cli-package/map-controls.test.mjs @@ -0,0 +1,289 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { copyFileSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { + FIXED_CONTROLS, + cutInsideValue, + encodeMappings, + report, + rotateSources, + shiftColumns, + shiftLines, + shiftOriginalColumns, + verdict, +} from './map-controls-core.mjs'; +import { SourceMapError, decodeMappings } from './source-map.mjs'; + +// Importing the runner runs no control: only its command-line entry point does. +const COUNTS = { sources: 3, names: 2 }; +const refuses = (code) => (error) => error instanceof Error && error.message === code; + +/** Deterministic mappings with empty lines, unmapped and named segments and backward moves. */ +function sample() { + let seed = 11; + const next = (bound) => { + seed = (seed * 1103515245 + 12345) % 2 ** 31; + // The high bits: an LCG's low bits cycle and would never draw some kinds. + return Math.floor(seed / 65536) % bound; + }; + const lines = []; + for (let line = 0; line < 30; line += 1) { + const segments = []; + let column = next(3); + for (let index = next(12); index > 0; index -= 1) { + column += 1 + next(400); + const kind = next(4); + if (kind === 0) segments.push({ generatedColumn: column }); + else { + const segment = { + generatedColumn: column, + source: next(3), + originalLine: next(200), + originalColumn: kind === 1 ? 0 : next(120), + }; + if (kind === 3) segment.name = next(2); + segments.push(segment); + } + } + lines.push(segments); + } + // Every kind of segment the encoder must write is present. + const all = lines.flat(); + assert.ok(all.some((segment) => segment.source === undefined)); + assert.ok(all.some((segment) => segment.name !== undefined)); + assert.ok(all.some((segment) => segment.originalColumn > 0)); + assert.ok(all.some((segment) => segment.originalColumn === 0)); + assert.ok(lines.some((segments) => segments.length === 0)); + return lines; +} + +test('the encoder writes back exactly what the strict decoder reads', () => { + const lines = sample(); + const mappings = encodeMappings(lines); + assert.deepEqual(decodeMappings(mappings, COUNTS), lines); + // Canonical input re-encodes to the same text, empty lines included. + assert.equal(encodeMappings(decodeMappings(mappings, COUNTS)), mappings); + assert.equal(encodeMappings(decodeMappings(';;AAAA;', COUNTS)), ';;AAAA;'); +}); + +test('a shifted map moves every original column after a line start one left, and nothing else', () => { + const lines = sample(); + const shifted = decodeMappings(shiftOriginalColumns(encodeMappings(lines), COUNTS), COUNTS); + assert.equal(shifted.length, lines.length); + lines.forEach((segments, line) => + segments.forEach((segment, index) => { + const moved = shifted[line][index]; + assert.equal(moved.generatedColumn, segment.generatedColumn); + assert.equal(moved.source, segment.source); + assert.equal(moved.originalLine, segment.originalLine); + assert.equal(moved.name, segment.name); + assert.equal( + moved.originalColumn, + segment.originalColumn === undefined || segment.originalColumn === 0 + ? segment.originalColumn + : segment.originalColumn - 1, + ); + }), + ); + // A map with nothing to shift refuses rather than leaving the control hollow. + assert.throws( + () => shiftOriginalColumns('AAAA;CAAA', COUNTS), + refuses('PACKED_CLI_MAP_CONTROL_NOTHING_SHIFTED'), + ); +}); + +test('rotated sources leave no source at its index', () => { + for (const sources of [ + ['a.ts', 'b.ts'], + ['a.ts', 'b.ts', 'c.ts'], + ['a.ts', 'b.ts', 'c.ts', 'd.ts', 'e.ts'], + ]) { + const rotated = rotateSources(sources); + assert.deepEqual([...rotated].sort(), [...sources].sort()); + assert.ok(rotated.every((source, index) => source !== sources[index])); + } + for (const sources of [[], ['a.ts'], ['a.ts', 'a.ts']]) { + assert.throws(() => rotateSources(sources), refuses('PACKED_CLI_MAP_CONTROL_SOURCES')); + } +}); + +test('the JS moves one line or one column, keeping a hashbang on the first line', () => { + assert.equal(shiftLines('a();\nb();\n'), '\na();\nb();\n'); + assert.equal(shiftLines('#!/usr/bin/env node\na();\n'), '#!/usr/bin/env node\n\na();\n'); + assert.throws( + () => shiftLines('#!/usr/bin/env node'), + refuses('PACKED_CLI_MAP_CONTROL_HASHBANG'), + ); + assert.equal(shiftColumns('a();\nb();\n'), ' a();\n b();\n '); + assert.equal(shiftColumns('#!/usr/bin/env node\na();'), '#!/usr/bin/env node\n a();'); +}); + +test('a value cut after its first continuation digit never ends', () => { + const mappings = encodeMappings(sample()); + const cut = cutInsideValue(mappings); + assert.ok(mappings.startsWith(cut)); + assert.match(cut.at(-1), /[g-z0-9+/]/u); + assert.throws( + () => decodeMappings(cut, COUNTS), + (error) => error instanceof SourceMapError && error.code === 'MAPPINGS_TRUNCATED', + ); + assert.throws( + () => cutInsideValue('AAAA;CAAA'), + refuses('PACKED_CLI_MAP_CONTROL_NO_CONTINUATION'), + ); +}); + +/** A passing run's results: every fixed control and four for each of `chunks` anchor-bearing chunks. */ +function passing(chunks) { + const ids = [ + ...FIXED_CONTROLS, + ...Array.from({ length: chunks }, (_, index) => + ['js-line', 'js-column', 'map-line', 'sources-rotated'].map((kind) => `${kind}-${index + 1}`), + ).flat(), + ]; + return ids.map((id) => ({ id, expected: 'CODE', observed: 'CODE' })); +} + +test('the verdict passes only a complete run whose every control had its expected outcome', () => { + assert.equal(verdict({ results: passing(2), chunks: 2 }).passed, true); + assert.equal(verdict({ results: passing(2), chunks: 2 }).expected, FIXED_CONTROLS.length + 8); + // No anchor-bearing chunk. + assert.equal(verdict({ results: passing(0), chunks: 0 }).passed, false); + // A control that did not run, here a fixed one and a per-chunk one. + for (const id of ['dist-plain-chunk', 'sources-rotated-2']) { + const judged = verdict({ results: passing(2).filter((result) => result.id !== id), chunks: 2 }); + assert.deepEqual([judged.passed, judged.missing], [false, [id]]); + } + // A control that ran twice, or one that is not expected. + const twice = verdict({ + results: [...passing(1), { id: 'restoration', expected: 'A', observed: 'A' }], + chunks: 1, + }); + assert.deepEqual([twice.passed, twice.unexpected], [false, ['restoration']]); + const stray = verdict({ + results: [...passing(1), { id: 'js-line-2', expected: 'A', observed: 'A' }], + chunks: 1, + }); + assert.deepEqual([stray.passed, stray.unexpected], [false, ['js-line-2']]); + // An observed outcome other than the expected one. + const results = passing(1); + results[3] = { ...results[3], observed: 'OTHER' }; + const failed = verdict({ results, chunks: 1 }); + assert.deepEqual([failed.passed, failed.failed], [false, [results[3].id]]); +}); + +test('the report prints one summary and returns 0 only for a passing run', () => { + const run = (outcome) => { + const written = []; + const warned = []; + const code = report(outcome, { + seconds: 3, + write: (line) => written.push(line), + warn: (line) => warned.push(line), + }); + return { code, written, warned }; + }; + const passed = run({ source: 'a'.repeat(40), dirty: false, results: passing(1), chunks: 1 }); + assert.deepEqual([passed.code, passed.warned], [0, []]); + assert.equal(passed.written.length, 1); + const summary = JSON.parse(passed.written[0]); + assert.deepEqual( + [summary.failed, summary.missing, summary.unexpected, summary.chunks], + [[], [], [], 1], + ); + const results = passing(1).filter((result) => result.id !== 'seal-verify'); + results[0] = { ...results[0], observed: 'OTHER' }; + const failed = run({ source: 'a'.repeat(40), dirty: false, results, chunks: 1 }); + assert.equal(failed.code, 1); + assert.deepEqual(failed.warned, [ + `✗ CLI product map controls failed: ${results[0].id}, seal-verify (missing)`, + ]); + const empty = run({ source: 'a'.repeat(40), dirty: false, results: [], chunks: 0 }); + assert.equal(empty.code, 1); + assert.equal(empty.warned.length, 1); +}); + +test('the required control set is exactly the seventeen the contract names', () => { + assert.deepEqual( + [...FIXED_CONTROLS], + [ + 'second-prepack', + 're-seal-receipt', + 're-seal-staging', + 're-recorded-baseline', + 'source-clean', + 'source-dirty', + 'sealed-dist', + 'dist-plain-chunk', + 'dist-rebuilt', + 'upstream-missing', + 'upstream-shape', + 'upstream-index', + 'upstream-cut-bytes', + 'upstream-cut-value', + 'upstream-shifted', + 'seal-verify', + 'restoration', + ], + ); + assert.equal(Object.isFrozen(FIXED_CONTROLS), true); +}); + +test('the entry point exits with the verdict and prints only fixed lines when the run fails', () => { + // The real entry point beside a stand-in core whose run returns or throws as each case needs; the + // report itself is the real one. + const here = (name) => fileURLToPath(new URL(`./${name}`, import.meta.url)); + const directory = mkdtempSync(join(tmpdir(), 'relavium-map-controls-entry-')); + try { + copyFileSync(here('map-controls.mjs'), join(directory, 'map-controls.mjs')); + copyFileSync(here('production-failure.mjs'), join(directory, 'production-failure.mjs')); + const outcome = (results) => ({ source: 'a'.repeat(40), dirty: false, results, chunks: 1 }); + const broken = passing(1); + broken[0] = { ...broken[0], observed: 'OTHER' }; + for (const [label, body, status, stderr] of [ + ['a passing run', `return ${JSON.stringify(outcome(passing(1)))};`, 0, /^$/u], + [ + 'a failed control', + `return ${JSON.stringify(outcome(broken))};`, + 1, + /^✗ CLI product map controls failed: second-prepack\n$/u, + ], + [ + 'a missing control', + `return ${JSON.stringify(outcome(passing(1).slice(1)))};`, + 1, + /^✗ CLI product map controls failed: second-prepack \(missing\)\n$/u, + ], + [ + 'a run that throws', + "throw new Error('PACKED_CLI_MAP_CONTROL_ORIGIN at /private/host/path');", + 1, + /^✗ CLI package production failed: PACKED_CLI_MAP_CONTROL_ORIGIN\n$/u, + ], + ]) { + writeFileSync( + join(directory, 'map-controls-core.mjs'), + `export { report } from ${JSON.stringify(pathToFileURL(here('map-controls-core.mjs')).href)};\n` + + `export async function runMapControls() {\n ${body}\n}\n`, + ); + const run = spawnSync(process.execPath, [join(directory, 'map-controls.mjs')], { + encoding: 'utf8', + timeout: 30_000, + }); + assert.equal(run.status, status, label); + assert.match(run.stderr, stderr, label); + // A completed run prints one summary line; a run that throws prints none. + assert.equal( + run.stdout.split('\n').filter(Boolean).length, + label === 'a run that throws' ? 0 : 1, + label, + ); + } + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/tools/cli-package/pack.mjs b/tools/cli-package/pack.mjs index eec38369..7385427f 100644 --- a/tools/cli-package/pack.mjs +++ b/tools/cli-package/pack.mjs @@ -1,40 +1,37 @@ -/** Fresh dependency emit, sole CLI prepack, independent seal. Never calls a cached build. */ +/** + * Command-line entry point of the package producer (`produce.mjs`): one empty artifact directory in, + * one tarball and its public receipt out. Never builds in the developer's workspace and never calls + * a cached build. A failure prints only `productionFailure`'s fixed lines, never a child's captured + * output, which can quote a map. + */ import assert from 'node:assert/strict'; -import { mkdtempSync, mkdirSync, readdirSync, rmSync, realpathSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join, resolve } from 'node:path'; +import { resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { isolatedEnvironment, runPnpm } from './process.mjs'; -import { sealPackage } from './seal.mjs'; -import { emitFreshPackage } from './pipeline.mjs'; +import { producePackage } from './produce.mjs'; +import { productionFailure } from './production-failure.mjs'; assert.equal(process.argv.length, 3, 'one empty artifact directory required'); -const repository = fileURLToPath(new URL('../../', import.meta.url)); -const destination = resolve(process.argv[2]); -mkdirSync(destination, { recursive: true }); -assert.deepEqual(readdirSync(destination), [], 'PACKED_CLI_FRESH_DESTINATION_REQUIRED'); -const home = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-package-build-'))); +let produced; try { - const env = isolatedEnvironment(home); - const version = await runPnpm(['--version'], { cwd: repository, env }); - assert.equal(version.stdout.trim(), '9.12.3', 'PACKED_CLI_PNPM_VERSION'); - const receipt = await emitFreshPackage({ - repository, - destination, - env, - pnpm: runPnpm, - removeDist: (path) => rmSync(path, { recursive: true, force: true }), - seal: sealPackage, + produced = await producePackage({ + repository: fileURLToPath(new URL('../../', import.meta.url)), + destination: resolve(process.argv[2]), + environment: process.env, }); - console.log( - JSON.stringify({ - mode: receipt.mode, - source: receipt.source, - sha256: receipt.tarballSha256, - files: receipt.files.length, - mapsAndSymbolication: 'not implemented until Wave 3 Step 9', - }), - ); -} finally { - rmSync(home, { recursive: true, force: true }); +} catch (error) { + for (const line of productionFailure(error)) console.error(line); + process.exit(1); } +const { receipt, identity } = produced; +console.log( + JSON.stringify({ + mode: receipt.mode, + source: receipt.source, + sha256: receipt.tarballSha256, + files: receipt.files.length, + candidateFiles: identity.files, + privateMaps: 'staged privately, then discarded', + privateManifest: 'sealed privately, then discarded', + symbolication: 'actual packed CLI and core frames verified', + }), +); diff --git a/tools/cli-package/paths.mjs b/tools/cli-package/paths.mjs new file mode 100644 index 00000000..2519745e --- /dev/null +++ b/tools/cli-package/paths.mjs @@ -0,0 +1,23 @@ +/** Path containment shared by the producer, the candidate tree, the finalizer and the consumer. */ +import { realpathSync } from 'node:fs'; +import { isAbsolute, relative, sep } from 'node:path'; + +/** + * Whether `child` is `parent` or lies inside it, decided by path segments: a child whose first segment + * merely starts with `..` (`..private`, `...`) is inside, while `..` and `../x` are not. + * @param {string} parent @param {string} child + */ +export function within(parent, child) { + const path = relative(parent, child); + return path === '' || (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)); +} + +/** + * Whether the existing `child` lies inside the existing `parent`, decided on the real paths and the + * letter case the filesystem stores (`realpathSync.native`), so neither a link nor another letter case + * of the same directory passes for elsewhere. + * @param {string} parent @param {string} child + */ +export function placedWithin(parent, child) { + return within(realpathSync.native(parent), realpathSync.native(child)); +} diff --git a/tools/cli-package/paths.test.mjs b/tools/cli-package/paths.test.mjs new file mode 100644 index 00000000..ce551cea --- /dev/null +++ b/tools/cli-package/paths.test.mjs @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict'; +import { existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join } from 'node:path'; +import { after, test } from 'node:test'; +import { placedWithin, within } from './paths.mjs'; + +const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-paths-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); + +test('containment is decided by whole path segments', () => { + assert.equal(within('/a/b', '/a/b'), true); + assert.equal(within('/a/b', '/a/b/c'), true); + // A segment that merely starts with two dots is inside. + assert.equal(within('/a/b', '/a/b/..c'), true); + assert.equal(within('/a/b', '/a'), false); + assert.equal(within('/a/b', '/a/c'), false); + assert.equal(within('/a/b', '/a/bc'), false); +}); + +test('placement compares the real paths and stored letter case of existing directories', () => { + const checkout = join(scratch, 'checkout'); + const inside = join(checkout, 'tmp'); + const outside = join(scratch, 'elsewhere'); + for (const directory of [inside, outside]) mkdirSync(directory, { recursive: true }); + assert.equal(placedWithin(checkout, inside), true); + assert.equal(placedWithin(checkout, outside), false); + // A link to the checkout is the checkout. + const alias = join(scratch, 'alias'); + symlinkSync(checkout, alias, process.platform === 'win32' ? 'junction' : 'dir'); + assert.equal(placedWithin(checkout, join(alias, 'tmp')), true); + assert.equal(placedWithin(alias, inside), true); + // On a case-insensitive filesystem, another letter case of the checkout is the checkout. + const variant = join(dirname(checkout), basename(checkout).toUpperCase()); + if (existsSync(variant)) { + assert.equal(placedWithin(checkout, join(variant, 'tmp')), true); + assert.equal(placedWithin(variant, inside), true); + } +}); diff --git a/tools/cli-package/pipeline.mjs b/tools/cli-package/pipeline.mjs index 7317cfd4..8beb5ddb 100644 --- a/tools/cli-package/pipeline.mjs +++ b/tools/cli-package/pipeline.mjs @@ -1,25 +1,55 @@ import assert from 'node:assert/strict'; import { join } from 'node:path'; +import { CANDIDATE_STAGING_ENV, ENGINE_PACKAGES } from './finalize.mjs'; -/** One ordered emitter; ports are tooling boundaries, never shipped product hooks. */ -export async function emitFreshPackage({ repository, destination, env, pnpm, removeDist, seal }) { - for (const name of ['shared', 'llm', 'core', 'db', 'mcp']) { +/** The installed compiler's candidate overrides: external upstream maps, no embedded sources or incremental state. */ +export const CANDIDATE_TSC_ARGS = Object.freeze([ + 'exec', + 'tsc', + '-p', + 'tsconfig.build.json', + '--sourceMap', + 'true', + '--inlineSources', + 'false', + '--incremental', + 'false', +]); + +/** + * One ordered emitter for an owned candidate tree; ports are tooling boundaries, never shipped + * product hooks. The inlined dependencies compile fresh with upstream maps, then prepack performs + * the sole final CLI build and finalizes the candidate into the private `staging` directory. + */ +export async function emitFreshPackage({ + repository, + destination, + staging, + env, + pnpm, + removeBuildState, + seal, +}) { + for (const name of ENGINE_PACKAGES) { const cwd = join(repository, 'packages', name); - await removeDist(join(cwd, 'dist')); - await pnpm(['exec', 'tsc', '-p', 'tsconfig.build.json', '--incremental', 'false'], { - cwd, - env, - }); + await removeBuildState(cwd); + await pnpm([...CANDIDATE_TSC_ARGS], { cwd, env }); } const packed = await pnpm(['pack', '--pack-destination', destination], { cwd: join(repository, 'apps/cli'), - env, + env: { ...env, [CANDIDATE_STAGING_ENV]: staging }, }); - // Independently observe the installed packer's sole prepack build, rather than merely counting API calls. + // Independently observe the installed packer's sole prepack build and its one finalization, + // rather than merely counting API calls. assert.equal( (packed.stdout.match(/\bESM Build start\b/gu) ?? []).length, 1, 'PACKED_CLI_SOLE_PREPACK_BUILD_REQUIRED', ); + assert.equal( + (packed.stdout.match(/^✓ CLI candidate finalized: /gmu) ?? []).length, + 1, + 'PACKED_CLI_CANDIDATE_FINALIZATION_REQUIRED', + ); return seal(destination); } diff --git a/tools/cli-package/pipeline.test.mjs b/tools/cli-package/pipeline.test.mjs index cbf9a989..f4472d22 100644 --- a/tools/cli-package/pipeline.test.mjs +++ b/tools/cli-package/pipeline.test.mjs @@ -2,23 +2,28 @@ import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { test } from 'node:test'; -import { emitFreshPackage } from './pipeline.mjs'; +import { CANDIDATE_TSC_ARGS, emitFreshPackage } from './pipeline.mjs'; -function fixture(failAt, stdout = 'ESM Build start\nESM Build success') { - const repository = join(process.cwd(), 'synthetic-owned-checkout'); +const BUILT = 'ESM Build start\nESM Build success\n'; +const FINALIZED = '✓ CLI candidate finalized: 3 output(s), 3 private map(s), 2 upstream map(s)\n'; + +function fixture(failAt, stdout = `${BUILT}${FINALIZED}`) { + const repository = join(process.cwd(), 'synthetic-owned-candidate'); const destination = join(process.cwd(), 'synthetic-public-output'); + const staging = join(process.cwd(), 'synthetic-private-staging'); const calls = []; let commands = 0; return { calls, repository, destination, + staging, env: { HOME: 'synthetic' }, - async removeDist(path) { + async removeBuildState(path) { calls.push({ remove: path }); }, async pnpm(args, options) { - calls.push({ args, cwd: options.cwd }); + calls.push({ args, cwd: options.cwd, env: options.env }); commands += 1; if (commands === failAt) throw new Error('synthetic dependency or prepack failure'); return { stdout }; @@ -30,18 +35,20 @@ function fixture(failAt, stdout = 'ESM Build start\nESM Build success') { }; } -test('fresh pipeline orders five dependency emits before the sole prepack and independent seal', async () => { +test('fresh pipeline orders five mapped dependency emits before the sole candidate prepack and seal', async () => { const ports = fixture(); assert.equal(await emitFreshPackage(ports), 'actual inspected receipt'); const commands = ports.calls.filter((call) => call.args); + const packages = ['shared', 'llm', 'core', 'db', 'mcp'].map((name) => + join(ports.repository, 'packages', name), + ); assert.deepEqual( commands.map((call) => call.cwd), - [ - ...['shared', 'llm', 'core', 'db', 'mcp'].map((name) => - join(ports.repository, 'packages', name), - ), - join(ports.repository, 'apps/cli'), - ], + [...packages, join(ports.repository, 'apps/cli')], + ); + assert.deepEqual( + ports.calls.filter((call) => call.remove).map((call) => call.remove), + packages, ); assert.deepEqual( commands.slice(0, 5).map((call) => call.args), @@ -50,13 +57,30 @@ test('fresh pipeline orders five dependency emits before the sole prepack and in 'tsc', '-p', 'tsconfig.build.json', + '--sourceMap', + 'true', + '--inlineSources', + 'false', '--incremental', 'false', ]), ); + assert.deepEqual(CANDIDATE_TSC_ARGS, commands[0].args); + // Each package's build state goes immediately before that package compiles, never after. + assert.deepEqual( + ports.calls + .slice(0, 10) + .map((call) => (call.remove ? `remove ${call.remove}` : `exec ${call.cwd}`)), + packages.flatMap((path) => [`remove ${path}`, `exec ${path}`]), + ); + // Only the sole prepack learns the private staging directory. + assert.ok(commands.slice(0, 5).every((call) => call.env === ports.env)); assert.deepEqual(commands[5].args, ['pack', '--pack-destination', ports.destination]); + assert.deepEqual(commands[5].env, { + HOME: 'synthetic', + RELAVIUM_CLI_CANDIDATE_STAGING: ports.staging, + }); assert.deepEqual(ports.calls.at(-1), { seal: ports.destination }); - assert.equal(ports.calls.filter((call) => call.remove).length, 5); assert.ok(commands.every((call) => !call.args.includes('turbo') && !call.args.includes('build'))); }); @@ -71,7 +95,7 @@ for (const failAt of [1, 2, 3, 4, 5, 6]) ); }); -for (const stdout of ['', 'ESM Build start\nESM Build start\n']) +for (const stdout of ['', `${BUILT}${BUILT}${FINALIZED}`, FINALIZED]) test(`missing/duplicate observed prepack build refuses before seal: ${JSON.stringify(stdout)}`, async () => { const ports = fixture(undefined, stdout); await assert.rejects(emitFreshPackage(ports), /SOLE_PREPACK_BUILD_REQUIRED/); @@ -81,9 +105,20 @@ for (const stdout of ['', 'ESM Build start\nESM Build start\n']) ); }); -test('the shipped prepack lifecycle has exactly one build and runs its closure gate', () => { +for (const stdout of [BUILT, `${BUILT}${FINALIZED}${FINALIZED}`, `${BUILT} ${FINALIZED}`]) + test(`missing/duplicate candidate finalization refuses before seal: ${JSON.stringify(stdout)}`, async () => { + const ports = fixture(undefined, stdout); + await assert.rejects(emitFreshPackage(ports), /CANDIDATE_FINALIZATION_REQUIRED/); + assert.equal( + ports.calls.some((call) => call.seal), + false, + ); + }); + +test('the shipped prepack lifecycle is the sole orchestrated build and closure gate', () => { const metadata = JSON.parse( readFileSync(new URL('../../apps/cli/package.json', import.meta.url), 'utf8'), ); - assert.equal(metadata.scripts.prepack, 'tsup && pnpm -w run lint:bundle-closure'); + assert.equal(metadata.scripts.prepack, 'node ../../tools/cli-package/prepack.mjs'); + assert.equal(metadata.scripts.build, 'tsup'); }); diff --git a/tools/cli-package/prepack.mjs b/tools/cli-package/prepack.mjs new file mode 100644 index 00000000..a297b266 --- /dev/null +++ b/tools/cli-package/prepack.mjs @@ -0,0 +1,62 @@ +/** + * The CLI's sole prepack lifecycle (ADR-0111). It runs exactly one installed tsup build, then + * either the ordinary map-free closure guard or — only when the trusted package producer names a + * private staging directory — the candidate finalizer. pnpm packs only after this exits 0, and + * nothing builds, transforms or writes the JS after it. + */ +import { AssertionError } from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { BundleClosureError } from '../bundle-closure/closure.mjs'; +import { + CANDIDATE_STAGING_ENV, + CandidateFinalizationError, + assertPrivateStaging, + finalizeCandidate, +} from './finalize.mjs'; + +const repository = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const cli = join(repository, 'apps', 'cli'); + +/** @param {string} message @returns {never} */ +function fail(message) { + console.error(`✗ ${message}`); + process.exit(1); +} + +/** @param {string} script @param {string} cwd */ +function runNode(script, cwd) { + const result = spawnSync(process.execPath, [script], { cwd, stdio: 'inherit', shell: false }); + return result.error === undefined && result.status === 0; +} + +const staging = process.env[CANDIDATE_STAGING_ENV]; +try { + if (resolve(process.cwd()) !== cli) fail('CLI prepack must run in its package directory'); + // Refuse an unusable staging directory before anything is built. + if (staging !== undefined) assertPrivateStaging(staging, repository); + const tsupManifest = createRequire(join(cli, 'package.json')).resolve('tsup/package.json'); + const tsup = join(dirname(tsupManifest), JSON.parse(readFileSync(tsupManifest, 'utf8')).bin.tsup); + if (!runNode(tsup, cli)) fail('CLI build failed'); + if (staging === undefined) { + if (!runNode(join(repository, 'tools', 'bundle-closure', 'check.mjs'), repository)) { + process.exit(1); + } + } else { + const record = finalizeCandidate({ repository, staging }); + console.log( + `✓ CLI candidate finalized: ${record.outputs.length} output(s), ` + + `${record.maps.length} private map(s), ${record.upstreamMaps} upstream map(s)`, + ); + } +} catch (error) { + // Every known refusal carries a fixed code and at most a repository-relative path. + if (error instanceof CandidateFinalizationError) fail(error.message); + if (error instanceof BundleClosureError) fail(`CLI candidate closure: ${error.message}`); + if (error instanceof AssertionError) fail(`CLI candidate finalization: ${error.message}`); + const code = error instanceof Error && 'code' in error ? String(error.code) : 'UNEXPECTED'; + fail(`CLI candidate finalization failed: ${code}`); +} diff --git a/tools/cli-package/prepack.test.mjs b/tools/cli-package/prepack.test.mjs new file mode 100644 index 00000000..4a8046d1 --- /dev/null +++ b/tools/cli-package/prepack.test.mjs @@ -0,0 +1,244 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +// The orchestrator runs from a copied tooling tree with a fake tsup that only records that it ran, +// so no case can rebuild the real CLI even if the orchestration regresses. +const tooling = fileURLToPath(new URL('../', import.meta.url)); +const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-prepack-orchestrator-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); +let count = 0; + +function fixture({ buildExit = 0, tsup: installed = true, finalizer } = {}) { + count += 1; + const root = join(scratch, `repository-${count}`); + for (const path of [ + 'cli-package/prepack.mjs', + 'cli-package/finalize.mjs', + 'cli-package/source-map.mjs', + 'cli-package/files.mjs', + 'cli-package/paths.mjs', + 'bundle-closure/closure.mjs', + 'bundle-closure/inventory.mjs', + 'bundle-closure/check.mjs', + ]) { + mkdirSync(dirname(join(root, 'tools', path)), { recursive: true }); + copyFileSync(join(tooling, path), join(root, 'tools', path)); + } + symlinkSync( + fileURLToPath(new URL('../../node_modules', import.meta.url)), + join(root, 'node_modules'), + process.platform === 'win32' ? 'junction' : 'dir', + ); + if (finalizer !== undefined) { + // The real staging check and error class, with a stand-in for the finalization itself. + const tools = join(root, 'tools', 'cli-package'); + copyFileSync(join(tools, 'finalize.mjs'), join(tools, 'finalize-real.mjs')); + writeFileSync( + join(tools, 'finalize.mjs'), + `export { CANDIDATE_STAGING_ENV, CandidateFinalizationError, assertPrivateStaging } from './finalize-real.mjs';\n` + + `import { BundleClosureError } from '../bundle-closure/closure.mjs';\n` + + `import { AssertionError } from 'node:assert/strict';\n` + + `import { writeFileSync } from 'node:fs';\n` + + `export function finalizeCandidate(options) {\n${finalizer}\n}\n` + + `void BundleClosureError, AssertionError, writeFileSync;\n`, + ); + } + const cli = join(root, 'apps', 'cli'); + const tsup = join(cli, 'node_modules', 'tsup'); + const marker = join(root, 'build-ran'); + mkdirSync(cli, { recursive: true }); + writeFileSync(join(cli, 'package.json'), JSON.stringify({ name: 'relavium', dependencies: {} })); + if (!installed) return { root, cli, marker }; + mkdirSync(join(tsup, 'dist'), { recursive: true }); + writeFileSync( + join(tsup, 'package.json'), + JSON.stringify({ name: 'tsup', type: 'module', bin: { tsup: 'dist/cli.js' } }), + ); + writeFileSync( + join(tsup, 'dist', 'cli.js'), + `import { appendFileSync } from 'node:fs';\n` + + `appendFileSync(${JSON.stringify(marker)}, (process.env.RELAVIUM_CLI_CANDIDATE_STAGING ?? 'ordinary') + '\\n');\n` + + `console.log('ESM Build start');\nprocess.exit(${buildExit});\n`, + ); + return { root, cli, marker }; +} + +function prepack({ cli, root }, { cwd = cli, staging } = {}) { + const env = { ...process.env }; + delete env.RELAVIUM_CLI_CANDIDATE_STAGING; + if (staging !== undefined) env.RELAVIUM_CLI_CANDIDATE_STAGING = staging; + const result = spawnSync(process.execPath, [join(root, 'tools', 'cli-package', 'prepack.mjs')], { + cwd, + env, + encoding: 'utf8', + timeout: 20_000, + }); + assert.equal(result.error, undefined); + return result; +} + +function privateStaging() { + count += 1; + const staging = join(scratch, `staging-${count}`); + mkdirSync(staging); + chmodSync(staging, 0o700); + return staging; +} + +test('prepack runs only in the CLI package directory', () => { + const candidate = fixture(); + const result = prepack(candidate, { cwd: candidate.root }); + assert.equal(result.status, 1); + assert.match(result.stderr, /✗ CLI prepack must run in its package directory/u); + assert.equal(existsSync(candidate.marker), false); +}); + +test('an unusable staging directory refuses before anything is built', () => { + for (const staging of ['relative', join(scratch, 'absent')]) { + const candidate = fixture(); + const result = prepack(candidate, { staging }); + assert.equal(result.status, 1); + assert.match(result.stderr, /CLI candidate finalization refused: STAGING_PATH/u); + assert.equal(existsSync(candidate.marker), false); + } +}); + +test('a second prepack into a sealed staging refuses before anything is built', () => { + // The staging of a sealed candidate: the finalizer's record, the metafile, the staged maps and the + // private manifest (ADR-0111's post-receipt rebuild refusal; Step 9a.2b). + const candidate = fixture(); + const staging = privateStaging(); + const sealed = { + 'finalized.json': '{"version":1}', + 'metafile-esm.json': '{"outputs":{}}', + 'manifest.json': '{"version":1}', + }; + for (const [name, text] of Object.entries(sealed)) writeFileSync(join(staging, name), text); + mkdirSync(join(staging, 'maps')); + writeFileSync(join(staging, 'maps', 'index.js.map'), '{"version":3}'); + const result = prepack(candidate, { staging }); + assert.equal(result.status, 1); + assert.match(result.stderr, /CLI candidate finalization refused: STAGING_NOT_EMPTY/u); + assert.equal(existsSync(candidate.marker), false); + assert.deepEqual(readdirSync(staging).sort(), [...Object.keys(sealed), 'maps'].sort()); + for (const [name, text] of Object.entries(sealed)) { + assert.equal(readFileSync(join(staging, name), 'utf8'), text); + } + assert.equal(readFileSync(join(staging, 'maps', 'index.js.map'), 'utf8'), '{"version":3}'); +}); + +test('a failed build stops before the closure gate or finalization', () => { + const candidate = fixture({ buildExit: 3 }); + const staging = privateStaging(); + const result = prepack(candidate, { staging }); + assert.equal(result.status, 1); + assert.match(result.stderr, /✗ CLI build failed/u); + assert.equal(readFileSync(candidate.marker, 'utf8'), `${staging}\n`); + assert.equal(existsSync(join(staging, 'maps')), false); +}); + +test('one build runs, then the ordinary closure gate or the candidate finalizer judges its output', () => { + // The fake build emits nothing, so both judges must refuse the missing output after one build. + const ordinary = fixture(); + const plain = prepack(ordinary); + assert.equal(plain.status, 1); + assert.equal(readFileSync(ordinary.marker, 'utf8'), 'ordinary\n'); + assert.match(plain.stdout, /ESM Build start/u); + assert.match(plain.stderr, /✗ CLI bundle closure: cannot read valid build inventory/u); + + const candidate = fixture(); + const staging = privateStaging(); + const mapped = prepack(candidate, { staging }); + assert.equal(mapped.status, 1); + assert.equal(readFileSync(candidate.marker, 'utf8'), `${staging}\n`); + assert.match( + mapped.stderr, + /✗ CLI candidate finalization refused: UNREADABLE_FILE \(metafile\)/u, + ); + assert.doesNotMatch(mapped.stdout, /CLI candidate finalized/u); +}); + +test('an unexpected error prints only its code, never its message or a host path', () => { + // Without an installed tsup the resolution fails with an error whose message names absolute + // fixture paths; only its code may reach the pack log. + const candidate = fixture({ tsup: false }); + const staging = privateStaging(); + const result = prepack(candidate, { staging }); + assert.equal(result.status, 1); + assert.equal(result.stdout, ''); + assert.equal(result.stderr, '✗ CLI candidate finalization failed: MODULE_NOT_FOUND\n'); + assert.equal(existsSync(candidate.marker), false); +}); + +test('a staging directory inside the repository but outside the CLI refuses before the build', () => { + const candidate = fixture(); + const staging = join(candidate.root, `staging-${count}`); + mkdirSync(staging); + chmodSync(staging, 0o700); + const result = prepack(candidate, { staging }); + assert.equal(result.status, 1); + assert.match(result.stderr, /CLI candidate finalization refused: STAGING_PATH/u); + assert.equal(existsSync(candidate.marker), false); +}); + +test('a successful finalization gets the repository and staging and reports on stdout', () => { + const record = join(scratch, `finalizer-options-${count + 1}.json`); + const candidate = fixture({ + finalizer: + `writeFileSync(${JSON.stringify(record)}, JSON.stringify(options));\n` + + `return { outputs: [1, 2], maps: [1, 2], upstreamMaps: 5 };`, + }); + const staging = privateStaging(); + const result = prepack(candidate, { staging }); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(readFileSync(record, 'utf8')), { + repository: candidate.root, + staging, + }); + // The pipeline counts exactly this line on stdout. + assert.equal( + (result.stdout.match(/^✓ CLI candidate finalized: /gmu) ?? []).length, + 1, + result.stdout, + ); + assert.match( + result.stdout, + /^✓ CLI candidate finalized: 2 output\(s\), 2 private map\(s\), 5 upstream map\(s\)$/mu, + ); +}); + +for (const [label, thrown, printed] of [ + [ + 'a closure refusal', + `throw new BundleClosureError('MAP_LINK', 'runtime output links a source map: dist/index.js');`, + '✗ CLI candidate closure: MAP_LINK: runtime output links a source map: dist/index.js\n', + ], + [ + 'an assertion', + `throw new AssertionError({ message: 'PACKED_CLI_SYNTHETIC_ASSERTION' });`, + '✗ CLI candidate finalization: PACKED_CLI_SYNTHETIC_ASSERTION\n', + ], +]) + test(`${label} prints its fixed line and nothing else`, () => { + const candidate = fixture({ finalizer: thrown }); + const result = prepack(candidate, { staging: privateStaging() }); + assert.equal(result.status, 1); + assert.equal(result.stderr, printed); + }); diff --git a/tools/cli-package/process.mjs b/tools/cli-package/process.mjs index 89127b3e..67f9488a 100644 --- a/tools/cli-package/process.mjs +++ b/tools/cli-package/process.mjs @@ -1,10 +1,33 @@ /** Native tooling boundary. Never starts a shell or inherits credentials/config/preloads. */ import assert from 'node:assert/strict'; -import { spawn } from 'node:child_process'; +import { spawn, spawnSync } from 'node:child_process'; import { Buffer } from 'node:buffer'; import { existsSync, lstatSync, readFileSync, realpathSync, statSync } from 'node:fs'; -import { delimiter, extname, isAbsolute, join } from 'node:path'; -import { clearTimeout, setTimeout } from 'node:timers'; +import { constants, homedir } from 'node:os'; +import { delimiter, extname, isAbsolute, join, resolve } from 'node:path'; +import { clearTimeout, setImmediate, setTimeout } from 'node:timers'; + +/** + * Corepack's own cache, computed as corepack computes it (`COREPACK_HOME`, else `XDG_CACHE_HOME`, else + * `LOCALAPPDATA`, else the home directory's `.cache` or `AppData/Local`, plus `node/corepack`), from + * this process's environment, which is the developer's own or carries a `COREPACK_HOME` a caller + * computed from it. An isolated child passes it on, so a corepack-managed pnpm runs from that cache — + * never from an isolated home in the shared temporary directory, where its optional module lookups + * would reach a `node_modules` that another local user can create. + * @param {Record} [environment] + */ +export function corepackHome(environment = process.env) { + return resolve( + environment.COREPACK_HOME ?? + join( + environment.XDG_CACHE_HOME ?? + environment.LOCALAPPDATA ?? + join(homedir(), process.platform === 'win32' ? join('AppData', 'Local') : '.cache'), + 'node', + 'corepack', + ), + ); +} export function isolatedEnvironment(home) { const env = { @@ -19,12 +42,39 @@ export function isolatedEnvironment(home) { NO_COLOR: '1', TERM: 'dumb', COREPACK_ENABLE_DOWNLOAD_PROMPT: '0', + // A location only, never configuration: see `corepackHome`. + COREPACK_HOME: corepackHome(), }; for (const key of ['PATH', 'SystemRoot', 'WINDIR', 'LANG']) if (process.env[key] !== undefined) env[key] = process.env[key]; return env; } +/** + * The null device as Git names it on every platform. Git for Windows maps only `nul` and this literal + * to the device; Node's `os.devNull` there (`\\.\nul`) reaches the C runtime, which refuses it, and + * Git then exits rather than read no configuration. + */ +export const GIT_NULL_DEVICE = '/dev/null'; + +/** + * Git with no global or system configuration, for reading the original checkout: its own `.git/config` + * and ignore rules still apply and a caller passes the developer's global excludes explicitly, but no + * `.gitconfig`, `git/config`, `git/ignore` or `git/attributes` in a home, configuration or system + * location is read — not even one inside the checkout itself (the callers also name the attributes and + * excludes files explicitly, so no default location applies either). Replace refs are ignored, so a status + * compares the working tree with the commit HEAD names, never a substitute for it. + */ +export function isolatedGitEnvironment() { + return { + ...isolatedEnvironment(GIT_NULL_DEVICE), + GIT_CONFIG_NOSYSTEM: '1', + GIT_ATTR_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: GIT_NULL_DEVICE, + GIT_NO_REPLACE_OBJECTS: '1', + }; +} + /** Resolve reviewed pnpm JS shims or the action's standalone executable, without cmd.exe. */ export function resolvePnpm(path = process.env.PATH ?? '', platform = process.platform) { for (const directory of path.split(delimiter).filter(Boolean)) { @@ -58,9 +108,125 @@ export function resolvePnpm(path = process.env.PATH ?? '', platform = process.pl throw new Error('PACKED_CLI_PINNED_PNPM_NOT_FOUND'); } +/** Every child tree a `runCommand` call still owns, so that an interrupted tool can stop them first. */ +const liveChildren = new Set(); + +/** + * Synchronously kill every child tree that `runCommand` still owns: a POSIX process group, or a + * Windows tree through taskkill. For a signal handler that must stop detached builds before it exits. + */ +export function stopLiveCommands() { + for (const child of liveChildren) { + if (child.grouped) { + try { + process.kill(-child.pid, 'SIGKILL'); + } catch { + // The group already exited. + } + continue; + } + const systemRoot = process.env.SystemRoot; + if (systemRoot && isAbsolute(systemRoot)) { + spawnSync( + join(systemRoot, 'System32', 'taskkill.exe'), + ['/PID', String(child.pid), '/T', '/F'], + { shell: false, stdio: 'ignore' }, + ); + } else { + try { + process.kill(child.pid, 'SIGKILL'); + } catch { + // The child already exited. + } + } + } + liveChildren.clear(); +} + +/** The interrupts after which a tool still stops its children and cleans up before it exits. */ +const INTERRUPTS = Object.freeze(['SIGINT', 'SIGTERM', 'SIGHUP']); + +/** + * Let the event loop poll for I/O once more. Node hands a caught signal to its listener only when the + * loop next polls, and work that ran from an I/O callback (a child's exit, a file read) can end its + * first turn before that poll: a listener removed then drops the signal. After two turns the loop has + * polled. A signal caught in the instant between that poll and the listener's removal is still lost; + * Node offers no way to drain it. + */ +async function deliverCaughtSignals() { + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); +} + +/** + * Run `body` while SIGINT, SIGTERM and SIGHUP stop every child tree `runCommand` owns, run the + * synchronous `interrupted` and end the process with 128 + the signal's number. The handlers stay + * until the loop has polled once after `body` settled (`deliverCaughtSignals`), so an interrupt caught + * during the body's synchronous tail or its own final cleanup is honoured, never dropped with its + * listener. A process killed outright runs nothing. + * @template T + * @param {() => void} interrupted + * @param {() => Promise} body + * @returns {Promise} + */ +export async function exitOnInterrupt(interrupted, body) { + const handler = (signal) => { + stopLiveCommands(); + interrupted(); + process.exit(128 + (constants.signals[signal] ?? 0)); + }; + for (const signal of INTERRUPTS) process.on(signal, handler); + try { + return await body(); + } finally { + await deliverCaughtSignals(); + for (const signal of INTERRUPTS) process.off(signal, handler); + } +} + +/** + * While a tool waits on children that clean up after themselves (the producer, the consumer), + * forward SIGINT, SIGTERM and SIGHUP to every child tree `runCommand` owns instead of dying first: a + * detached child shares no terminal process group, so it would never receive the interrupt. On + * Windows the children share the console and receive Ctrl-C themselves. The caller runs its own cleanup, awaits `dispose` + * and exits with 128 + the number of the signal `received` reports. + * @returns {{received: () => string | undefined, dispose: () => Promise}} + */ +export function forwardInterrupts() { + let received; + const forward = (signal) => { + received ??= signal; + for (const child of liveChildren) { + if (!child.grouped) continue; + try { + process.kill(-child.pid, signal); + } catch { + // The group already exited. + } + } + }; + for (const signal of INTERRUPTS) process.on(signal, forward); + return { + received: () => received, + // An interrupt caught during the caller's cleanup is still recorded (`deliverCaughtSignals`). + dispose: async () => { + await deliverCaughtSignals(); + for (const signal of INTERRUPTS) process.off(signal, forward); + }, + }; +} + +/** + * Run one tool child in its own process group (POSIX) and settle when it closed. Past `timeoutMs` its + * group (on Windows its whole tree) is killed. On POSIX with a positive `graceMs` the group first gets + * SIGTERM and `graceMs` more, so a child that stops its own detached children and removes its private + * directories on an interrupt (the producer, the consumer) does so before anything is killed; the group + * is killed when the grace ends or as soon as the child closes, whichever comes first. + */ export function runCommand(command, args, options = {}) { - const { timeoutMs = 300_000, maxBytes = 8 * 1024 * 1024, ...spawnOptions } = options; + const { timeoutMs = 300_000, graceMs = 0, maxBytes = 8 * 1024 * 1024, ...spawnOptions } = options; assert.ok(Number.isSafeInteger(timeoutMs) && timeoutMs > 0); + assert.ok(Number.isSafeInteger(graceMs) && graceMs >= 0); assert.ok(Number.isSafeInteger(maxBytes) && maxBytes > 0); return new Promise((resolve, reject) => { const grouped = process.platform !== 'win32'; @@ -70,11 +236,14 @@ export function runCommand(command, args, options = {}) { shell: false, stdio: ['ignore', 'pipe', 'pipe'], }); + const live = { pid: child.pid, grouped }; + if (child.pid !== undefined) liveChildren.add(live); let bytes = 0, fault, closed = false, killer, - killerTimer; + killerTimer, + graceTimer; const output = { stdout: [], stderr: [] }; function stop() { if (closed || child.pid === undefined) return; @@ -103,7 +272,19 @@ export function runCommand(command, args, options = {}) { } const timer = setTimeout(() => { fault ??= new Error('PACKED_CLI_CHILD_TIMEOUT'); - stop(); + if (graceMs === 0 || !grouped || child.pid === undefined) { + stop(); + return; + } + try { + process.kill(-child.pid, 'SIGTERM'); + } catch { + // The group already exited; the timeout fault already stands. + } + graceTimer = setTimeout(() => { + graceTimer = undefined; + stop(); + }, graceMs); }, timeoutMs); for (const sink of ['stdout', 'stderr']) child[sink].on('data', (chunk) => { @@ -118,7 +299,18 @@ export function runCommand(command, args, options = {}) { }); child.once('close', async (code, signal) => { closed = true; + liveChildren.delete(live); clearTimeout(timer); + if (graceTimer !== undefined) { + // The leader exited on the SIGTERM before the grace ran out: a group member that ignored it + // is killed now, as it would have been when the grace ended. + clearTimeout(graceTimer); + try { + process.kill(-child.pid, 'SIGKILL'); + } catch (error) { + if (error.code !== 'ESRCH') fault ??= error; + } + } if (killer && killer.exitCode === null && killer.signalCode === null) await new Promise((done) => killer.once('close', done)); clearTimeout(killerTimer); diff --git a/tools/cli-package/process.test.mjs b/tools/cli-package/process.test.mjs index 318db0f3..79be020f 100644 --- a/tools/cli-package/process.test.mjs +++ b/tools/cli-package/process.test.mjs @@ -1,16 +1,45 @@ import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; import { test } from 'node:test'; -import { mkdtempSync, mkdirSync, writeFileSync, rmSync, realpathSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; import { + existsSync, + mkdtempSync, + mkdirSync, + readFileSync, + writeFileSync, + rmSync, + realpathSync, +} from 'node:fs'; +import { homedir, tmpdir } from 'node:os'; +import { join, resolve, sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { exitWithin } from './child-exit.mjs'; +import { + corepackHome, exerciseInstalledCommand, + forwardInterrupts, isolatedEnvironment, + isolatedGitEnvironment, resolvePnpm, runCommand, runPnpm, + stopLiveCommands, } from './process.mjs'; +test('Git reading the original checkout gets no home, global or system configuration nor replace refs', () => { + const env = isolatedGitEnvironment(); + assert.deepEqual(env, { + ...isolatedEnvironment('/dev/null'), + GIT_CONFIG_NOSYSTEM: '1', + GIT_ATTR_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_NO_REPLACE_OBJECTS: '1', + }); + // The literal Git maps to the null device on every platform, never Node's Windows `\\.\nul`. + for (const key of ['HOME', 'USERPROFILE', 'XDG_CONFIG_HOME']) assert.equal(env[key], '/dev/null'); +}); + test('installer receives an explicit credential/config/preload-free environment', () => { const home = join(tmpdir(), 'synthetic-installer-home'); const env = isolatedEnvironment(home); @@ -155,3 +184,435 @@ test('actual pnpm install command shim runs help/version and a broken or absent rmSync(root, { recursive: true, force: true }); } }); + +function alive(pid) { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function waitForFile(file) { + for (let waited = 0; !existsSync(file) || readFileSync(file, 'utf8') === ''; waited += 25) { + assert.ok(waited < 20_000, `${file} never appeared`); + await delay(25); + } + return readFileSync(file, 'utf8'); +} + +test( + 'a tool forwards an interrupt to the detached children that clean up after themselves', + { skip: process.platform === 'win32' }, + async () => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-forwarded-interrupt-'))); + const pidFile = join(root, 'child.pid'); + const marker = join(root, 'received'); + // The child cleans up on SIGINT, as the producer does; a SIGKILL would leave no marker. + const childCode = + `const { writeFileSync } = require('node:fs');` + + `process.on('SIGINT', () => { writeFileSync(${JSON.stringify(marker)}, 'SIGINT'); process.exit(0); });` + + `writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + `setInterval(() => {}, 1000);`; + const harness = ` +import { constants } from 'node:os'; +import { forwardInterrupts, isolatedEnvironment, runCommand } from ${JSON.stringify(new URL('./process.mjs', import.meta.url).href)}; +const interrupts = forwardInterrupts(); +try { + await runCommand(process.execPath, ['-e', ${JSON.stringify(childCode)}], { + env: isolatedEnvironment(${JSON.stringify(root)}), + timeoutMs: 60_000, + }); +} catch (error) { + if (interrupts.received() === undefined) throw error; +} finally { + await interrupts.dispose(); +} +const signal = interrupts.received(); +if (signal !== undefined) process.exit(128 + constants.signals[signal]); +`; + const tool = spawn(process.execPath, ['--input-type=module', '-e', harness], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + let child; + try { + child = Number(await waitForFile(pidFile)); + assert.ok(alive(child)); + const exited = exitWithin(tool); + tool.kill('SIGINT'); + assert.deepEqual(await exited, [130, null]); + assert.equal(readFileSync(marker, 'utf8'), 'SIGINT'); + for (let waited = 0; alive(child); waited += 25) { + assert.ok(waited < 5_000, 'the child outlived the forwarded interrupt'); + await delay(25); + } + } finally { + if (tool.exitCode === null && tool.signalCode === null) tool.kill('SIGKILL'); + if (child !== undefined && alive(child)) process.kill(child, 'SIGKILL'); + rmSync(root, { recursive: true, force: true }); + } + }, +); + +test( + 'an interrupt caught during the cleanup after the last child is still recorded', + { skip: process.platform === 'win32' }, + async () => { + // The last await completes from an I/O callback, so the cleanup runs inside the loop's poll + // phase: a listener removed after one more turn would drop the caught signal. + const harness = ` +import { readFile } from 'node:fs/promises'; +import { constants } from 'node:os'; +import { forwardInterrupts } from ${JSON.stringify(new URL('./process.mjs', import.meta.url).href)}; +const interrupts = forwardInterrupts(); +try { + await readFile(${JSON.stringify(fileURLToPath(import.meta.url))}); +} finally { + process.kill(process.pid, 'SIGINT'); + for (const until = Date.now() + 50; Date.now() < until; ); + await interrupts.dispose(); +} +const signal = interrupts.received(); +if (signal !== undefined) process.exit(128 + constants.signals[signal]); +console.log('not interrupted'); +`; + const tool = spawn(process.execPath, ['--input-type=module', '-e', harness], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + tool.stdout.on('data', (chunk) => (stdout += chunk)); + const [code, signal] = await exitWithin(tool); + assert.equal(signal, null); + assert.equal(code, 130); + assert.equal(stdout, ''); + }, +); + +test( + 'a settled child leaves nothing for an interrupt to kill', + { skip: process.platform === 'win32' }, + async (t) => { + // A finished child's process group may be recycled; an interrupt must never signal it. + await runCommand(process.execPath, ['-e', ''], { timeoutMs: 20_000 }); + await assert.rejects( + runCommand(process.execPath, ['-e', 'process.exit(3)'], { timeoutMs: 20_000 }), + /PACKED_CLI_CHILD_FAILED/u, + ); + await assert.rejects( + runCommand(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { timeoutMs: 200 }), + /PACKED_CLI_CHILD_TIMEOUT/u, + ); + const kill = t.mock.method(process, 'kill', () => true); + stopLiveCommands(); + assert.equal(kill.mock.callCount(), 0); + }, +); + +test('the corepack cache is computed as corepack computes it, and isolated children keep it', () => { + assert.equal( + corepackHome({ COREPACK_HOME: '/explicit/corepack' }), + resolve('/explicit/corepack'), + ); + assert.equal( + corepackHome({ XDG_CACHE_HOME: '/xdg', LOCALAPPDATA: '/local' }), + resolve('/xdg', 'node', 'corepack'), + ); + assert.equal(corepackHome({ LOCALAPPDATA: '/local' }), resolve('/local', 'node', 'corepack')); + // A relative location is resolved here, never by a child against its own working directory. + assert.equal(corepackHome({ COREPACK_HOME: 'relative/cache' }), resolve('relative/cache')); + assert.equal( + corepackHome({}), + resolve( + homedir(), + process.platform === 'win32' ? join('AppData', 'Local') : '.cache', + 'node', + 'corepack', + ), + ); + // An isolated child carries this process's corepack cache, never one inside its own home. + const home = join(tmpdir(), 'synthetic-installer-home'); + const env = isolatedEnvironment(home); + assert.equal(env.COREPACK_HOME, corepackHome()); + assert.equal(env.COREPACK_HOME.startsWith(`${home}${sep}`), false); +}); + +test( + 'a corepack-managed pnpm never runs from the temporary directory an isolated home sits in', + { skip: process.platform === 'win32' }, + async () => { + // A module another local user could plant in the shared temporary directory, on the lookup + // path of code that runs from a home inside it. + const temporary = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-corepack-temporary-'))); + try { + const marker = join(temporary, 'planted-module-ran'); + for (const name of ['v8-compile-cache', 'supports-color']) { + mkdirSync(join(temporary, 'node_modules', name), { recursive: true }); + writeFileSync( + join(temporary, 'node_modules', name, 'index.js'), + `require('node:fs').writeFileSync(${JSON.stringify(marker)}, ${JSON.stringify(name)});`, + ); + } + const home = realpathSync(mkdtempSync(join(temporary, 'relavium-package-build-'))); + writeFileSync(join(home, 'package.json'), '{"packageManager":"pnpm@9.12.3"}\n'); + const version = await runPnpm(['--version'], { + cwd: home, + env: isolatedEnvironment(home), + timeoutMs: 120_000, + }); + assert.equal(version.stdout.trim(), '9.12.3'); + assert.equal(existsSync(marker), false, 'a module planted in the temporary directory ran'); + } finally { + rmSync(temporary, { recursive: true, force: true }); + } + }, +); + +test( + 'past its deadline a child is interrupted first, stops its own children and then settles', + { skip: process.platform === 'win32' }, + async () => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-graceful-deadline-'))); + const marker = join(root, 'cleaned'); + const pidFile = join(root, 'grandchild.pid'); + try { + // A child that handles the interrupt as the producer does, around a detached grandchild. + const child = ` +import { writeFileSync } from 'node:fs'; +import { exitOnInterrupt, isolatedEnvironment, runCommand } from ${JSON.stringify(new URL('./process.mjs', import.meta.url).href)}; +await exitOnInterrupt( + () => writeFileSync(${JSON.stringify(marker)}, 'cleaned'), + () => runCommand(process.execPath, ['-e', ${JSON.stringify( + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid)); setInterval(() => {}, 1000);`, + )}], { env: isolatedEnvironment(${JSON.stringify(root)}), timeoutMs: 60_000 }), +); +`; + const started = Date.now(); + await assert.rejects( + runCommand(process.execPath, ['--input-type=module', '-e', child], { + timeoutMs: 3000, + graceMs: 20_000, + }), + /PACKED_CLI_CHILD_TIMEOUT/u, + ); + // It settled when the child exited on the interrupt, long before the grace ran out. + assert.ok(Date.now() - started < 15_000); + assert.equal(readFileSync(marker, 'utf8'), 'cleaned'); + const grandchild = Number(readFileSync(pidFile, 'utf8')); + for (let waited = 0; alive(grandchild); waited += 25) { + assert.ok(waited < 5_000, 'the grandchild outlived the cleanup of its parent'); + await delay(25); + } + } finally { + // A regressed deadline leaves the detached grandchild running: it never outlives the test. + const pid = existsSync(pidFile) ? Number(readFileSync(pidFile, 'utf8')) : NaN; + if (Number.isSafeInteger(pid) && pid > 0 && alive(pid)) process.kill(pid, 'SIGKILL'); + rmSync(root, { recursive: true, force: true }); + } + }, +); + +test( + 'a child that ignores the interrupt is killed once the grace runs out', + { skip: process.platform === 'win32' }, + async (t) => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-ignored-interrupt-'))); + const pidFile = join(root, 'child.pid'); + // A regressed grace leaves the child running: the bound fails the test and this stops it. + t.after(() => { + const pid = existsSync(pidFile) ? Number(readFileSync(pidFile, 'utf8')) : NaN; + if (Number.isSafeInteger(pid) && pid > 0 && alive(pid)) process.kill(pid, 'SIGKILL'); + rmSync(root, { recursive: true, force: true }); + }); + const started = Date.now(); + const settled = runCommand( + process.execPath, + [ + '-e', + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);", + ], + { timeoutMs: 500, graceMs: 1000 }, + ).then( + () => 'resolved', + (error) => error, + ); + const outcome = await Promise.race([ + settled, + delay(30_000, undefined, { ref: false }).then(() => 'still running'), + ]); + assert.ok(outcome instanceof Error, String(outcome)); + assert.match(outcome.message, /PACKED_CLI_CHILD_TIMEOUT/u); + assert.equal(outcome.result.signal, 'SIGKILL'); + assert.ok(Date.now() - started >= 1500); + }, +); + +test('an isolated child of an isolated tool keeps the corepack cache its caller named', async () => { + // check.mjs isolates the producer's HOME and caches in the temporary check root and names the + // developer's cache explicitly; the producer's own children must keep that cache. + const home = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-corepack-inherited-'))); + try { + const script = `import { isolatedEnvironment } from ${JSON.stringify(new URL('./process.mjs', import.meta.url).href)}; +console.log(isolatedEnvironment('/isolated-home').COREPACK_HOME);`; + const result = await runCommand(process.execPath, ['--input-type=module', '-e', script], { + env: { + PATH: process.env.PATH, + HOME: home, + XDG_CACHE_HOME: home, + COREPACK_HOME: '/developer/corepack', + }, + timeoutMs: 20_000, + }); + assert.equal(result.stdout.trim(), resolve('/developer/corepack')); + } finally { + rmSync(home, { recursive: true, force: true }); + } +}); + +/** + * Code for a group leader that starts a member in its own process group (not detached), records the + * member's pid and stays alive until a SIGTERM ends it; the member ignores SIGTERM. + */ +function groupWithStubbornMember(pidFile) { + const member = "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);"; + return ( + `const { spawn } = require('node:child_process');` + + `const member = spawn(process.execPath, ['-e', ${JSON.stringify(member)}], { stdio: 'ignore' });` + + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(member.pid));` + + 'setInterval(() => {}, 1000);' + ); +} + +test( + 'past its deadline the whole group is killed when the leader exits on the interrupt first', + { skip: process.platform === 'win32' }, + async (t) => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-group-deadline-'))); + const pidFile = join(root, 'member.pid'); + t.after(() => { + const pid = existsSync(pidFile) ? Number(readFileSync(pidFile, 'utf8')) : NaN; + if (Number.isSafeInteger(pid) && pid > 0 && alive(pid)) process.kill(pid, 'SIGKILL'); + rmSync(root, { recursive: true, force: true }); + }); + await assert.rejects( + runCommand(process.execPath, ['-e', groupWithStubbornMember(pidFile)], { + timeoutMs: 1000, + graceMs: 4000, + }), + /PACKED_CLI_CHILD_TIMEOUT/u, + ); + // The leader exited on the SIGTERM; its member, which ignored it, goes with the settlement. + const member = Number(readFileSync(pidFile, 'utf8')); + for (let waited = 0; alive(member); waited += 25) { + assert.ok(waited < 2_000, 'a member of the group outlived its deadline'); + await delay(25); + } + }, +); + +test( + 'past its deadline every member of the group is interrupted, not only the leader', + { skip: process.platform === 'win32' }, + async (t) => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-group-interrupt-'))); + const pidFile = join(root, 'member.pid'); + const marker = join(root, 'member-interrupted'); + t.after(() => { + const pid = existsSync(pidFile) ? Number(readFileSync(pidFile, 'utf8')) : NaN; + if (Number.isSafeInteger(pid) && pid > 0 && alive(pid)) process.kill(pid, 'SIGKILL'); + rmSync(root, { recursive: true, force: true }); + }); + const member = + `const fs = require('node:fs'); fs.writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + `process.on('SIGTERM', () => { fs.writeFileSync(${JSON.stringify(marker)}, 'SIGTERM'); process.exit(0); });` + + 'setInterval(() => {}, 1000);'; + // The leader ignores the interrupt, so nothing but the grace's end kills the group. + const leader = + `require('node:child_process').spawn(process.execPath, ['-e', ${JSON.stringify(member)}], { stdio: 'ignore' });` + + "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);"; + await assert.rejects( + runCommand(process.execPath, ['-e', leader], { timeoutMs: 2000, graceMs: 1500 }), + (error) => + /PACKED_CLI_CHILD_TIMEOUT/u.test(error.message) && error.result.signal === 'SIGKILL', + ); + assert.equal(readFileSync(marker, 'utf8'), 'SIGTERM'); + }, +); + +test( + 'without a grace a child past its deadline is killed outright, never interrupted first', + { skip: process.platform === 'win32' }, + async (t) => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-no-grace-'))); + const marker = join(root, 'interrupted'); + t.after(() => rmSync(root, { recursive: true, force: true })); + await assert.rejects( + runCommand( + process.execPath, + [ + '-e', + `process.on('SIGTERM', () => require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'SIGTERM'));` + + 'setInterval(() => {}, 1000);', + ], + { timeoutMs: 500 }, + ), + (error) => + /PACKED_CLI_CHILD_TIMEOUT/u.test(error.message) && error.result.signal === 'SIGKILL', + ); + assert.equal(existsSync(marker), false); + }, +); + +test('the grace is a non-negative whole number of milliseconds', () => { + for (const graceMs of [-1, 1.5, Number.NaN]) + assert.throws(() => runCommand(process.execPath, ['-e', ''], { graceMs }), String(graceMs)); +}); + +test( + 'an interrupt stops a child tree that ignores SIGTERM', + { skip: process.platform === 'win32' }, + async (t) => { + // The tool's children are killed outright: a build that traps SIGTERM cannot outlive it. + const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-stubborn-child-'))); + const pidFile = join(root, 'child.pid'); + t.after(() => { + const pid = existsSync(pidFile) ? Number(readFileSync(pidFile, 'utf8')) : NaN; + if (Number.isSafeInteger(pid) && pid > 0 && alive(pid)) process.kill(pid, 'SIGKILL'); + rmSync(root, { recursive: true, force: true }); + }); + const harness = ` +import { exitOnInterrupt, runCommand } from ${JSON.stringify(new URL('./process.mjs', import.meta.url).href)}; +await exitOnInterrupt(() => {}, () => + runCommand(process.execPath, ['-e', ${JSON.stringify( + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);", + )}], { timeoutMs: 60_000 }), +); +`; + const tool = spawn(process.execPath, ['--input-type=module', '-e', harness], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + const child = Number(await waitForFile(pidFile)); + const exited = exitWithin(tool); + tool.kill('SIGINT'); + assert.deepEqual(await exited, [130, null]); + for (let waited = 0; alive(child); waited += 25) { + assert.ok(waited < 3_000, 'the child outlived the interrupt'); + await delay(25); + } + }, +); + +test('forwarding interrupts adds one listener per signal and disposing removes them', async () => { + const signals = ['SIGINT', 'SIGTERM', 'SIGHUP']; + const counts = () => signals.map((signal) => process.listenerCount(signal)); + const before = counts(); + const interrupts = forwardInterrupts(); + assert.deepEqual( + counts(), + before.map((count) => count + 1), + ); + await interrupts.dispose(); + assert.deepEqual(counts(), before); +}); diff --git a/tools/cli-package/produce.mjs b/tools/cli-package/produce.mjs new file mode 100644 index 00000000..86244c5d --- /dev/null +++ b/tools/cli-package/produce.mjs @@ -0,0 +1,192 @@ +/** + * The package producer (ADR-0111, Wave 3 Step 9a.1): owned candidate copy, fresh mapped dependency + * emit, sole CLI prepack with candidate finalization, independent seal. It never builds in the + * developer's workspace and never calls a cached build. The private maps, metafile and hash record + * are staged owner-only outside the artifact directory and removed when production succeeds, fails or + * is interrupted (SIGINT, SIGTERM, SIGHUP); a producer killed outright, a crash or a lost host can + * leave its owner-only root in the temporary directory, and no secure erasure is claimed. Only the + * tarball and its public receipt are published. `pack.mjs` is its command-line entry point; the ports + * are tooling seams for tests, never shipped product hooks. + */ +import assert from 'node:assert/strict'; +import { + chmodSync, + mkdirSync, + mkdtempSync, + readdirSync, + realpathSync, + rmSync, + writeSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { isAbsolute, join } from 'node:path'; +import { assertNoBuildCache, globalExcludesFile, prepareCandidate } from './candidate.mjs'; +import { within } from './paths.mjs'; +import { exitOnInterrupt, isolatedEnvironment, runPnpm } from './process.mjs'; +import { sealPackage } from './seal.mjs'; +import { captureSupported } from './symbolicate.mjs'; +import { emitFreshPackage } from './pipeline.mjs'; + +/** + * Produce one package-test artifact from the original checkout `repository` into the empty + * `destination`. The candidate tree and its private staging are siblings in one temporary root, + * owner-only where POSIX modes are observable, that shares no path with the artifact directory or the + * checkout. That root and the build home are removed whether production succeeds or fails — the + * staging directory, which holds the private maps, first — each removal attempted even when another + * fails. A failed production keeps its own error, also empties the artifact directory, which was empty + * at the start, and reports a failed removal only as one fixed line. An interrupt (SIGINT, SIGTERM, + * SIGHUP) stops the build's child trees, removes the staging directory, empties the artifact directory + * and only then removes the candidate tree and the build home, before the process exits with the + * signal's code — also when it arrives after the last child exited, during the seal or during the + * final removal (`exitOnInterrupt`). A step of that cleanup that fails leaves one fixed line, never a + * path, and the exit code stays the signal's. + * @param {{repository: string, destination: string, environment: Record, + * temporary?: string, ports?: {prepare?: typeof prepareCandidate, emit?: typeof emitFreshPackage, + * seal?: typeof sealPackage, noBuildCache?: typeof assertNoBuildCache, + * capturable?: typeof captureSupported, + * excludesFile?: typeof globalExcludesFile, pnpm?: typeof runPnpm, + * remove?: (path: string) => void}}} options + */ +export async function producePackage({ + repository, + destination, + environment, + temporary = tmpdir(), + ports = {}, +}) { + const { + prepare = prepareCandidate, + emit = emitFreshPackage, + seal = sealPackage, + noBuildCache = assertNoBuildCache, + capturable = captureSupported, + excludesFile = globalExcludesFile, + pnpm = runPnpm, + // Retries absorb a child killed by the interrupt that was still finishing a write. + remove = (path) => rmSync(path, { recursive: true, force: true, maxRetries: 3 }), + } = ports; + // The seal's symbolication needs Node 22.15 or later; refuse an older producer before anything exists. + assert.ok(capturable(), 'PACKED_CLI_PRODUCER_NODE'); + // The artifact directory is the maintainer's or CI's argument and may be anywhere; it must be + // absolute (pack.mjs resolves it) and empty. No model supplies it, which is what Sonar's S8707 + // taint rule assumes of a command-line argument. + assert.ok(isAbsolute(destination), 'PACKED_CLI_FRESH_DESTINATION_REQUIRED'); + mkdirSync(destination, { recursive: true }); // NOSONAR — a maintainer/CI path, see above + const present = readdirSync(destination); // NOSONAR — a maintainer/CI path, see above + assert.deepEqual(present, [], 'PACKED_CLI_FRESH_DESTINATION_REQUIRED'); + const home = realpathSync.native(mkdtempSync(join(temporary, 'relavium-package-build-'))); + let root; + /** Remove each of `paths`, attempting every one; the first failure, if any. */ + const removeAll = (paths) => { + let failure; + for (const path of paths) { + try { + remove(path); + } catch (error) { + failure ??= error; + } + } + return failure; + }; + const staging = () => (root === undefined ? [] : [join(root, 'staging')]); + const rest = () => (root === undefined ? [home] : [root, home]); + /** Empty the artifact directory, which was empty at the start; the first failure, if any. */ + const emptyArtifact = () => { + let names; + try { + names = readdirSync(destination); + } catch (error) { + return error; + } + return removeAll(names.map((name) => join(destination, name))); + }; + /** One fixed diagnostic line; a failed write cannot replace the outcome it reports. */ + const report = (line) => { + try { + writeSync(2, line); + } catch { + // Nothing else can carry it. + } + }; + // An interrupt's cleanup never throws: a failed step leaves one fixed line, without a path, and + // the producer still exits with the signal's code. The maps go first, then the artifact directory, + // then the large candidate tree and the build home. + const interrupted = () => { + const failures = [removeAll(staging()), emptyArtifact(), removeAll(rest())]; + if (failures.some((failure) => failure !== undefined)) + report('✗ CLI package production interrupted: CLEANUP_FAILED\n'); + }; + return exitOnInterrupt(interrupted, async () => { + let failed = false; + let failure; + let produced; + try { + root = realpathSync.native(mkdtempSync(join(temporary, 'relavium-package-candidate-'))); + chmodSync(root, 0o700); + const candidate = join(root, 'source'); + const [privateStaging] = staging(); + // The symbolication run's home, project and captured frames: owner-only, outside the candidate. + const work = join(root, 'symbolication'); + mkdirSync(candidate, { mode: 0o700 }); + for (const directory of [privateStaging, work]) { + mkdirSync(directory, { mode: 0o700 }); + chmodSync(directory, 0o700); + } + // Native real paths carry the letter case the filesystem stores, so no alias passes for elsewhere. + const published = realpathSync.native(destination); + const checkout = realpathSync.native(repository); + assert.ok( + !within(root, published) && !within(published, root) && !within(checkout, root), + 'PACKED_CLI_PRIVATE_STAGING_PLACEMENT', + ); + const env = isolatedEnvironment(home); + const version = await pnpm(['--version'], { cwd: repository, env }); + assert.equal(version.stdout.trim(), '9.12.3', 'PACKED_CLI_PNPM_VERSION'); + + const excludes = await excludesFile({ repository, environment }); + const identity = await prepare({ repository, candidate, environment, home, env, excludes }); + const receipt = await emit({ + repository: candidate, + destination, + staging: privateStaging, + env, + pnpm, + removeBuildState: (cwd) => { + rmSync(join(cwd, 'dist'), { recursive: true, force: true }); + rmSync(join(cwd, 'tsconfig.build.tsbuildinfo'), { force: true }); + }, + seal: (artifactDirectory) => { + noBuildCache(candidate); + return seal(artifactDirectory, { + repository: candidate, + staging: privateStaging, + source: identity.source, + dirty: identity.dirty, + checkout: repository, + work, + }); + }, + }); + produced = { receipt, identity }; + } catch (error) { + failed = true; + failure = error; + } + // The interrupt handler is still installed: an interrupt during this removal is honoured once it + // is done, never in the middle of it. + const removal = removeAll([...staging(), ...rest()]); + if (failed) { + // A failed production leaves no artifact and keeps its own error. + const emptied = emptyArtifact(); + if (removal !== undefined || emptied !== undefined) + report('✗ CLI package production cleanup failed: CLEANUP_FAILED\n'); + throw failure; + } + if (removal !== undefined) { + // Without its cleanup the production failed: no artifact remains, and the removal's error is it. + emptyArtifact(); + throw removal; + } + return produced; + }); +} diff --git a/tools/cli-package/produce.test.mjs b/tools/cli-package/produce.test.mjs new file mode 100644 index 00000000..b7ebe443 --- /dev/null +++ b/tools/cli-package/produce.test.mjs @@ -0,0 +1,1031 @@ +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import nodeModule from 'node:module'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join, sep } from 'node:path'; +import { after, test } from 'node:test'; +import { setTimeout as delay } from 'node:timers/promises'; +import { assertNoBuildCache } from './candidate.mjs'; +import { exitWithin } from './child-exit.mjs'; +import { isolatedEnvironment } from './process.mjs'; +import { producePackage } from './produce.mjs'; +import { productionFailure } from './production-failure.mjs'; + +const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-produce-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); +const posixModes = process.platform !== 'win32'; +let count = 0; + +function fresh(name) { + count += 1; + const path = join(scratch, `${name}-${count}`); + mkdirSync(path); + return path; +} + +const SHA = 'c'.repeat(40); + +/** + * Fake ports that record what the producer composes. The fake emit writes into the candidate it is + * given, runs the producer's seal wrapper and can plant a build-cache marker first. + */ +function ports({ plantCache = false, failEmit = false, dirty = false } = {}) { + const seen = {}; + return { + seen, + ports: { + pnpm: async (args, options) => { + assert.deepEqual(args, ['--version']); + seen.pnpm = options; + return { stdout: '9.12.3\n', stderr: '' }; + }, + excludesFile: async ({ repository, environment }) => { + seen.excludesRepository = repository; + seen.excludesEnvironment = environment; + return '/developer/excludes'; + }, + prepare: async (options) => { + seen.prepare = options; + return { source: SHA, dirty, files: 7 }; + }, + emit: async (options) => { + seen.emit = options; + seen.stagingMode = statSync(options.staging).mode & 0o777; + seen.rootMode = statSync(join(options.staging, '..')).mode & 0o777; + mkdirSync(join(options.repository, 'apps', 'cli', 'dist'), { recursive: true }); + // Build state an engine package could carry; the emit removes it before each compile. + const core = join(options.repository, 'packages', 'core'); + mkdirSync(join(core, 'dist'), { recursive: true }); + writeFileSync(join(core, 'dist', 'stale.js'), 'stale\n'); + writeFileSync(join(core, 'tsconfig.build.tsbuildinfo'), '{}\n'); + await options.removeBuildState(core); + seen.buildStateLeft = ['dist', 'tsconfig.build.tsbuildinfo'].filter((name) => + existsSync(join(core, name)), + ); + if (plantCache) writeFileSync(join(options.repository, '.turbo'), 'cache\n'); + if (failEmit) throw new Error('synthetic emit failure'); + writeFileSync(join(options.destination, 'relavium-0.1.1.tgz'), 'tarball'); + return options.seal(options.destination); + }, + noBuildCache: (candidate) => { + seen.noBuildCache = candidate; + assertNoBuildCache(candidate); + }, + seal: async (artifactDirectory, candidate) => { + seen.seal = { artifactDirectory, ...candidate }; + seen.workMode = statSync(candidate.work).mode & 0o777; + return { mode: 'package_test', source: candidate.source, files: [] }; + }, + }, + }; +} + +function leftovers(temporary) { + return readdirSync(temporary).filter((name) => name.startsWith('relavium-package-')); +} + +const INTERRUPTS = ['SIGINT', 'SIGTERM', 'SIGHUP']; +const listeners = () => INTERRUPTS.map((signal) => process.listenerCount(signal)); + +/** Credentials, a preload and a registry in the caller's own environment, which no build may see. */ +const PLANTED = Object.freeze({ + NPM_TOKEN: 'planted-npm-token', + NODE_OPTIONS: '--require=/planted/preload.cjs', + npm_config_registry: 'https://registry.planted.invalid/', +}); + +test('the producer builds and seals only the owned candidate, never the original checkout', async (t) => { + const repository = fresh('original-checkout'); + const destination = join(fresh('artifact-parent'), 'artifact'); + const temporary = fresh('tmp'); + const { seen, ports: fakes } = ports(); + const before = listeners(); + const saved = Object.fromEntries(Object.keys(PLANTED).map((key) => [key, process.env[key]])); + t.after(() => { + for (const [key, value] of Object.entries(saved)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + }); + Object.assign(process.env, PLANTED); + const environment = { HOME: '/home/someone', NPM_TOKEN: 'environment-npm-token' }; + const result = await producePackage({ + repository, + destination, + environment, + temporary, + ports: fakes, + }); + // The developer's global excludes and dependency store are looked up in the caller's own + // environment, never in the isolated build environment. + assert.equal(seen.excludesEnvironment, environment); + assert.equal(seen.prepare.environment, environment); + assert.deepEqual(result.identity, { source: SHA, dirty: false, files: 7 }); + // The whole build — pnpm, the candidate's preparation and the emit with its pack — runs in the + // isolated environment of the private build home, never with the caller's own variables. + const home = seen.prepare.home; + assert.equal(dirname(home), temporary); + assert.match(basename(home), /^relavium-package-build-/u); + assert.deepEqual(seen.prepare.env, isolatedEnvironment(home)); + assert.deepEqual(seen.emit.env, seen.prepare.env); + assert.deepEqual(seen.pnpm.env, seen.prepare.env); + // pnpm's version is checked in the original checkout, and the emit gets that same pnpm port. + assert.equal(seen.pnpm.cwd, repository); + assert.equal(seen.emit.pnpm, fakes.pnpm); + for (const key of Object.keys(PLANTED)) assert.equal(key in seen.prepare.env, false); + const candidate = seen.prepare.candidate; + // The candidate and its staging are siblings in one private root outside the checkout and the artifact. + assert.notEqual(candidate, repository); + assert.equal(seen.prepare.repository, repository); + assert.equal(seen.prepare.excludes, '/developer/excludes'); + assert.equal(seen.excludesRepository, repository); + assert.equal(seen.emit.repository, candidate); + assert.equal(seen.emit.destination, destination); + assert.equal(seen.emit.staging, join(candidate, '..', 'staging')); + assert.equal(seen.noBuildCache, candidate); + assert.deepEqual(seen.buildStateLeft, []); + // The seal symbolicates against the original checkout's objects, in a private work directory + // beside the candidate and its staging. + assert.deepEqual(seen.seal, { + artifactDirectory: destination, + repository: candidate, + staging: seen.emit.staging, + source: SHA, + dirty: false, + checkout: repository, + work: join(candidate, '..', 'symbolication'), + }); + if (posixModes) assert.equal(seen.workMode, 0o700); + if (posixModes) { + assert.equal(seen.stagingMode, 0o700); + assert.equal(seen.rootMode, 0o700); + } + // The private root and the build home are gone, and only the tarball remains in the artifact. + assert.deepEqual(leftovers(temporary), []); + // The interrupt handlers are gone too. + assert.deepEqual(listeners(), before); + assert.equal(existsSync(candidate), false); + assert.deepEqual(readdirSync(destination), ['relavium-0.1.1.tgz']); +}); + +test('without a named temporary directory the producer works in the system one', async (t) => { + const repository = fresh('checkout-system-temporary'); + const destination = join(fresh('artifact-parent'), 'artifact'); + const temporary = fresh('system-tmp'); + const names = ['TMPDIR', 'TMP', 'TEMP']; + const saved = names.map((name) => [name, process.env[name]]); + t.after(() => { + for (const [name, value] of saved) + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + }); + for (const name of names) process.env[name] = temporary; + const { seen, ports: fakes } = ports(); + await producePackage({ repository, destination, environment: {}, ports: fakes }); + assert.equal(dirname(seen.prepare.home), temporary); + assert.equal(dirname(dirname(seen.prepare.candidate)), temporary); + assert.deepEqual(leftovers(temporary), []); +}); + +test('another pnpm than the pinned one refuses before anything is prepared', async () => { + const temporary = fresh('tmp'); + const { seen, ports: fakes } = ports(); + fakes.pnpm = async () => ({ stdout: '9.12.4\n', stderr: '' }); + await assert.rejects( + producePackage({ + repository: fresh('original-checkout'), + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary, + ports: fakes, + }), + /PACKED_CLI_PNPM_VERSION/u, + ); + assert.equal(seen.prepare, undefined); + assert.deepEqual(leftovers(temporary), []); +}); + +test('a producer Node that cannot run the capture refuses before anything exists', async () => { + const temporary = fresh('tmp'); + const destination = join(fresh('artifact-parent'), 'artifact'); + const { seen, ports: fakes } = ports(); + fakes.capturable = () => false; + await assert.rejects( + producePackage({ + repository: fresh('original-checkout'), + destination, + environment: {}, + temporary, + ports: fakes, + }), + { message: 'PACKED_CLI_PRODUCER_NODE' }, + ); + assert.equal(seen.prepare, undefined); + assert.equal(existsSync(destination), false); + assert.deepEqual(leftovers(temporary), []); + // By default the producer asks this Node itself. + const registerHooks = nodeModule.registerHooks; + delete nodeModule.registerHooks; + try { + const { ports: defaults } = ports(); + delete defaults.capturable; + await assert.rejects( + producePackage({ + repository: fresh('original-checkout'), + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary, + ports: defaults, + }), + { message: 'PACKED_CLI_PRODUCER_NODE' }, + ); + } finally { + nodeModule.registerHooks = registerHooks; + } +}); + +test( + 'the private root and staging are exactly 0700 under a umask without the owner write bit', + { skip: !posixModes }, + async () => { + const seen = {}; + // The test's own directories exist before the umask applies. + const repository = fresh('original-checkout'); + const destination = join(fresh('artifact-parent'), 'artifact'); + const temporary = fresh('tmp'); + const previous = process.umask(0o277); + try { + await producePackage({ + repository, + destination, + environment: {}, + temporary, + ports: { + pnpm: async () => ({ stdout: '9.12.3\n', stderr: '' }), + excludesFile: async () => undefined, + prepare: async () => ({ source: SHA, dirty: false, files: 1 }), + emit: async ({ staging, destination, seal }) => { + seen.staging = statSync(staging).mode & 0o777; + seen.root = statSync(join(staging, '..')).mode & 0o777; + return seal(destination); + }, + noBuildCache: () => {}, + seal: () => ({ mode: 'package_test' }), + }, + }); + } finally { + process.umask(previous); + } + assert.deepEqual(seen, { staging: 0o700, root: 0o700 }); + assert.deepEqual(leftovers(temporary), []); + }, +); + +test('a dirty source identity reaches the seal and the returned identity', async () => { + const { seen, ports: fakes } = ports({ dirty: true }); + const result = await producePackage({ + repository: fresh('original-checkout'), + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary: fresh('tmp'), + ports: fakes, + }); + assert.equal(seen.seal.dirty, true); + assert.equal(result.identity.dirty, true); +}); + +test('a build cache left in the candidate refuses before the seal, and nothing private remains', async () => { + const temporary = fresh('tmp'); + const destination = join(fresh('artifact-parent'), 'artifact'); + const { seen, ports: fakes } = ports({ plantCache: true }); + await assert.rejects( + producePackage({ + repository: fresh('original-checkout'), + destination, + environment: {}, + temporary, + ports: fakes, + }), + /CANDIDATE_BUILD_CACHE/u, + ); + assert.equal(seen.seal, undefined); + assert.deepEqual(leftovers(temporary), []); + assert.deepEqual(readdirSync(destination), []); +}); + +test('a failure inside the build still removes the private root', async () => { + const temporary = fresh('tmp'); + const { ports: fakes } = ports({ failEmit: true }); + const before = listeners(); + await assert.rejects( + producePackage({ + repository: fresh('original-checkout'), + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary, + ports: fakes, + }), + /synthetic emit failure/u, + ); + assert.deepEqual(leftovers(temporary), []); + assert.deepEqual(listeners(), before); +}); + +test('private staging never shares a path with the checkout or the artifact', async () => { + // A temporary directory inside the original checkout would put the private root there. + const repository = fresh('original-checkout'); + const inside = join(repository, 'tmp'); + mkdirSync(inside); + const { seen, ports: fakes } = ports(); + await assert.rejects( + producePackage({ + repository, + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary: inside, + ports: fakes, + }), + /PRIVATE_STAGING_PLACEMENT/u, + ); + assert.equal(seen.prepare, undefined); + assert.deepEqual(leftovers(inside), []); + // A directory whose name merely starts with two dots is inside the checkout too. + const dotted = join(repository, '..tmp'); + mkdirSync(dotted); + await assert.rejects( + producePackage({ + repository, + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary: dotted, + ports: ports().ports, + }), + /PRIVATE_STAGING_PLACEMENT/u, + ); + assert.deepEqual(leftovers(dotted), []); + // On a case-insensitive filesystem, another letter case of the checkout is the checkout. + const variant = join(dirname(repository), basename(repository).toUpperCase()); + if (existsSync(variant)) { + await assert.rejects( + producePackage({ + repository, + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary: join(variant, 'tmp'), + ports: ports().ports, + }), + /PRIVATE_STAGING_PLACEMENT/u, + ); + assert.deepEqual(leftovers(inside), []); + // The same holds when the checkout itself is named in another letter case. + const { seen: cased, ports: casedFakes } = ports(); + await assert.rejects( + producePackage({ + repository: variant, + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary: inside, + ports: casedFakes, + }), + /PRIVATE_STAGING_PLACEMENT/u, + ); + assert.equal(cased.prepare, undefined); + assert.deepEqual(leftovers(inside), []); + } + // A link to the checkout or to the artifact directory is that directory too. + const linkKind = process.platform === 'win32' ? 'junction' : 'dir'; + const checkoutAlias = join(fresh('alias-parent'), 'checkout'); + symlinkSync(repository, checkoutAlias, linkKind); + await assert.rejects( + producePackage({ + repository: checkoutAlias, + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary: inside, + ports: ports().ports, + }), + /PRIVATE_STAGING_PLACEMENT/u, + ); + const realArtifact = fresh('real-artifact'); + const artifactAlias = join(fresh('alias-parent'), 'artifact'); + symlinkSync(realArtifact, artifactAlias, linkKind); + await assert.rejects( + producePackage({ + repository, + destination: artifactAlias, + environment: {}, + temporary: realArtifact, + ports: ports().ports, + }), + /PRIVATE_STAGING_PLACEMENT/u, + ); + assert.deepEqual(readdirSync(realArtifact), []); + // On a case-insensitive filesystem, another letter case of the artifact directory is that directory. + const casedArtifact = fresh('cased-artifact'); + const casedVariant = join(dirname(casedArtifact), basename(casedArtifact).toUpperCase()); + if (existsSync(casedVariant)) { + await assert.rejects( + producePackage({ + repository, + destination: casedVariant, + environment: {}, + temporary: casedArtifact, + ports: ports().ports, + }), + /PRIVATE_STAGING_PLACEMENT/u, + ); + assert.deepEqual(readdirSync(casedArtifact), []); + } + // So would a temporary directory inside the artifact directory. + const artifact = join(fresh('artifact-parent'), 'artifact'); + mkdirSync(artifact); + const { seen: inArtifact, ports: artifactFakes } = ports(); + await assert.rejects( + producePackage({ + repository, + destination: artifact, + environment: {}, + temporary: artifact, + ports: artifactFakes, + }), + /PRIVATE_STAGING_PLACEMENT/u, + ); + assert.equal(inArtifact.prepare, undefined); + assert.deepEqual(readdirSync(artifact), []); + // A non-empty artifact directory refuses before anything private exists. + const temporary = fresh('tmp'); + const destination = fresh('artifact'); + writeFileSync(join(destination, 'stale.tgz'), 'old'); + await assert.rejects( + producePackage({ repository, destination, environment: {}, temporary, ports: fakes }), + /FRESH_DESTINATION_REQUIRED/u, + ); + assert.deepEqual(leftovers(temporary), []); +}); + +/** A producer run whose build is a long-lived child, as `pnpm pack` is, behind fake ports. */ +function interruptedProducer(temporary, destination, pidFile, grandchildFile, removalLog) { + const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); + const script = ` +import { appendFileSync, mkdirSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { producePackage } from ${url('./produce.mjs')}; +import { isolatedEnvironment, runCommand } from ${url('./process.mjs')}; +await producePackage({ + repository: ${JSON.stringify(join(temporary, 'checkout'))}, + destination: ${JSON.stringify(destination)}, + environment: {}, + temporary: ${JSON.stringify(temporary)}, + ports: { + pnpm: async () => ({ stdout: '9.12.3\\n', stderr: '' }), + excludesFile: async () => undefined, + prepare: async () => ({ source: '${SHA}', dirty: false, files: 1 }), + emit: async ({ staging, destination }) => { + writeFileSync(join(staging, 'index.js.map'), 'private map'); + writeFileSync(join(destination, 'relavium-0.1.1.tgz'), 'unsealed'); + mkdirSync(join(destination, 'nested')); + writeFileSync(join(destination, 'nested', 'part'), 'part'); + await runCommand(process.execPath, ['-e', ${JSON.stringify( + `const { spawn } = require('node:child_process');` + + `const { writeFileSync } = require('node:fs');` + + // A grandchild in the same process group, as tsup is under pnpm pack and prepack. + `spawn(process.execPath, ['-e', ${JSON.stringify( + `require('node:fs').writeFileSync(${JSON.stringify(grandchildFile)}, String(process.pid)); setInterval(() => {}, 1000);`, + )}], { stdio: 'ignore' });` + + `writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + `setInterval(() => {}, 1000);`, + )}], { env: isolatedEnvironment(${JSON.stringify(temporary)}), timeoutMs: 60_000 }); + }, + // Records each removal and how many entries the artifact directory still had at that moment. + remove: (path) => { + const entry = path.startsWith(${JSON.stringify(destination)}); + const kind = entry ? 'artifact' : path.endsWith('staging') ? 'staging' : path.includes('relavium-package-candidate-') ? 'candidate' : 'build'; + appendFileSync(${JSON.stringify(removalLog)}, (entry ? kind : kind + ':' + readdirSync(${JSON.stringify(destination)}).length) + '\\n'); + rmSync(path, { recursive: true, force: true }); + }, + }, +}); +`; + return spawn(process.execPath, ['--input-type=module', '-e', script], { + stdio: ['ignore', 'pipe', 'pipe'], + }); +} + +function alive(pid) { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +for (const [signal, code] of [ + ['SIGINT', 130], + ['SIGTERM', 143], + ['SIGHUP', 129], +]) { + test( + `an interrupted producer (${signal}) stops its build and removes everything private`, + { skip: !posixModes }, + async () => { + const temporary = fresh('tmp'); + mkdirSync(join(temporary, 'checkout')); + const destination = join(fresh('artifact-parent'), 'artifact'); + const pids = fresh('pid'); + const pidFile = join(pids, 'sleeper.pid'); + const grandchildFile = join(pids, 'grandchild.pid'); + const removalLog = join(pids, 'removals'); + const child = interruptedProducer( + temporary, + destination, + pidFile, + grandchildFile, + removalLog, + ); + let stderr = ''; + child.stderr.on('data', (chunk) => (stderr += chunk)); + const tree = []; + try { + for (const file of [pidFile, grandchildFile]) { + for ( + let waited = 0; + !existsSync(file) || readFileSync(file, 'utf8') === ''; + waited += 25 + ) { + assert.ok(waited < 20_000, 'the fake build never started'); + await delay(25); + } + tree.push(Number(readFileSync(file, 'utf8'))); + } + assert.ok(tree.every(alive)); + const exited = exitWithin(child); + child.kill(signal); + assert.deepEqual(await exited, [code, null]); + // The whole build tree is stopped, not only the process the producer started. + for (let waited = 0; tree.some(alive); waited += 25) { + assert.ok(waited < 5_000, 'part of the build tree outlived the interrupt'); + await delay(25); + } + assert.deepEqual(leftovers(temporary), []); + assert.deepEqual(readdirSync(destination), []); + // The maps go first, then the artifact directory (its nested entry too), then the large + // candidate tree and the build home; a clean interrupt prints nothing. + assert.equal( + readFileSync(removalLog, 'utf8'), + 'staging:2\nartifact\nartifact\ncandidate:0\nbuild:0\n', + ); + assert.equal(stderr, ''); + } finally { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + for (const pid of tree) if (alive(pid)) process.kill(pid, 'SIGKILL'); + } + }, + ); +} + +test('a failed removal still removes the other private directory and rethrows', async () => { + const temporary = fresh('tmp'); + const destination = join(fresh('artifact-parent'), 'artifact'); + const { ports: fakes } = ports(); + const refusal = Object.assign(new Error('EACCES: synthetic refusal'), { code: 'EACCES' }); + fakes.remove = (path) => { + if (path.includes('relavium-package-candidate-') && !path.endsWith('staging')) throw refusal; + rmSync(path, { recursive: true, force: true }); + }; + const before = listeners(); + await assert.rejects( + producePackage({ + repository: fresh('original-checkout'), + destination, + environment: {}, + temporary, + ports: fakes, + }), + (error) => error === refusal, + ); + // The build home is gone although the private root's removal failed first, and without its + // cleanup the production leaves no artifact. + assert.deepEqual( + leftovers(temporary).map((name) => name.replace(/-[^-]+$/u, '')), + ['relavium-package-candidate'], + ); + assert.deepEqual(readdirSync(destination), []); + assert.deepEqual(listeners(), before); +}); + +test('every removal is attempted and the first failure is the one rethrown', async () => { + const temporary = fresh('tmp'); + const destination = join(fresh('artifact-parent'), 'artifact'); + const { ports: fakes } = ports(); + const first = new Error('synthetic staging refusal'); + const second = new Error('synthetic candidate refusal'); + const removed = []; + const kind = (path) => { + if (path.startsWith(destination)) return 'artifact'; + if (path.endsWith('staging')) return 'staging'; + return path.includes('relavium-package-candidate-') ? 'candidate' : 'build'; + }; + fakes.remove = (path) => { + removed.push(kind(path)); + if (kind(path) === 'staging') throw first; + if (kind(path) === 'candidate') throw second; + rmSync(path, { recursive: true, force: true }); + }; + await assert.rejects( + producePackage({ + repository: fresh('original-checkout'), + destination, + environment: {}, + temporary, + ports: fakes, + }), + (error) => error === first, + ); + // The staging directory, which holds the private maps, goes first; without its cleanup the + // production then leaves no artifact. + assert.deepEqual(removed, ['staging', 'candidate', 'build', 'artifact']); +}); + +test('a failed production keeps its own error and leaves no artifact', async () => { + const temporary = fresh('tmp'); + const destination = join(fresh('artifact-parent'), 'artifact'); + const { ports: fakes } = ports(); + const refusal = new Error('CLI package archive refused: EXTRA_FILE'); + fakes.seal = async () => { + throw refusal; + }; + await assert.rejects( + producePackage({ + repository: fresh('original-checkout'), + destination, + environment: {}, + temporary, + ports: fakes, + }), + (error) => error === refusal, + ); + // The fake emit wrote the tarball before the seal refused it. + assert.deepEqual(readdirSync(destination), []); + assert.deepEqual(leftovers(temporary), []); +}); + +for (const [label, { cleanupFails, closeStderr }, stderr] of [ + [ + 'whose cleanup also fails keeps its own error and adds one fixed line', + { cleanupFails: true, closeStderr: false }, + '✗ CLI package production cleanup failed: CLEANUP_FAILED\n', + ], + ['whose cleanup succeeds keeps its own error and adds no line', { cleanupFails: false }, ''], + // A line that cannot be written never replaces the error it would have reported. + [ + 'keeps its own error even when the cleanup line cannot be written', + { cleanupFails: true, closeStderr: true }, + '', + ], +]) { + test(`a failed production ${label}`, { skip: !posixModes }, () => { + const temporary = fresh('tmp'); + mkdirSync(join(temporary, 'checkout')); + const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); + const script = ` +import { closeSync, rmSync } from 'node:fs'; +import { producePackage } from ${url('./produce.mjs')}; +if (${closeStderr === true}) closeSync(2); +try { + await producePackage({ + repository: ${JSON.stringify(join(temporary, 'checkout'))}, + destination: ${JSON.stringify(join(fresh('artifact-parent'), 'artifact'))}, + environment: {}, + temporary: ${JSON.stringify(temporary)}, + ports: { + pnpm: async () => ({ stdout: '9.12.3\\n', stderr: '' }), + excludesFile: async () => undefined, + prepare: async () => { throw new Error('PACKED_CLI_SOURCE_CHANGED_DURING_COPY'); }, + remove: (path) => { + if (${cleanupFails} && path.includes('relavium-package-candidate-')) throw Object.assign(new Error('EBUSY: ' + path), { code: 'EBUSY' }); + rmSync(path, { recursive: true, force: true }); + }, + }, + }); +} catch (error) { + console.log('REJECTED:' + error.message); +} +`; + const result = spawnSync(process.execPath, ['--input-type=module', '-e', script], { + encoding: 'utf8', + timeout: 20_000, + }); + assert.equal(result.stdout, 'REJECTED:PACKED_CLI_SOURCE_CHANGED_DURING_COPY\n'); + assert.equal(result.stderr, stderr); + }); +} + +test( + 'a failed removal during an interrupt leaves one fixed line and still exits with the signal', + { skip: !posixModes }, + async () => { + const temporary = fresh('tmp'); + mkdirSync(join(temporary, 'checkout')); + const destination = join(fresh('artifact-parent'), 'artifact'); + const pidFile = join(fresh('pid'), 'build.pid'); + const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); + const script = ` +import { rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { producePackage } from ${url('./produce.mjs')}; +import { isolatedEnvironment, runCommand } from ${url('./process.mjs')}; +await producePackage({ + repository: ${JSON.stringify(join(temporary, 'checkout'))}, + destination: ${JSON.stringify(destination)}, + environment: {}, + temporary: ${JSON.stringify(temporary)}, + ports: { + pnpm: async () => ({ stdout: '9.12.3\\n', stderr: '' }), + excludesFile: async () => undefined, + prepare: async () => ({ source: '${SHA}', dirty: false, files: 1 }), + emit: async ({ destination }) => { + writeFileSync(join(destination, 'relavium-0.1.1.tgz'), 'unsealed'); + await runCommand(process.execPath, ['-e', ${JSON.stringify( + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid)); setInterval(() => {}, 1000);`, + )}], { env: isolatedEnvironment(${JSON.stringify(temporary)}), timeoutMs: 60_000 }); + }, + remove: (path) => { + if (path.includes('relavium-package-candidate-') && !path.endsWith('staging')) + throw Object.assign(new Error('EACCES: permission denied, unlink ' + JSON.stringify(path)), { code: 'EACCES' }); + rmSync(path, { recursive: true, force: true }); + }, + }, +}); +`; + const child = spawn(process.execPath, ['--input-type=module', '-e', script], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stderr = ''; + child.stderr.on('data', (chunk) => (stderr += chunk)); + let build; + try { + for ( + let waited = 0; + !existsSync(pidFile) || readFileSync(pidFile, 'utf8') === ''; + waited += 25 + ) { + assert.ok(waited < 20_000, 'the fake build never started'); + await delay(25); + } + build = Number(readFileSync(pidFile, 'utf8')); + const exited = exitWithin(child); + child.kill('SIGINT'); + assert.deepEqual(await exited, [130, null]); + assert.equal(stderr, '✗ CLI package production interrupted: CLEANUP_FAILED\n'); + // The build was killed; its exit is reaped a moment later. + for (let waited = 0; alive(build); waited += 25) { + assert.ok(waited < 5_000, 'the build outlived the interrupt'); + await delay(25); + } + // The other steps still ran: the build home is gone and the artifact directory is empty. + assert.deepEqual( + leftovers(temporary).map((name) => name.replace(/-[^-]+$/u, '')), + ['relavium-package-candidate'], + ); + assert.deepEqual(readdirSync(destination), []); + } finally { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + if (build !== undefined && alive(build)) process.kill(build, 'SIGKILL'); + } + }, +); + +/** + * A producer whose interrupt arrives after its build's last child exited: during the seal or during + * the private root's removal, both synchronous. The fake emit's last await completes from an I/O + * callback, as pnpm pack's exit does, so the producer's tail runs inside the event loop's poll phase. + */ +function lateInterrupt(temporary, destination, during) { + const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); + const script = ` +import { readFile } from 'node:fs/promises'; +import { rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { producePackage } from ${url('./produce.mjs')}; +const interrupt = () => { + process.kill(process.pid, 'SIGINT'); + // The rest of the synchronous work, during which the signal is only caught. + for (const until = Date.now() + 50; Date.now() < until; ); +}; +await producePackage({ + repository: ${JSON.stringify(join(temporary, 'checkout'))}, + destination: ${JSON.stringify(destination)}, + environment: {}, + temporary: ${JSON.stringify(temporary)}, + ports: { + pnpm: async () => ({ stdout: '9.12.3\\n', stderr: '' }), + excludesFile: async () => undefined, + prepare: async () => ({ source: '${SHA}', dirty: false, files: 1 }), + emit: async ({ staging, destination, seal }) => { + writeFileSync(join(staging, 'index.js.map'), 'private map'); + writeFileSync(join(destination, 'relavium-0.1.1.tgz'), 'tarball'); + await readFile(join(staging, 'index.js.map')); + return seal(destination); + }, + noBuildCache: () => {}, + seal: () => { + if (${JSON.stringify(during)} === 'seal') interrupt(); + return { mode: 'package_test' }; + }, + remove: (path) => { + if (${JSON.stringify(during)} === 'removal' && path.includes('relavium-package-candidate-')) + interrupt(); + rmSync(path, { recursive: true, force: true }); + }, + }, +}); +console.log('produced'); +`; + return spawn(process.execPath, ['--input-type=module', '-e', script], { + stdio: ['ignore', 'pipe', 'pipe'], + }); +} + +for (const during of ['seal', 'removal']) + test( + `an interrupt during the ${during}, after the last child exited, still ends the producer`, + { skip: !posixModes }, + async () => { + const temporary = fresh('tmp'); + mkdirSync(join(temporary, 'checkout')); + const destination = join(fresh('artifact-parent'), 'artifact'); + const child = lateInterrupt(temporary, destination, during); + let stdout = ''; + child.stdout.on('data', (chunk) => (stdout += chunk)); + try { + const [code, signal] = await exitWithin(child); + // Honoured once the synchronous work is done: the removal completes, the artifact is + // emptied and the producer exits with the signal's code instead of reporting success. + assert.equal(signal, null); + assert.equal(code, 130); + assert.equal(stdout, ''); + assert.deepEqual(leftovers(temporary), []); + assert.deepEqual(readdirSync(destination), []); + } finally { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + } + }, + ); + +/** Output a failed build child could carry: a pnpm lifecycle header and esbuild quoting a map. */ +const QUOTED_MAP = 'zzMAPMARKERzz'; +const REFUSAL = + '✗ CLI candidate finalization refused: UPSTREAM_MAP_SHAPE (packages/core/dist/index.js.map)'; +function failedChild() { + const error = new Error('PACKED_CLI_CHILD_FAILED: 1'); + error.result = { + stdout: `> relavium@0.1.1 prepack /private/candidate/apps/cli\n1 │ {"names":["${QUOTED_MAP}"],"mappings":"AAAA;!!"}\n`, + stderr: `${REFUSAL}\n✗ CLI ${QUOTED_MAP} │ {"names":[]}\n${QUOTED_MAP}\n`, + }; + return error; +} + +test('a failed production prints only prepack diagnostics and a fixed code, never child output', () => { + assert.deepEqual(productionFailure(failedChild()), [ + REFUSAL, + '✗ CLI package production failed: PACKED_CLI_CHILD_FAILED', + ]); + // A system error gives its code, never its message with a host path; anything else is unexpected. + const missing = Object.assign(new Error("ENOENT: no such file, open '/private/x'"), { + code: 'ENOENT', + }); + assert.deepEqual(productionFailure(missing), ['✗ CLI package production failed: ENOENT']); + for (const other of [new Error('a message with /private/paths'), 'thrown text', undefined]) + assert.deepEqual(productionFailure(other), ['✗ CLI package production failed: UNEXPECTED']); +}); + +test('the producer entry point prints only the fixed failure lines and exits 1', () => { + // The real entry point and failure report, with a producer stand-in that fails as a build child. + const directory = fresh('entry'); + for (const name of ['pack.mjs', 'production-failure.mjs']) + writeFileSync(join(directory, name), readFileSync(new URL(name, import.meta.url))); + writeFileSync( + join(directory, 'produce.mjs'), + `export async function producePackage() {\n` + + ` const error = new Error('PACKED_CLI_CHILD_FAILED: 1');\n` + + ` error.result = ${JSON.stringify(failedChild().result)};\n` + + ` throw error;\n}\n`, + ); + const result = spawnSync( + process.execPath, + [join(directory, 'pack.mjs'), join(directory, 'out')], + { + encoding: 'utf8', + timeout: 20_000, + }, + ); + assert.equal(result.status, 1); + assert.equal(result.stdout, ''); + assert.equal( + result.stderr, + `${REFUSAL}\n✗ CLI package production failed: PACKED_CLI_CHILD_FAILED\n`, + ); + assert.equal(result.stderr.includes(QUOTED_MAP), false); +}); + +test('a refused manifest or symbolication prints its own fixed code', async () => { + const { ManifestError } = await import('./manifest.mjs'); + const { SymbolicationError } = await import('./symbolicate.mjs'); + assert.deepEqual(productionFailure(new SymbolicationError('ANCHOR_UNMAPPED')), [ + '✗ CLI package production failed: PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', + ]); + assert.deepEqual(productionFailure(new ManifestError('MAP_DRIFT')), [ + '✗ CLI package production failed: PACKED_CLI_MANIFEST_MAP_DRIFT', + ]); +}); + +test('the failure report bounds what it forwards and forwards nothing but anchored diagnostics', () => { + const failed = (stdout, stderr) => + Object.assign(new Error('PACKED_CLI_CHILD_FAILED: 1'), { result: { stdout, stderr } }); + const code = '✗ CLI package production failed: PACKED_CLI_CHILD_FAILED'; + // At most 240 printable characters after the prefix. + const longest = `✗ CLI ${'a'.repeat(240)}`; + assert.deepEqual(productionFailure(failed('', `${longest}\n✗ CLI ${'a'.repeat(241)}\n`)), [ + longest, + code, + ]); + // A line that merely ends in the prefix's text, after quoted map content, is not a diagnostic. + assert.deepEqual( + productionFailure(failed('', `1 | {"names":["${QUOTED_MAP}"],"x":"✗ CLI tail"}\n`)), + [code], + ); + // At most eight diagnostics, from stdout as well as stderr, trimmed. + const twelve = Array.from({ length: 12 }, (_, index) => `✗ CLI line ${index}`); + assert.deepEqual(productionFailure(failed(` ${twelve.join('\n')}`, '')), [ + ...twelve.slice(0, 8), + code, + ]); + // A system code must look like one; anything else is unexpected. + for (const value of ['/private/host/path', 'ERR with space', 1, 'lower']) + assert.deepEqual(productionFailure(Object.assign(new Error('x'), { code: value })), [ + '✗ CLI package production failed: UNEXPECTED', + ]); +}); + +test('the producer entry point hands the checkout, the artifact directory and its own environment over', () => { + const root = fresh('entry-wiring'); + const tools = join(root, 'tools', 'cli-package'); + mkdirSync(tools, { recursive: true }); + for (const name of ['pack.mjs', 'production-failure.mjs']) + writeFileSync(join(tools, name), readFileSync(new URL(name, import.meta.url))); + writeFileSync( + join(tools, 'produce.mjs'), + `import { writeFileSync } from 'node:fs';\n` + + `export async function producePackage({ repository, destination, environment }) {\n` + + ` writeFileSync(process.env.RELAVIUM_ENTRY_MARKER, JSON.stringify({ repository, destination, marker: environment.RELAVIUM_ENTRY_MARKER, same: environment === process.env }));\n` + + ` return { receipt: { mode: 'package_test', source: 's', tarballSha256: 'h', files: [1, 2] }, identity: { files: 3 } };\n` + + `}\n`, + ); + const cwd = fresh('entry-cwd'); + const marker = join(fresh('entry-marker'), 'called'); + const env = { PATH: process.env.PATH, RELAVIUM_ENTRY_MARKER: marker }; + const run = (args) => + spawnSync(process.execPath, [join(tools, 'pack.mjs'), ...args], { + cwd, + env, + encoding: 'utf8', + timeout: 20_000, + }); + const result = run(['out']); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(readFileSync(marker, 'utf8')), { + repository: `${root}${sep}`, + destination: join(cwd, 'out'), + marker, + same: true, + }); + assert.deepEqual(JSON.parse(result.stdout), { + mode: 'package_test', + source: 's', + sha256: 'h', + files: 2, + candidateFiles: 3, + privateMaps: 'staged privately, then discarded', + privateManifest: 'sealed privately, then discarded', + symbolication: 'actual packed CLI and core frames verified', + }); + // Exactly one artifact directory: any other count refuses before the producer runs. + rmSync(marker); + for (const args of [[], ['one', 'two']]) { + assert.notEqual(run(args).status, 0, String(args.length)); + assert.equal(existsSync(marker), false); + } +}); diff --git a/tools/cli-package/production-failure.mjs b/tools/cli-package/production-failure.mjs new file mode 100644 index 00000000..531bfec3 --- /dev/null +++ b/tools/cli-package/production-failure.mjs @@ -0,0 +1,35 @@ +/** + * What a failed package production may print (ADR-0111). A build child's captured output can quote a + * map — esbuild excerpts a malformed upstream map in its warning — and maps are excluded from every + * diagnostic, so that output is never printed. Only prepack's own fixed `✗ CLI …` lines, which carry + * a code and at most a repository-relative path, and one fixed code for the failure itself remain. + */ + +/** A prepack diagnostic line: its fixed prefix followed by printable ASCII only. */ +const PREPACK_DIAGNOSTIC = /^✗ CLI [ -~]{1,240}$/u; +const PRINTED_DIAGNOSTICS = 8; + +/** + * The lines a failed production prints, in order: the prepack diagnostics found in the failed child's + * captured output, then `✗ CLI package production failed: `, where the code is the error's + * leading `PACKED_CLI_*` token, else its system error code, else `UNEXPECTED`. + * @param {unknown} error + * @returns {string[]} + */ +export function productionFailure(error) { + const result = + error !== null && typeof error === 'object' && 'result' in error ? error.result : undefined; + const captured = [result?.stdout, result?.stderr].filter((text) => typeof text === 'string'); + const diagnostics = captured + .flatMap((text) => text.split(/\r?\n/u)) + .map((line) => line.trim()) + .filter((line) => PREPACK_DIAGNOSTIC.test(line)) + .slice(0, PRINTED_DIAGNOSTICS); + const message = error instanceof Error ? error.message : ''; + const system = + error !== null && typeof error === 'object' && 'code' in error ? String(error.code) : ''; + const code = + /^PACKED_CLI_[A-Z0-9_]+/u.exec(message)?.[0] ?? + (/^[A-Z][A-Z0-9_]{1,63}$/u.test(system) ? system : 'UNEXPECTED'); + return [...diagnostics, `✗ CLI package production failed: ${code}`]; +} diff --git a/tools/cli-package/release-binding.mjs b/tools/cli-package/release-binding.mjs new file mode 100644 index 00000000..b01d4824 --- /dev/null +++ b/tools/cli-package/release-binding.mjs @@ -0,0 +1,374 @@ +/** + * The release binding receipt (Step 9b.2; packaging.md, "Recipient custody implementation boundary — + * Step 9b.2"): schema v1, its validator and the verified reuse of a retained candidate. The receipt is + * public JSON beside the ciphertext, written once in canonical bytes. It names files by SHA-256 and signs + * nothing: every field is a claim, and its origin and map-check fields are checked for shape only until + * Step 9d binds them to the trusted run and a successful `ci` run. Scope always comes from the verifier's + * expectation, never from the receipt. This module's import closure starts no process, so reuse makes no + * GnuPG call. + */ +import { Buffer } from 'node:buffer'; +import { lstatSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { isDeepStrictEqual, TextDecoder } from 'node:util'; +import { ARCHIVE_LIMITS } from './archive.mjs'; +import { CustodyError, refuseCustody } from './custody-error.mjs'; +import { readOrdinaryFile, sha256, byCodeUnit } from './files.mjs'; +import { isProductionRecipient, isSyntheticSource } from './release-recipient.mjs'; + +/** The receipt's file name beside the ciphertext, as the interim release guard keys on it. */ +export const RELEASE_BINDING_FILE = 'release-binding-receipt.json'; +export const RELEASE_BINDING_KIND = 'release_binding'; +/** The custody container's 128 MiB total plus the 64 KiB a ciphertext may add to it (9b.1's bounds). */ +export const CIPHERTEXT_BYTES = 128 * 1024 * 1024 + 64 * 1024; +const RECEIPT_BYTES = 64 * 1024; +const MAX_INPUTS = 64; + +/** @param {string} version */ +export const tarballName = (version) => `relavium-${version}.tgz`; +/** @param {string} version */ +export const ciphertextFileName = (version) => `relavium-${version}-private-maps.tar.gpg`; + +const KEYS = Object.freeze([ + 'version', + 'kind', + 'packageVersion', + 'source', + 'tarball', + 'ciphertext', + 'recipient', + 'toolchain', + 'target', + 'inputs', + 'mapCheck', + 'origin', +]); +const SHA256 = /^[0-9a-f]{64}$/u; +const GIT_OBJECT = /^[0-9a-f]{40}$/u; +const FINGERPRINT = /^[0-9A-F]{40}$/u; +const PACKAGE_VERSION = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/u; +const TOOL_VERSION = /^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]{1,64})?$/u; +const SEGMENT = /^[A-Za-z0-9_.-]+$/u; + +/** + * @typedef {import('./release-recipient.mjs').ProductionRecipient} ProductionRecipient + * @typedef {{scope: 'production', recipient: ProductionRecipient, packageVersion: string, source: string} + * | {scope: 'synthetic', pins: {primary: string, subkey: string}, packageVersion: string}} Expected + */ + +/** @param {unknown} value @returns {value is Record} */ +const isRecord = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); + +/** @param {unknown} value @param {readonly string[]} keys */ +function exact(value, keys) { + if ( + !isRecord(value) || + !isDeepStrictEqual(Object.keys(value).sort(byCodeUnit), [...keys].sort(byCodeUnit)) + ) { + refuseCustody('BINDING_SHAPE'); + } + return /** @type {Record} */ (value); +} +/** @param {unknown} value @param {RegExp} pattern @param {number} [max] @returns {string} */ +function string(value, pattern, max = 256) { + if (typeof value !== 'string' || value.length > max || !pattern.test(value)) { + refuseCustody('BINDING_SHAPE'); + } + return /** @type {string} */ (value); +} +/** @param {unknown} value @param {number} minimum @param {number} maximum @returns {number} */ +function size(value, minimum, maximum) { + if (typeof value !== 'number' || !Number.isSafeInteger(value)) refuseCustody('BINDING_SHAPE'); + if (value < minimum || value > maximum) refuseCustody('BINDING_SIZE'); + return value; +} +/** + * A repository-relative POSIX path with no empty, `.` or `..` segment, and not an integer-like name, which + * JavaScript orders ahead of every other key so that the canonical bytes could not list it by path. + * @param {string} path + */ +const isRelativePath = (path) => + path.length <= 256 && + !/^(?:0|[1-9]\d*)$/u.test(path) && + path.split('/').every((segment) => SEGMENT.test(segment) && segment !== '.' && segment !== '..'); + +/** + * A snapshot of the expectation, each field read once: the admitted recipient pairs, the package version + * and, for production, the source commit. + * @param {unknown} expected + * @returns {{scope: 'production' | 'synthetic', history: readonly {primary: unknown, subkey: unknown}[], + * packageVersion: unknown, source: unknown}} + */ +function scopeOf(expected) { + if (!isRecord(expected)) return refuseCustody('BINDING_SCOPE'); + const { scope, recipient, pins, packageVersion, source } = expected; + if (scope === 'production') { + if (!isProductionRecipient(recipient)) refuseCustody('RECIPIENT_SOURCE'); + const { history } = /** @type {ProductionRecipient} */ (recipient); + return Object.freeze({ scope, history, packageVersion, source }); + } + if (scope !== 'synthetic') refuseCustody('BINDING_SCOPE'); + const pair = isRecord(pins) ? { primary: pins.primary, subkey: pins.subkey } : {}; + return Object.freeze({ + scope, + history: Object.freeze([Object.freeze({ primary: pair.primary, subkey: pair.subkey })]), + packageVersion, + source: undefined, + }); +} + +/** + * @param {unknown} value @param {string} version @param {(version: string) => string} name + * @param {number} minimum @param {number} maximum + */ +function asset(value, version, name, minimum, maximum) { + // Each field is read once: the value checked is the value returned. + const { name: given, bytes, sha256: digest } = exact(value, ['name', 'bytes', 'sha256']); + if (given !== name(version)) refuseCustody('BINDING_NAME'); + return Object.freeze({ + name: /** @type {string} */ (given), + bytes: size(bytes, minimum, maximum), + sha256: string(digest, SHA256), + }); +} + +/** + * Validate a release binding receipt against the verifier's expectation and return its normalized, + * frozen copy in canonical key order. `expected` has no default: a production expectation needs the + * loaded production recipient, whose whole history verifies; a synthetic one needs the synthetic pins. + * @param {unknown} value + * @param {Expected} expected + */ +export function validateReleaseBinding(value, expected) { + return bounded(() => validate(value, scopeOf(expected))); +} + +/** + * Run `work` behind the fixed-code boundary: a host error's message, or a throwing accessor's, never + * leaves; only custody codes do. + * @template T @param {() => T} work @returns {T} + */ +function bounded(work) { + try { + return work(); + } catch (error) { + if (error instanceof CustodyError) throw error; + return refuseCustody('UNEXPECTED'); + } +} + +/** @param {unknown} value @param {ReturnType} scope */ +function validate(value, scope) { + if (!isRecord(value) || value.version !== 1 || value.kind !== RELEASE_BINDING_KIND) { + refuseCustody('BINDING_VERSION'); + } + const receipt = exact(value, KEYS); + const packageVersion = string(receipt.packageVersion, PACKAGE_VERSION, 100); + const { commit, tree, dirty } = exact(receipt.source, ['commit', 'tree', 'dirty']); + if (typeof dirty !== 'boolean') refuseCustody('BINDING_SHAPE'); + const source = Object.freeze({ + commit: string(commit, GIT_OBJECT), + tree: string(tree, GIT_OBJECT), + dirty, + }); + if (packageVersion !== scope.packageVersion) refuseCustody('BINDING_CANDIDATE'); + if (scope.scope === 'production') { + if (source.commit !== scope.source) refuseCustody('BINDING_CANDIDATE'); + if (isSyntheticSource(source)) refuseCustody('BINDING_SCOPE'); + if (source.dirty) refuseCustody('BINDING_DIRTY'); + } else if (!isSyntheticSource(source)) { + refuseCustody('BINDING_SCOPE'); + } + + // A gzip member is at least 18 bytes (archive.mjs); a ciphertext at least one. + const tarball = asset( + receipt.tarball, + packageVersion, + tarballName, + 18, + ARCHIVE_LIMITS.compressedBytes, + ); + const ciphertext = asset( + receipt.ciphertext, + packageVersion, + ciphertextFileName, + 1, + CIPHERTEXT_BYTES, + ); + + const recipientRecord = exact(receipt.recipient, ['primary', 'subkey']); + const recipient = Object.freeze({ + primary: string(recipientRecord.primary, FINGERPRINT), + subkey: string(recipientRecord.subkey, FINGERPRINT), + }); + if ( + !scope.history.some( + (entry) => entry.primary === recipient.primary && entry.subkey === recipient.subkey, + ) + ) { + refuseCustody('BINDING_RECIPIENT'); + } + + const tools = exact(receipt.toolchain, [ + 'node', + 'pnpm', + 'typescript', + 'tsup', + 'esbuild', + 'gnupg', + ]); + const toolchain = Object.freeze({ + node: string(tools.node, /^v\d+\.\d+\.\d+$/u, 40), + pnpm: string(tools.pnpm, TOOL_VERSION, 100), + typescript: string(tools.typescript, TOOL_VERSION, 100), + tsup: string(tools.tsup, TOOL_VERSION, 100), + esbuild: string(tools.esbuild, TOOL_VERSION, 100), + gnupg: string(tools.gnupg, /^\d+\.\d+\.\d+$/u, 40), + }); + const target = string(receipt.target, /^node\d{2,3}$/u, 8); + + const { inputs: inputsRecord, origin: originValue } = receipt; + if (!isRecord(inputsRecord)) refuseCustody('BINDING_SHAPE'); + const entries = Object.entries(inputsRecord); + if (entries.length < 1 || entries.length > MAX_INPUTS) refuseCustody('BINDING_SHAPE'); + const inputs = Object.freeze( + Object.fromEntries( + entries + .map(([path, digest]) => { + if (!isRelativePath(path)) refuseCustody('BINDING_SHAPE'); + return [path, string(digest, SHA256)]; + }) + .sort(([left], [right]) => (left < right ? -1 : 1)), + ), + ); + + const mapCheck = exact(receipt.mapCheck, ['status']); + if (mapCheck.status !== 'complete') refuseCustody('BINDING_MAP_CHECK'); + + if (!isRecord(originValue) || originValue.kind !== 'github') refuseCustody('BINDING_ORIGIN'); + const originRecord = exact(originValue, [ + 'kind', + 'repository', + 'workflow', + 'runId', + 'runAttempt', + 'event', + 'source', + ]); + if (originRecord.event !== 'push' || originRecord.source !== source.commit) { + refuseCustody('BINDING_ORIGIN'); + } + const origin = Object.freeze({ + kind: 'github', + repository: string(originRecord.repository, /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u, 200), + workflow: string(originRecord.workflow, /^\P{Cc}+$/u, 200), + runId: string(originRecord.runId, /^[1-9]\d*$/u, 20), + runAttempt: string(originRecord.runAttempt, /^[1-9]\d*$/u, 6), + event: 'push', + source: source.commit, + }); + + return Object.freeze({ + version: 1, + kind: RELEASE_BINDING_KIND, + packageVersion, + source, + tarball, + ciphertext, + recipient, + toolchain, + target, + inputs, + mapCheck: Object.freeze({ status: 'complete' }), + origin, + }); +} + +/** The canonical bytes of a normalized receipt; the sealer writes exactly these, once. @param {object} receipt */ +export const serializeReleaseBinding = (receipt) => + Buffer.from(`${JSON.stringify(receipt, null, 2)}\n`); + +/** + * Parse a receipt's exact bytes: anything but the canonical serialization of its validated value + * refuses, so a duplicated key, a reordering, a BOM or trailing data never reaches a verifier. + * @param {Buffer} bytes @param {Expected} expected + */ +export function parseReleaseBinding(bytes, expected) { + return bounded(() => parse(bytes, scopeOf(expected))); +} + +/** @param {Buffer} bytes @param {ReturnType} scope */ +function parse(bytes, scope) { + if (!Buffer.isBuffer(bytes) || bytes.length > RECEIPT_BYTES) refuseCustody('BINDING_BYTES'); + let value; + try { + value = JSON.parse(new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(bytes)); + } catch { + return refuseCustody('BINDING_BYTES'); + } + const receipt = validate(value, scope); + if (!serializeReleaseBinding(receipt).equals(bytes)) refuseCustody('BINDING_BYTES'); + return receipt; +} + +/** + * Verify a retained candidate for reuse: the directory holds exactly its receipt, tarball and ciphertext, + * the receipt is valid for the expectation, and each asset has the receipt's size and SHA-256. Returns the + * verified bytes, never a path to open again; writes, renames and removes nothing. + * @param {{directory: string, expected: Expected}} options + */ +export function verifyRetainedCandidate(options) { + return bounded(() => { + const { directory, expected } = isRecord(options) ? options : {}; + return verify(directory, expected); + }); +} + +/** @param {string} directory @param {Expected} expected */ +function verify(directory, expected) { + const scope = scopeOf(expected); + const { packageVersion } = scope; + if (typeof packageVersion !== 'string' || !PACKAGE_VERSION.test(packageVersion)) { + refuseCustody('BINDING_CANDIDATE'); + } + const names = [ + RELEASE_BINDING_FILE, + tarballName(packageVersion), + ciphertextFileName(packageVersion), + ].sort(byCodeUnit); + if (!isDeepStrictEqual(readdirSync(directory).sort(byCodeUnit), names)) + refuseCustody('CANDIDATE_ENTRIES'); + let bytes; + try { + bytes = readOrdinaryFile(join(directory, RELEASE_BINDING_FILE), RECEIPT_BYTES); + } catch { + refuseCustody('CANDIDATE_ASSET'); + } + const receipt = parse(bytes, scope); + /** @param {{name: string, bytes: number, sha256: string}} entry */ + const retained = (entry) => { + const path = join(directory, entry.name); + let info; + try { + info = lstatSync(path); + } catch { + return refuseCustody('CANDIDATE_ASSET'); + } + if (!info.isFile()) refuseCustody('CANDIDATE_ASSET'); + if (info.size !== entry.bytes) refuseCustody('CANDIDATE_SIZE'); + let data; + try { + data = readOrdinaryFile(path, entry.bytes); + } catch { + return refuseCustody('CANDIDATE_ASSET'); + } + if (sha256(data) !== entry.sha256) refuseCustody('CANDIDATE_DIGEST'); + return data; + }; + return Object.freeze({ + receipt, + receiptBytes: bytes, + tarball: retained(receipt.tarball), + ciphertext: retained(receipt.ciphertext), + }); +} diff --git a/tools/cli-package/release-binding.test.mjs b/tools/cli-package/release-binding.test.mjs new file mode 100644 index 00000000..904f8559 --- /dev/null +++ b/tools/cli-package/release-binding.test.mjs @@ -0,0 +1,956 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { Buffer } from 'node:buffer'; +import { + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + readlinkSync, + renameSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { createRequire, syncBuiltinESMExports } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { after, test } from 'node:test'; +import { ARCHIVE_LIMITS } from './archive.mjs'; +import { isPackageVersion } from './binding.mjs'; +import { CustodyError } from './custody-error.mjs'; +import { parsePins } from './custody.mjs'; +import { sha256 } from './files.mjs'; +import { + CIPHERTEXT_BYTES, + RELEASE_BINDING_FILE, + RELEASE_BINDING_KIND, + ciphertextFileName, + parseReleaseBinding, + serializeReleaseBinding, + tarballName, + validateReleaseBinding, + verifyRetainedCandidate, +} from './release-binding.mjs'; +import { SYNTHETIC_SOURCE, loadRecipientForTests } from './release-recipient.mjs'; + +const scratch = mkdtempSync(join(tmpdir(), 'relavium-release-binding-')); +after(() => rmSync(scratch, { recursive: true, force: true })); +let count = 0; +const fresh = (name) => join(scratch, `${name}-${(count += 1)}`); +const refuses = (code) => (error) => + error instanceof CustodyError && + error.code === code && + /^PACKED_CLI_CUSTODY_[A-Z_]+$/u.test(error.message) && + !error.message.includes(scratch); +const posix = process.platform !== 'win32'; + +const VERSION = '0.1.1'; +const TARBALL = Buffer.from('a synthetic tarball, never a package\n'); +const CIPHERTEXT = Buffer.from('synthetic ciphertext bytes'); +const PINS = Object.freeze({ primary: '4'.repeat(40), subkey: '5'.repeat(40) }); +const A = { primary: '1'.repeat(40), subkey: '2'.repeat(40) }; +const B = { primary: '1'.repeat(40), subkey: '3'.repeat(40) }; +const C = { primary: '6'.repeat(40), subkey: '7'.repeat(40) }; +const CLEAN = { commit: 'd'.repeat(40), tree: 'e'.repeat(40), dirty: false }; +const SYNTHETIC = { ...SYNTHETIC_SOURCE, dirty: true }; + +/** A loaded production recipient with the given history, from a temporary root. */ +function production(...recipients) { + const root = fresh('recipient'); + mkdirSync(root); + writeFileSync(join(root, 'recipient.asc'), 'key'); + writeFileSync( + join(root, 'pins.json'), + `${JSON.stringify({ version: 1, recipients }, null, 2)}\n`, + ); + return loadRecipientForTests({ root }); +} +const ROTATED = production({ ...A, status: 'retired' }, { ...B, status: 'current' }); +const synthetic = Object.freeze({ scope: 'synthetic', pins: PINS, packageVersion: VERSION }); +const productionExpected = (recipient = ROTATED) => + Object.freeze({ + scope: 'production', + recipient, + packageVersion: VERSION, + source: CLEAN.commit, + }); + +/** A receipt in canonical key order; synthetic unless told otherwise. */ +function receipt({ + source = SYNTHETIC, + recipient = PINS, + tarball = TARBALL, + ciphertext = CIPHERTEXT, +} = {}) { + return { + version: 1, + kind: RELEASE_BINDING_KIND, + packageVersion: VERSION, + source: { ...source }, + tarball: { name: tarballName(VERSION), bytes: tarball.length, sha256: sha256(tarball) }, + ciphertext: { + name: ciphertextFileName(VERSION), + bytes: ciphertext.length, + sha256: sha256(ciphertext), + }, + recipient: { ...recipient }, + toolchain: { + node: 'v22.23.1', + pnpm: '9.12.3', + typescript: '5.6.3', + tsup: '8.3.5', + esbuild: '0.24.0', + gnupg: '2.5.21', + }, + target: 'node22', + inputs: { 'apps/cli/package.json': 'a'.repeat(64), 'package.json': 'b'.repeat(64) }, + mapCheck: { status: 'complete' }, + origin: { + kind: 'github', + repository: 'example/relavium', + workflow: 'release', + runId: '1', + runAttempt: '1', + event: 'push', + source: source.commit, + }, + }; +} +const productionReceipt = (recipient = B) => receipt({ source: CLEAN, recipient }); +/** A deep copy changed by `change`. */ +const changed = (value, change) => { + const copy = globalThis.structuredClone(value); + change(copy); + return copy; +}; + +test('a valid receipt validates to its frozen canonical copy, and its canonical bytes parse', () => { + for (const [value, expected] of [ + [receipt(), synthetic], + [productionReceipt(), productionExpected()], + ]) { + const normalized = validateReleaseBinding(value, expected); + assert.deepEqual(normalized, value); + assert.notEqual(normalized, value); + assert.deepEqual(Object.keys(normalized), Object.keys(value)); + const frozen = (object) => + Object.isFrozen(object) && + Object.values(object).every((child) => typeof child !== 'object' || frozen(child)); + assert.ok(frozen(normalized)); + const bytes = serializeReleaseBinding(normalized); + assert.equal(bytes.toString('utf8'), `${JSON.stringify(value, null, 2)}\n`); + assert.deepEqual(parseReleaseBinding(bytes, expected), normalized); + } + // Inputs come back sorted by path. + const unsorted = changed(receipt(), (copy) => { + copy.inputs = { 'package.json': 'b'.repeat(64), 'apps/cli/package.json': 'a'.repeat(64) }; + }); + assert.deepEqual(Object.keys(validateReleaseBinding(unsorted, synthetic).inputs), [ + 'apps/cli/package.json', + 'package.json', + ]); +}); + +test('every key at every level is exact', () => { + const value = receipt(); + for (const key of Object.keys(value)) { + const code = key === 'version' || key === 'kind' ? 'BINDING_VERSION' : 'BINDING_SHAPE'; + assert.throws( + () => + validateReleaseBinding( + changed(value, (copy) => delete copy[key]), + synthetic, + ), + refuses(code), + key, + ); + } + assert.throws( + () => validateReleaseBinding({ ...value, extra: true }, synthetic), + refuses('BINDING_SHAPE'), + ); + for (const parent of [ + 'source', + 'tarball', + 'ciphertext', + 'recipient', + 'toolchain', + 'mapCheck', + 'origin', + ]) { + for (const key of Object.keys(value[parent])) { + // An origin without its kind is not a GitHub origin. + const code = parent === 'origin' && key === 'kind' ? 'BINDING_ORIGIN' : 'BINDING_SHAPE'; + assert.throws( + () => + validateReleaseBinding( + changed(value, (copy) => delete copy[parent][key]), + synthetic, + ), + refuses(code), + `${parent}.${key}`, + ); + } + assert.throws( + () => + validateReleaseBinding( + changed(value, (copy) => (copy[parent].extra = 'x')), + synthetic, + ), + refuses('BINDING_SHAPE'), + `${parent}.extra`, + ); + for (const replacement of [null, [], 'x']) { + const code = parent === 'origin' ? 'BINDING_ORIGIN' : 'BINDING_SHAPE'; + assert.throws( + () => + validateReleaseBinding( + changed(value, (copy) => (copy[parent] = replacement)), + synthetic, + ), + refuses(code), + `${parent} = ${JSON.stringify(replacement)}`, + ); + } + } + for (const inputs of [null, [], 'x']) { + assert.throws( + () => + validateReleaseBinding( + changed(value, (copy) => (copy.inputs = inputs)), + synthetic, + ), + refuses('BINDING_SHAPE'), + ); + } +}); + +test('every value has its exact format', () => { + const value = receipt(); + const cases = [ + (copy) => (copy.packageVersion = 'v0.1.1'), + (copy) => (copy.packageVersion = `0.1.1-${'a'.repeat(100)}`), + (copy) => (copy.source.commit = copy.source.commit.toUpperCase()), + (copy) => (copy.source.tree = 'b'.repeat(39)), + (copy) => (copy.source.dirty = 'true'), + (copy) => (copy.tarball.sha256 = copy.tarball.sha256.toUpperCase()), + (copy) => (copy.ciphertext.sha256 = 'z'.repeat(64)), + (copy) => (copy.tarball.bytes = 18.5), + (copy) => (copy.ciphertext.bytes = '26'), + (copy) => (copy.recipient.primary = copy.recipient.primary.toLowerCase().replace(/4/gu, 'a')), + (copy) => (copy.recipient.subkey = `${copy.recipient.subkey.slice(0, 39)}!`), + (copy) => (copy.toolchain.node = '22.23.1'), + (copy) => (copy.toolchain.pnpm = 'latest'), + (copy) => (copy.toolchain.typescript = '5.6'), + (copy) => (copy.toolchain.tsup = 8), + (copy) => (copy.toolchain.esbuild = '0.24.0-'), + (copy) => (copy.toolchain.gnupg = '2.5.21-beta'), + (copy) => (copy.target = 'node2'), + (copy) => (copy.target = 'deno22'), + (copy) => (copy.inputs = {}), + (copy) => + (copy.inputs = Object.fromEntries( + Array.from({ length: 65 }, (_, index) => [`file-${index}.json`, 'a'.repeat(64)]), + )), + ...['../package.json', './package.json', 'apps//cli', '/package.json', 'apps\\cli', 'a b'].map( + (path) => (copy) => (copy.inputs = { [path]: 'a'.repeat(64) }), + ), + (copy) => (copy.inputs = { [`${'a'.repeat(257)}`]: 'a'.repeat(64) }), + (copy) => (copy.inputs = { 'package.json': 'A'.repeat(64) }), + (copy) => (copy.origin.repository = 'relavium'), + (copy) => (copy.origin.workflow = 'release\u0007'), + (copy) => (copy.origin.runId = '0'), + (copy) => (copy.origin.runAttempt = '01'), + (copy) => (copy.origin.runAttempt = '1234567'), + (copy) => (copy.origin.repository = `${'a'.repeat(100)}/${'b'.repeat(100)}`), + // Every pattern is anchored at both ends: a prefix before a valid value refuses. + (copy) => (copy.source.commit = `x${copy.source.commit}`), + (copy) => (copy.source.tree = `x${copy.source.tree}`), + (copy) => (copy.tarball.sha256 = `x${copy.tarball.sha256}`), + (copy) => (copy.recipient.primary = `X${copy.recipient.primary}`), + (copy) => (copy.toolchain.node = `x${copy.toolchain.node}`), + (copy) => (copy.toolchain.typescript = `x${copy.toolchain.typescript}`), + (copy) => (copy.toolchain.gnupg = `x${copy.toolchain.gnupg}`), + (copy) => (copy.target = `x${copy.target}`), + (copy) => (copy.origin.repository = ` ${copy.origin.repository}`), + (copy) => (copy.origin.workflow = `\u0007${copy.origin.workflow}`), + (copy) => (copy.origin.runId = `x${copy.origin.runId}`), + (copy) => (copy.origin.runAttempt = `x${copy.origin.runAttempt}`), + (copy) => (copy.inputs = { 'package.json': `x${'a'.repeat(64)}` }), + // ...and at the end: a suffix after a valid value refuses. + (copy) => (copy.source.commit = `${copy.source.commit}x`), + (copy) => (copy.source.tree = `${copy.source.tree}x`), + (copy) => (copy.tarball.sha256 = `${copy.tarball.sha256}a`), + (copy) => (copy.recipient.subkey = `${copy.recipient.subkey}A`), + (copy) => (copy.toolchain.node = `${copy.toolchain.node}x`), + (copy) => (copy.toolchain.typescript = `${copy.toolchain.typescript} `), + (copy) => (copy.toolchain.gnupg = `${copy.toolchain.gnupg}x`), + (copy) => (copy.target = `${copy.target}x`), + (copy) => (copy.origin.repository = `${copy.origin.repository} x`), + (copy) => (copy.origin.workflow = `${copy.origin.workflow}\u0007`), + (copy) => (copy.origin.runId = `${copy.origin.runId}x`), + (copy) => (copy.origin.runAttempt = `${copy.origin.runAttempt}x`), + (copy) => (copy.inputs = { 'package.json': `${'a'.repeat(64)}a` }), + // The toolchain's length bounds: one past each refuses. + (copy) => (copy.toolchain.node = `v1.1.${'1'.repeat(36)}`), + ...['pnpm', 'typescript', 'tsup', 'esbuild'].map( + (tool) => (copy) => (copy.toolchain[tool] = `1.1.${'1'.repeat(97)}`), + ), + (copy) => (copy.toolchain.gnupg = `1.1.${'1'.repeat(37)}`), + // A value whose string form matches is still not a string. + (copy) => (copy.toolchain.node = [copy.toolchain.node]), + (copy) => (copy.source.commit = [copy.source.commit]), + (copy) => (copy.origin.workflow = 'w'.repeat(201)), + (copy) => (copy.origin.runId = '1'.repeat(21)), + // Integer-like names would sort ahead of every path in the canonical bytes. + ...['0', '9', '10', '4294967295'].map( + (path) => (copy) => + (copy.inputs = { [path]: 'a'.repeat(64), 'package.json': 'b'.repeat(64) }), + ), + ]; + for (const [index, change] of cases.entries()) { + assert.throws( + () => validateReleaseBinding(changed(value, change), synthetic), + refuses('BINDING_SHAPE'), + `case ${index}`, + ); + } + // At the bounds: 64 inputs and a 256-character path are admitted. + const bounded = changed(value, (copy) => { + copy.inputs = Object.fromEntries( + Array.from({ length: 64 }, (_, index) => [ + index === 0 ? 'a'.repeat(256) : `file-${String(index).padStart(2, '0')}.json`, + 'a'.repeat(64), + ]), + ); + }); + assert.equal(Object.keys(validateReleaseBinding(bounded, synthetic).inputs).length, 64); + const single = changed(value, (copy) => (copy.inputs = { 'package.json': 'a'.repeat(64) })); + // The origin's bounds are inclusive, and a numeric-looking path with a leading zero is a name. + const origins = changed(value, (copy) => { + copy.origin.repository = `${'a'.repeat(99)}/${'b'.repeat(100)}`; + copy.origin.workflow = 'w'.repeat(200); + copy.origin.runId = '1'.repeat(20); + copy.origin.runAttempt = '123456'; + copy.inputs = { '01': 'a'.repeat(64), 'package.json': 'b'.repeat(64) }; + }); + assert.deepEqual(validateReleaseBinding(origins, synthetic).origin, origins.origin); + // The toolchain's length bounds are inclusive. + const tools = changed(value, (copy) => { + copy.toolchain.node = `v1.1.${'1'.repeat(35)}`; + for (const tool of ['pnpm', 'typescript', 'tsup', 'esbuild']) { + copy.toolchain[tool] = `1.1.${'1'.repeat(96)}`; + } + copy.toolchain.gnupg = `1.1.${'1'.repeat(36)}`; + }); + assert.deepEqual(validateReleaseBinding(tools, synthetic).toolchain, tools.toolchain); + assert.deepEqual(validateReleaseBinding(single, synthetic).inputs, { + 'package.json': 'a'.repeat(64), + }); +}); + +test('the version, candidate, names, sizes, map check and origin refuse with their own codes', () => { + const value = receipt(); + for (const [code, change] of [ + ['BINDING_VERSION', (copy) => (copy.version = 2)], + ['BINDING_VERSION', (copy) => (copy.kind = 'package_test')], + ['BINDING_CANDIDATE', (copy) => (copy.packageVersion = '0.1.2')], + ['BINDING_NAME', (copy) => (copy.tarball.name = 'relavium-0.1.2.tgz')], + ['BINDING_NAME', (copy) => (copy.ciphertext.name = 'relavium-0.1.1.tgz.gpg')], + ['BINDING_SIZE', (copy) => (copy.tarball.bytes = 17)], + ['BINDING_SIZE', (copy) => (copy.tarball.bytes = ARCHIVE_LIMITS.compressedBytes + 1)], + ['BINDING_SIZE', (copy) => (copy.ciphertext.bytes = 0)], + ['BINDING_SIZE', (copy) => (copy.ciphertext.bytes = CIPHERTEXT_BYTES + 1)], + ['BINDING_MAP_CHECK', (copy) => (copy.mapCheck.status = 'pending')], + ['BINDING_ORIGIN', (copy) => (copy.origin = { kind: 'local' })], + ['BINDING_ORIGIN', (copy) => (copy.origin.event = 'pull_request')], + ['BINDING_ORIGIN', (copy) => (copy.origin.event = 'workflow_dispatch')], + ['BINDING_ORIGIN', (copy) => (copy.origin.source = 'f'.repeat(40))], + ['BINDING_RECIPIENT', (copy) => (copy.recipient = { ...C })], + ['BINDING_RECIPIENT', (copy) => (copy.recipient.subkey = C.subkey)], + // Both halves of the pair must match: the expected subkey under another primary refuses. + ['BINDING_RECIPIENT', (copy) => (copy.recipient.primary = C.primary)], + ['BINDING_SCOPE', (copy) => (copy.source = { ...CLEAN })], + ['BINDING_SCOPE', (copy) => (copy.source.tree = 'e'.repeat(40))], + ]) { + assert.throws( + () => validateReleaseBinding(changed(value, change), synthetic), + refuses(code), + code, + ); + } + for (const bad of [null, [], 'receipt', 1]) { + assert.throws(() => validateReleaseBinding(bad, synthetic), refuses('BINDING_VERSION')); + } + // The sizes are inclusive bounds. + for (const change of [ + (copy) => (copy.tarball.bytes = 18), + (copy) => (copy.tarball.bytes = ARCHIVE_LIMITS.compressedBytes), + (copy) => (copy.ciphertext.bytes = 1), + (copy) => (copy.ciphertext.bytes = CIPHERTEXT_BYTES), + ]) { + validateReleaseBinding(changed(value, change), synthetic); + } + assert.equal(CIPHERTEXT_BYTES, 128 * 1024 * 1024 + 64 * 1024); +}); + +test('production scope admits only a clean, non-synthetic source of the expected commit and a recipient from the history', () => { + const value = productionReceipt(); + for (const [code, change] of [ + ['BINDING_CANDIDATE', (copy) => (copy.source.commit = 'f'.repeat(40))], + ['BINDING_DIRTY', (copy) => (copy.source.dirty = true)], + ['BINDING_RECIPIENT', (copy) => (copy.recipient = { ...C })], + ['BINDING_RECIPIENT', (copy) => (copy.recipient = { primary: A.primary, subkey: C.subkey })], + ['BINDING_RECIPIENT', (copy) => (copy.recipient = { primary: C.primary, subkey: B.subkey })], + ]) { + assert.throws( + () => + validateReleaseBinding( + changed(value, (copy) => { + change(copy); + copy.origin.source = copy.source.commit; + }), + productionExpected(), + ), + refuses(code), + code, + ); + } + // The synthetic sentinel never validates as a release, even with the expected commit. + const sentinel = receipt({ source: { ...SYNTHETIC_SOURCE, dirty: false }, recipient: B }); + assert.throws( + () => + validateReleaseBinding(sentinel, { + ...productionExpected(), + source: SYNTHETIC_SOURCE.commit, + }), + refuses('BINDING_SCOPE'), + ); + // A rotation keeps every earlier candidate valid: a retired recipient still verifies, a recipient + // outside the history never does, and a history that dropped one refuses that one's candidates. + validateReleaseBinding(productionReceipt(A), productionExpected()); + validateReleaseBinding(productionReceipt(B), productionExpected()); + assert.throws( + () => validateReleaseBinding(productionReceipt(C), productionExpected()), + refuses('BINDING_RECIPIENT'), + ); + const dropped = production({ ...B, status: 'current' }); + assert.throws( + () => validateReleaseBinding(productionReceipt(A), productionExpected(dropped)), + refuses('BINDING_RECIPIENT'), + ); +}); + +test('the expectation is required, and production needs the loaded recipient itself', () => { + const value = productionReceipt(); + for (const expected of [ + undefined, + null, + {}, + { scope: 'release' }, + { ...synthetic, scope: 'Synthetic' }, + ]) { + assert.throws(() => validateReleaseBinding(value, expected), refuses('BINDING_SCOPE')); + } + for (const recipient of [ + { ...ROTATED }, + globalThis.structuredClone(ROTATED), + { file: ROTATED.file, current: ROTATED.current, history: ROTATED.history }, + undefined, + ]) { + assert.throws( + () => validateReleaseBinding(value, { ...productionExpected(), recipient }), + refuses('RECIPIENT_SOURCE'), + ); + } + // A synthetic expectation without pins admits no recipient. + assert.throws( + () => validateReleaseBinding(receipt(), { scope: 'synthetic', packageVersion: VERSION }), + refuses('BINDING_RECIPIENT'), + ); +}); + +test('only the exact canonical bytes parse', () => { + const bytes = serializeReleaseBinding(validateReleaseBinding(receipt(), synthetic)); + const text = bytes.toString('utf8'); + for (const [name, bad] of [ + ['duplicate key', Buffer.from(text.replace('{\n', '{\n "version": 1,\n'))], + [ + 'reordered keys', + Buffer.from(`${JSON.stringify({ kind: RELEASE_BINDING_KIND, ...receipt() }, null, 2)}\n`), + ], + ['compact', Buffer.from(`${JSON.stringify(receipt())}\n`)], + ['no final newline', Buffer.from(text.slice(0, -1))], + ['BOM', Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), bytes])], + ['trailing byte', Buffer.concat([bytes, Buffer.from(' ')])], + ['CRLF', Buffer.from(text.replaceAll('\n', '\r\n'))], + [ + 'invalid UTF-8 inside a string', + (() => { + const bad = Buffer.from(bytes); + bad[bytes.indexOf('"workflow": "release"') + '"workflow": "r'.length] = 0xff; + return bad; + })(), + ], + ['not JSON', Buffer.from('receipt')], + // Also refused as non-canonical: no canonical receipt reaches the 64 KiB bound, which only spares the + // parser an oversized input. + ['too large', Buffer.concat([bytes, Buffer.alloc(64 * 1024 + 1 - bytes.length, 0x20)])], + [ + 'unsorted inputs', + Buffer.from( + `${JSON.stringify( + changed(receipt(), (copy) => { + copy.inputs = { + 'package.json': 'b'.repeat(64), + 'apps/cli/package.json': 'a'.repeat(64), + }; + }), + null, + 2, + )}\n`, + ), + ], + ]) { + assert.throws(() => parseReleaseBinding(bad, synthetic), refuses('BINDING_BYTES'), name); + } + // Only a Buffer read from the file is parsed: a string or another view refuses. + assert.throws(() => parseReleaseBinding(text, synthetic), refuses('BINDING_BYTES')); + assert.throws( + () => parseReleaseBinding(new Uint8Array(bytes), synthetic), + refuses('BINDING_BYTES'), + ); + // A canonical receipt invalid for the expectation still refuses with its own code. + assert.throws( + () => parseReleaseBinding(bytes, { ...synthetic, packageVersion: '0.1.2' }), + refuses('BINDING_CANDIDATE'), + ); +}); + +test("the package version and fingerprint rules are binding.mjs's and parsePins'", () => { + /** @param {() => unknown} call */ + const admits = (call) => { + try { + call(); + return true; + } catch (error) { + if (error instanceof CustodyError) return false; + throw error; + } + }; + for (const version of [ + '0.1.1', + '1.2.3-rc.1', + '1.2.3+build.5', + '01.2.3', + '1.2', + 'v1.2.3', + '1.2.3-', + '1.2.3+', + `1.2.3-${'a'.repeat(94)}`, + `1.2.3-${'a'.repeat(95)}`, + ]) { + const value = changed(receipt(), (copy) => { + copy.packageVersion = version; + copy.tarball.name = tarballName(version); + copy.ciphertext.name = ciphertextFileName(version); + }); + assert.equal( + admits(() => validateReleaseBinding(value, { ...synthetic, packageVersion: version })), + isPackageVersion(version), + version, + ); + } + for (const fingerprint of [ + 'A'.repeat(40), + '0'.repeat(40), + 'a'.repeat(40), + 'A'.repeat(39), + 'A'.repeat(41), + 'G'.repeat(40), + `${'A'.repeat(39)}!`, + ]) { + const pins = { primary: fingerprint, subkey: PINS.subkey }; + assert.equal( + admits(() => validateReleaseBinding(receipt({ recipient: pins }), { ...synthetic, pins })), + admits(() => parsePins(pins)), + fingerprint, + ); + } +}); + +/** A retained candidate directory: its receipt, tarball and ciphertext. */ +function candidate({ + value = receipt(), + tarball = TARBALL, + ciphertext = CIPHERTEXT, + expected = synthetic, +} = {}) { + const directory = fresh('candidate'); + mkdirSync(directory); + writeFileSync( + join(directory, RELEASE_BINDING_FILE), + serializeReleaseBinding(validateReleaseBinding(value, expected)), + ); + writeFileSync(join(directory, tarballName(VERSION)), tarball); + writeFileSync(join(directory, ciphertextFileName(VERSION)), ciphertext); + return directory; +} +/** Every entry's kind and bytes, to show a refusal changed nothing; only regular files are read. */ +const snapshot = (directory) => + readdirSync(directory) + .sort() + .map((name) => { + const path = join(directory, name); + const info = lstatSync(path); + if (info.isFile()) return [name, sha256(readFileSync(path))]; + return [name, info.isSymbolicLink() ? `link ${readlinkSync(path)}` : String(info.mode)]; + }); + +test('a retained candidate is reused from its verified bytes, with no rebuild and no GnuPG call', () => { + for (const [value, expected] of [ + [receipt(), synthetic], + [productionReceipt(), productionExpected()], + ]) { + const directory = candidate({ value, expected }); + const before = snapshot(directory); + const verified = verifyRetainedCandidate({ directory, expected }); + assert.ok(Object.isFrozen(verified)); + assert.deepEqual(verified.receipt, value); + assert.deepEqual(verified.receiptBytes, readFileSync(join(directory, RELEASE_BINDING_FILE))); + assert.deepEqual(verified.tarball, TARBALL); + assert.deepEqual(verified.ciphertext, CIPHERTEXT); + // The bytes returned are the bytes verified: replacing a file afterwards changes nothing returned. + writeFileSync(join(directory, tarballName(VERSION)), Buffer.alloc(TARBALL.length, 0x78)); + assert.deepEqual(verified.tarball, TARBALL); + writeFileSync(join(directory, tarballName(VERSION)), TARBALL); + assert.deepEqual(snapshot(directory), before); + } + // A candidate sealed under a retired recipient is still reused after a rotation. + const retired = productionReceipt(A); + verifyRetainedCandidate({ + directory: candidate({ value: retired, expected: productionExpected() }), + expected: productionExpected(), + }); +}); + +test('a retained candidate whose assets or entries differ from its receipt refuses and is left as it was', () => { + const flip = (bytes) => { + const copy = Buffer.from(bytes); + copy[0] ^= 1; + return copy; + }; + const cases = [ + [ + 'CANDIDATE_DIGEST', + (directory) => writeFileSync(join(directory, tarballName(VERSION)), flip(TARBALL)), + ], + [ + 'CANDIDATE_DIGEST', + (directory) => writeFileSync(join(directory, ciphertextFileName(VERSION)), flip(CIPHERTEXT)), + ], + [ + 'CANDIDATE_SIZE', + (directory) => + writeFileSync( + join(directory, ciphertextFileName(VERSION)), + Buffer.concat([CIPHERTEXT, Buffer.from('x')]), + ), + ], + [ + 'CANDIDATE_SIZE', + (directory) => writeFileSync(join(directory, tarballName(VERSION)), TARBALL.subarray(1)), + ], + ['CANDIDATE_ENTRIES', (directory) => writeFileSync(join(directory, 'notes.txt'), 'extra')], + ['CANDIDATE_ENTRIES', (directory) => rmSync(join(directory, ciphertextFileName(VERSION)))], + ['CANDIDATE_ENTRIES', (directory) => rmSync(join(directory, RELEASE_BINDING_FILE))], + // Same count, another name: the listing is compared by name, not by size. + [ + 'CANDIDATE_ENTRIES', + (directory) => + renameSync( + join(directory, tarballName(VERSION)), + join(directory, tarballName(VERSION).toUpperCase()), + ), + ], + [ + 'CANDIDATE_ASSET', + (directory) => { + rmSync(join(directory, tarballName(VERSION))); + mkdirSync(join(directory, tarballName(VERSION))); + }, + ], + [ + 'BINDING_BYTES', + (directory) => + writeFileSync( + join(directory, RELEASE_BINDING_FILE), + Buffer.concat([readFileSync(join(directory, RELEASE_BINDING_FILE)), Buffer.from(' ')]), + ), + ], + ]; + if (posix) { + cases.push( + [ + 'CANDIDATE_ASSET', + (directory) => { + const target = fresh('linked-ciphertext'); + writeFileSync(target, CIPHERTEXT); + rmSync(join(directory, ciphertextFileName(VERSION))); + symlinkSync(target, join(directory, ciphertextFileName(VERSION))); + }, + ], + [ + 'CANDIDATE_ASSET', + (directory) => { + const target = fresh('linked-receipt'); + writeFileSync(target, readFileSync(join(directory, RELEASE_BINDING_FILE))); + rmSync(join(directory, RELEASE_BINDING_FILE)); + symlinkSync(target, join(directory, RELEASE_BINDING_FILE)); + }, + ], + [ + 'CANDIDATE_ASSET', + (directory) => { + rmSync(join(directory, tarballName(VERSION))); + assert.equal(spawnSync('mkfifo', [join(directory, tarballName(VERSION))]).status, 0); + }, + ], + ); + } + for (const [code, change] of cases) { + const directory = candidate(); + change(directory); + const before = snapshot(directory); + assert.throws( + () => verifyRetainedCandidate({ directory, expected: synthetic }), + refuses(code), + code, + ); + assert.deepEqual(snapshot(directory), before); + } + // A receipt over its bound is not read. + const large = candidate(); + writeFileSync(join(large, RELEASE_BINDING_FILE), Buffer.alloc(64 * 1024 + 1, 0x20)); + assert.throws( + () => verifyRetainedCandidate({ directory: large, expected: synthetic }), + refuses('CANDIDATE_ASSET'), + ); + // A stale receipt, for another candidate of the same version, refuses. + const stale = candidate({ value: productionReceipt(), expected: productionExpected() }); + assert.throws( + () => + verifyRetainedCandidate({ + directory: stale, + expected: { ...productionExpected(), source: 'f'.repeat(40) }, + }), + refuses('BINDING_CANDIDATE'), + ); + // So does a receipt verified in the wrong scope. + assert.throws( + () => verifyRetainedCandidate({ directory: stale, expected: synthetic }), + refuses('BINDING_SCOPE'), + ); +}); + +test('each expectation field is read once, so a changing expectation cannot change scope midway', () => { + let reads = 0; + const flipping = { + get scope() { + reads += 1; + return reads === 1 ? 'synthetic' : 'production'; + }, + recipient: ROTATED, + pins: PINS, + packageVersion: VERSION, + source: CLEAN.commit, + }; + // Read as synthetic once: a clean source is never a synthetic candidate. + assert.throws( + () => validateReleaseBinding(receipt({ source: CLEAN, recipient: PINS }), flipping), + refuses('BINDING_SCOPE'), + ); + assert.equal(reads, 1); +}); + +test( + 'a retained asset swapped for a link after its size check refuses as not an ordinary file', + { skip: !posix }, + () => { + const directory = candidate(); + const path = join(directory, ciphertextFileName(VERSION)); + const other = fresh('other-ciphertext'); + writeFileSync(other, Buffer.alloc(CIPHERTEXT.length, 0x78)); + const fs = createRequire(import.meta.url)('node:fs'); + const lstat = fs.lstatSync; + let swapped = false; + fs.lstatSync = (target, ...rest) => { + const info = lstat(target, ...rest); + if (target === path && !swapped) { + swapped = true; + rmSync(path); + symlinkSync(other, path); + } + return info; + }; + syncBuiltinESMExports(); + try { + assert.throws( + () => verifyRetainedCandidate({ directory, expected: synthetic }), + refuses('CANDIDATE_ASSET'), + ); + } finally { + fs.lstatSync = lstat; + syncBuiltinESMExports(); + } + assert.equal(swapped, true); + }, +); + +test('each receipt field is read once, and a throwing accessor refuses with a fixed code', () => { + // A name checked on its first read is the name returned, never a later value. + let reads = 0; + const value = receipt(); + Object.defineProperty(value.tarball, 'name', { + enumerable: true, + get() { + reads += 1; + return reads === 1 ? tarballName(VERSION) : '/private/host/path'; + }, + }); + assert.equal(validateReleaseBinding(value, synthetic).tarball.name, tarballName(VERSION)); + assert.equal(reads, 1); + // A clean source checked once stays clean in the receipt returned. + let dirtyReads = 0; + const clean = productionReceipt(); + Object.defineProperty(clean.source, 'dirty', { + enumerable: true, + get() { + dirtyReads += 1; + return dirtyReads !== 1; + }, + }); + assert.equal(validateReleaseBinding(clean, productionExpected()).source.dirty, false); + assert.equal(dirtyReads, 1); + // The origin and the inputs are read once each. + const counts = { origin: 0, inputs: 0 }; + const counted = receipt(); + for (const key of ['origin', 'inputs']) { + const original = counted[key]; + Object.defineProperty(counted, key, { + enumerable: true, + get() { + counts[key] += 1; + return original; + }, + }); + } + validateReleaseBinding(counted, synthetic); + assert.deepEqual(counts, { origin: 1, inputs: 1 }); + const throwing = receipt(); + Object.defineProperty(throwing, 'target', { + enumerable: true, + get() { + throw new Error(`at ${scratch}`); + }, + }); + assert.throws(() => validateReleaseBinding(throwing, synthetic), refuses('UNEXPECTED')); + const expectation = { + ...synthetic, + get pins() { + throw new Error(`at ${scratch}`); + }, + }; + assert.throws( + () => + parseReleaseBinding( + serializeReleaseBinding(validateReleaseBinding(receipt(), synthetic)), + expectation, + ), + refuses('UNEXPECTED'), + ); +}); + +test('reuse reads its expectation once, and a vanished asset refuses as that asset', () => { + const directory = candidate(); + let reads = 0; + const flipping = { + scope: 'synthetic', + pins: PINS, + get packageVersion() { + reads += 1; + return reads === 1 ? VERSION : '0.1.2'; + }, + }; + verifyRetainedCandidate({ directory, expected: flipping }); + assert.equal(reads, 1); + // An asset that disappears after the listing refuses as that asset, never a host error. + const fs = createRequire(import.meta.url)('node:fs'); + const lstat = fs.lstatSync; + const path = join(directory, tarballName(VERSION)); + fs.lstatSync = (target, ...rest) => { + if (target === path) throw Object.assign(new Error(`ENOENT: ${path}`), { code: 'ENOENT' }); + return lstat(target, ...rest); + }; + syncBuiltinESMExports(); + try { + assert.throws( + () => verifyRetainedCandidate({ directory, expected: synthetic }), + refuses('CANDIDATE_ASSET'), + ); + } finally { + fs.lstatSync = lstat; + syncBuiltinESMExports(); + } + // A malformed call refuses with a fixed code, and a throwing option with no host text. + for (const options of [undefined, null, 'options']) { + assert.throws(() => verifyRetainedCandidate(options), refuses('BINDING_SCOPE')); + } + assert.throws( + () => + verifyRetainedCandidate({ + get directory() { + throw new Error(`at ${scratch}`); + }, + }), + refuses('UNEXPECTED'), + ); + // The receipt bytes returned are the ones verified, even when the file changes during verification. + const changing = candidate(); + const receiptPath = join(changing, RELEASE_BINDING_FILE); + const original = readFileSync(receiptPath); + const tarballPath = join(changing, tarballName(VERSION)); + fs.lstatSync = (target, ...rest) => { + if (target === tarballPath) writeFileSync(receiptPath, Buffer.from('{"replaced": true}\n')); + return lstat(target, ...rest); + }; + syncBuiltinESMExports(); + let verified; + try { + verified = verifyRetainedCandidate({ directory: changing, expected: synthetic }); + } finally { + fs.lstatSync = lstat; + syncBuiltinESMExports(); + } + assert.deepEqual(verified.receiptBytes, original); +}); + +test('reuse needs a valid expectation, and a host failure refuses with a fixed code and no path', () => { + const directory = candidate(); + for (const [code, expected] of [ + ['BINDING_SCOPE', undefined], + ['BINDING_CANDIDATE', { ...synthetic, packageVersion: '../0.1.1' }], + ['BINDING_CANDIDATE', { ...synthetic, packageVersion: 1 }], + ]) { + assert.throws(() => verifyRetainedCandidate({ directory, expected }), refuses(code), code); + } + assert.throws( + () => + verifyRetainedCandidate({ directory: join(fresh('none'), 'candidate'), expected: synthetic }), + refuses('UNEXPECTED'), + ); + // The reuse API takes no GnuPG binary or run port. + assert.equal(verifyRetainedCandidate.length, 1); +}); diff --git a/tools/cli-package/release-custody.mjs b/tools/cli-package/release-custody.mjs new file mode 100644 index 00000000..97347f93 --- /dev/null +++ b/tools/cli-package/release-custody.mjs @@ -0,0 +1,288 @@ +/** + * Release custody (Step 9b.2; packaging.md, "Recipient custody implementation boundary — Step 9b.2"): the + * scope gate in front of the unchanged `encryptOnce`, and the release binding receipt's sealer. Production + * scope encrypts only a clean, non-synthetic candidate to the loaded production recipient; synthetic scope + * encrypts only a synthetic candidate, so neither a fixture key nor the per-run ephemeral key can ever + * encrypt a product staging. The sealer accepts only a record this gate returned, and writes the receipt + * once, after re-reading the ciphertext it names. This is the only release module that reaches GnuPG. + */ +import { Buffer } from 'node:buffer'; +import { randomUUID } from 'node:crypto'; +import { + closeSync, + fsyncSync, + linkSync, + lstatSync, + openSync, + readdirSync, + realpathSync, + rmSync, + writeSync, +} from 'node:fs'; +import { join, resolve } from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; +import { ARCHIVE_LIMITS } from './archive.mjs'; +import { readPrivateContainer } from './container.mjs'; +import { CustodyError, refuseCustody } from './custody-error.mjs'; +import { encryptOnce } from './custody.mjs'; +import { readOrdinaryFile, sha256, byCodeUnit } from './files.mjs'; +import { + RELEASE_BINDING_FILE, + RELEASE_BINDING_KIND, + ciphertextFileName, + serializeReleaseBinding, + tarballName, + validateReleaseBinding, +} from './release-binding.mjs'; +import { isProductionRecipient, isSyntheticSource } from './release-recipient.mjs'; + +/** + * @typedef {import('./custody.mjs').CustodyRecord} CustodyRecord + * @typedef {import('./release-recipient.mjs').ProductionRecipient} ProductionRecipient + * @typedef {{kind: 'production', recipient: ProductionRecipient} + * | {kind: 'synthetic', recipient: {file: string, pins: unknown}}} Scope + */ + +/** + * The records the gate returned, with what the sealer needs to bind them. + * @type {WeakMap} + */ +const records = new WeakMap(); + +/** @param {unknown} value @returns {value is Record} */ +const isRecord = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); + +/** Whether `path` names an entry, a dangling link included. @param {string} path */ +function present(path) { + try { + lstatSync(path); + return true; + } catch { + return false; + } +} + +/** + * Encrypt a candidate's container once for its scope. Every refusal here comes before any GnuPG call; + * `encryptOnce` then runs on the caller's destination, so its own refusals follow unchanged. + * @param {{scope: Scope, container: Buffer, destination: string, privateRoot: string, now?: number, + * run?: typeof import('./process.mjs').runCommand, gpg?: import('./gnupg.mjs').GnupgInstallation}} + * options + * @returns {Promise} + */ +export async function encryptCandidate(options) { + let admitted; + let ports; + try { + const { scope, container, destination, privateRoot, now, run, gpg } = isRecord(options) + ? options + : {}; + admitted = admit(scope, container, destination, privateRoot); + ports = { destination, privateRoot, now, run, gpg }; + } catch (error) { + // A host error's message names a path: only fixed custody codes leave. + if (error instanceof CustodyError) throw error; + refuseCustody('UNEXPECTED'); + } + // The admitted copy is the one encrypted: the caller's buffer may change, it may not. + const record = await encryptOnce({ + ...ports, + container: admitted.container, + recipient: admitted.recipient, + }); + records.set(record, { + kind: admitted.kind, + recipient: admitted.production, + manifest: admitted.manifest, + destination: admitted.destination, + }); + return record; +} + +/** + * Admit one encryption, reading the caller's scope and container once: the scope's rules, then a plain, + * unsealed destination that holds exactly the candidate's tarball, all before any GnuPG call. + * @param {unknown} scope @param {unknown} container @param {string} destination + * @param {string} privateRoot + */ +function admit(scope, container, destination, privateRoot) { + const { kind, recipient: given } = isRecord(scope) ? scope : {}; + if ((kind !== 'production' && kind !== 'synthetic') || !isRecord(given)) refuseCustody('SCOPE'); + if (kind === 'production' && !isProductionRecipient(given)) refuseCustody('RECIPIENT_SOURCE'); + const bytes = Buffer.isBuffer(container) ? Buffer.from(container) : container; + const { manifest } = readPrivateContainer(/** @type {Buffer} */ (bytes)); + let recipient; + let production; + if (kind === 'production') { + if (manifest.source.dirty) refuseCustody('DIRTY_SOURCE'); + if (isSyntheticSource(manifest.source)) refuseCustody('SYNTHETIC_SOURCE'); + production = /** @type {ProductionRecipient} */ (given); + recipient = { file: production.file, pins: production.current }; + } else { + if (!isSyntheticSource(manifest.source)) refuseCustody('FIXTURE_SCOPE'); + recipient = { file: given.file, pins: given.pins }; + } + // Normalized absolute paths only: a trailing separator, `.` or `..` would make lstat follow a link. + if (resolve(privateRoot) !== privateRoot) refuseCustody('PRIVATE_ROOT'); + if (resolve(destination) !== destination) refuseCustody('PLACEMENT'); + // A plain directory, as encryptOnce requires, checked before anything is read through it and + // resolved now so that nothing after the ciphertext is published can fail on it. + let resolved; + try { + if (!lstatSync(destination).isDirectory()) refuseCustody('PLACEMENT'); + resolved = realpathSync(destination); + } catch (error) { + if (error instanceof CustodyError) throw error; + refuseCustody('PLACEMENT'); + } + if (present(join(resolved, RELEASE_BINDING_FILE))) refuseCustody('ALREADY_SEALED'); + // The sealer admits exactly the tarball and the ciphertext: a destination holding anything else, or a + // missing or different tarball, would leave an unsealable ciphertext, so it refuses first. + const tarballFile = tarballName(manifest.packageVersion); + const entries = readdirSync(resolved); + if (entries.includes(ciphertextFileName(manifest.packageVersion))) { + refuseCustody('ALREADY_ENCRYPTED'); + } + if (!isDeepStrictEqual(entries, [tarballFile])) refuseCustody('CANDIDATE_ENTRIES'); + let tarball; + try { + tarball = readOrdinaryFile(join(resolved, tarballFile), ARCHIVE_LIMITS.compressedBytes); + } catch { + refuseCustody('TARBALL_DRIFT'); + } + if (sha256(tarball) !== manifest.tarballSha256) refuseCustody('TARBALL_DRIFT'); + return { kind, recipient, production, manifest, container: bytes, destination: resolved }; +} + +/** Write a new public file durably; an existing file refuses. @param {string} path @param {Buffer} bytes */ +function writePublicFile(path, bytes) { + const descriptor = openSync(path, 'wx', 0o644); + try { + let offset = 0; + while (offset < bytes.length) offset += writeSync(descriptor, bytes, offset); + fsyncSync(descriptor); + } finally { + closeSync(descriptor); + } +} + +/** + * Publish once and atomically: the bytes go to a fresh sibling, which is then hard-linked to the final + * name. A link never replaces an existing receipt, and a failed write never leaves a partial file under + * the final name. Once the link exists the receipt is published: a sibling that cannot be removed then + * is left behind rather than reported as a refusal. + * @param {string} directory @param {Buffer} bytes @param {(path: string, bytes: Buffer) => void} write + */ +function publish(directory, bytes, write) { + const temporary = join(directory, `.${RELEASE_BINDING_FILE}.${randomUUID()}.partial`); + try { + write(temporary, bytes); + try { + linkSync(temporary, join(directory, RELEASE_BINDING_FILE)); + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'EEXIST') { + refuseCustody('ALREADY_SEALED'); + } + throw error; + } + } finally { + try { + rmSync(temporary, { force: true }); + } catch { + // Best effort: the final name alone is the published claim. + } + } +} + +/** + * Seal the release binding receipt for a record `encryptCandidate` returned, beside its ciphertext and + * the candidate's tarball. `origin` is the originating trusted run (Step 9d supplies and binds it); `write` + * is a test seam for fault injection. + * @param {CustodyRecord} record + * @param {{origin?: unknown, write?: (path: string, bytes: Buffer) => void}} [options] + * @returns {{receipt: ReturnType, bytes: Buffer}} + */ +export function sealReleaseBinding(record, options) { + try { + const { origin, write = writePublicFile } = isRecord(options) ? options : {}; + return seal(record, origin, write); + } catch (error) { + if (error instanceof CustodyError) throw error; + return refuseCustody('UNEXPECTED'); + } +} + +/** + * @param {CustodyRecord} record @param {unknown} origin + * @param {(path: string, bytes: Buffer) => void} write + */ +function seal(record, origin, write) { + const entry = records.get(record); + if (entry === undefined) refuseCustody('RECORD_SOURCE'); + const { kind, recipient, manifest, destination } = entry; + // An existing receipt refuses before anything is read; one is never written over. + if (present(join(destination, RELEASE_BINDING_FILE))) refuseCustody('ALREADY_SEALED'); + const tarball = tarballName(manifest.packageVersion); + if ( + !isDeepStrictEqual( + readdirSync(destination).sort(byCodeUnit), + [record.name, tarball].sort(byCodeUnit), + ) + ) { + refuseCustody('CANDIDATE_ENTRIES'); + } + // The ciphertext must still be the bytes encrypted: the receipt is sealed only after it exists. + let ciphertext; + try { + ciphertext = readOrdinaryFile(join(destination, record.name), record.bytes); + } catch { + refuseCustody('CIPHERTEXT_DRIFT'); + } + if (ciphertext.length !== record.bytes || sha256(ciphertext) !== record.sha256) { + refuseCustody('CIPHERTEXT_DRIFT'); + } + let tarballBytes; + try { + tarballBytes = readOrdinaryFile(join(destination, tarball), ARCHIVE_LIMITS.compressedBytes); + } catch { + refuseCustody('TARBALL_DRIFT'); + } + const tarballSha256 = sha256(tarballBytes); + if (tarballSha256 !== manifest.tarballSha256) refuseCustody('TARBALL_DRIFT'); + + const expected = + kind === 'production' + ? { + scope: /** @type {const} */ ('production'), + recipient, + packageVersion: manifest.packageVersion, + source: manifest.source.commit, + } + : { + scope: /** @type {const} */ ('synthetic'), + pins: record.recipient, + packageVersion: manifest.packageVersion, + }; + const receipt = validateReleaseBinding( + { + version: 1, + kind: RELEASE_BINDING_KIND, + packageVersion: manifest.packageVersion, + source: { ...manifest.source }, + tarball: { name: tarball, bytes: tarballBytes.length, sha256: tarballSha256 }, + ciphertext: { name: record.name, bytes: record.bytes, sha256: record.sha256 }, + recipient: { ...record.recipient }, + toolchain: { ...manifest.toolchain, gnupg: record.gnupg }, + target: manifest.target, + inputs: { ...manifest.inputs }, + mapCheck: { status: 'complete' }, + origin, + }, + expected, + ); + const bytes = serializeReleaseBinding(receipt); + publish(destination, bytes, write); + records.delete(record); + return Object.freeze({ receipt, bytes }); +} diff --git a/tools/cli-package/release-custody.test.mjs b/tools/cli-package/release-custody.test.mjs new file mode 100644 index 00000000..29848508 --- /dev/null +++ b/tools/cli-package/release-custody.test.mjs @@ -0,0 +1,795 @@ +import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; +import { + appendFileSync, + chmodSync, + mkdirSync, + readFileSync, + readdirSync, + renameSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { createRequire, syncBuiltinESMExports } from 'node:module'; +import { basename, join, relative } from 'node:path'; +import { before, test } from 'node:test'; +import { fresh, posixModes } from './candidate-fixture.mjs'; +import { readPrivateContainer } from './container.mjs'; +import { CustodyError } from './custody-error.mjs'; +import { encryptOnce } from './custody.mjs'; +import { sha256 } from './files.mjs'; +import { PINS, ephemeralKey, fixture } from './gpg-fixture.mjs'; +import { runCommand } from './process.mjs'; +import { + RELEASE_BINDING_FILE, + ciphertextFileName, + parseReleaseBinding, + serializeReleaseBinding, + tarballName, + verifyRetainedCandidate, +} from './release-binding.mjs'; +import { encryptCandidate, sealReleaseBinding } from './release-custody.mjs'; +import { + CLEAN, + TARBALL, + VERSION, + containerWith, + containers, + origin, + places, + recipientRoot, +} from './release-fixture.mjs'; +import { SYNTHETIC_SOURCE, loadRecipientForTests } from './release-recipient.mjs'; + +const refuses = (code) => (error) => + error instanceof CustodyError && + error.code === code && + /^PACKED_CLI_CUSTODY_[A-Z_]+$/u.test(error.message); + +/** @type {{staging: string, synthetic: Buffer, clean: Buffer}} */ +let built; +before(async () => { + built = await containers(); +}); + +/** A run port that records every GnuPG call; everything else is the real GnuPG. */ +function recording(before = () => {}) { + const calls = []; + /** @type {typeof runCommand} */ + const run = async (command, args, options) => { + before(args); + calls.push( + args.find( + (arg) => + arg.startsWith('--') && + ![ + '--homedir', + '--batch', + '--no-tty', + '--no-options', + '--no-autostart', + '--no-keyring', + ].includes(arg), + ), + ); + return runCommand(command, args, options); + }; + return { run, calls }; +} +const synthetic = Object.freeze({ + kind: 'synthetic', + recipient: { file: fixture('good'), pins: PINS.good }, +}); + +/** No receipt of either kind exists: custody never downgrades to a package-test artifact. */ +function noReceipt(destination) { + const names = readdirSync(destination); + assert.equal(names.includes(RELEASE_BINDING_FILE), false, 'a release receipt was written'); + assert.equal( + names.includes('package-test-receipt.json'), + false, + 'a package-test receipt was written', + ); +} + +test('the gate refuses a wrong scope, recipient or candidate before any GnuPG call', async () => { + const unbranded = { + kind: 'production', + recipient: { file: fixture('good'), current: PINS.good, history: [] }, + }; + const ephemeral = ephemeralKey(); + const production = { + kind: 'production', + recipient: loadRecipientForTests({ + root: recipientRoot(ephemeral.file, [{ ...ephemeral.pins, status: 'current' }]), + }), + }; + const dirtyClean = containerWith(built.staging, { ...CLEAN, dirty: true }); + const cleanSentinel = containerWith(built.staging, { ...SYNTHETIC_SOURCE, dirty: false }); + for (const [code, scope, container] of [ + ['SCOPE', undefined, built.synthetic], + ['SCOPE', { kind: 'release', recipient: synthetic.recipient }, built.synthetic], + ['SCOPE', { kind: 'synthetic' }, built.synthetic], + ['RECIPIENT_SOURCE', unbranded, built.clean], + ['RECIPIENT_SOURCE', { ...production, recipient: { ...production.recipient } }, built.clean], + ['DIRTY_SOURCE', production, dirtyClean], + ['DIRTY_SOURCE', production, built.synthetic], + ['SYNTHETIC_SOURCE', production, cleanSentinel], + ['FIXTURE_SCOPE', synthetic, built.clean], + ['FIXTURE_SCOPE', synthetic, dirtyClean], + ]) { + const { root, destination } = places(); + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ scope, container, destination, privateRoot: root, run }), + refuses(code), + code, + ); + assert.deepEqual(calls, [], code); + assert.deepEqual(readdirSync(destination), [tarballName(VERSION)], code); + } + // A destination that cannot be resolved, or whose tarball is missing or another, refuses before any + // call: a ciphertext encrypted into it could never be sealed. + // A link refuses before anything is read through it: its target's stray entry is never reached. + const linked = (destination) => { + const target = fresh('linked-destination-target'); + mkdirSync(target); + writeFileSync(join(target, tarballName(VERSION)), TARBALL); + writeFileSync(join(target, 'stray'), 'x'); + rmSync(destination, { recursive: true }); + symlinkSync(target, destination); + }; + for (const [code, change] of [ + ['PLACEMENT', (destination) => rmSync(destination, { recursive: true })], + [ + 'PLACEMENT', + (destination) => + posixModes ? linked(destination) : rmSync(destination, { recursive: true }), + ], + ['CANDIDATE_ENTRIES', (destination) => rmSync(join(destination, tarballName(VERSION)))], + [ + 'CANDIDATE_ENTRIES', + (destination) => writeFileSync(join(destination, 'package-test-receipt.json'), '{}'), + ], + [ + 'ALREADY_ENCRYPTED', + (destination) => + writeFileSync(join(destination, ciphertextFileName(VERSION)), 'earlier ciphertext'), + ], + [ + 'TARBALL_DRIFT', + (destination) => + writeFileSync(join(destination, tarballName(VERSION)), Buffer.from('another tarball')), + ], + [ + 'TARBALL_DRIFT', + (destination) => { + rmSync(join(destination, tarballName(VERSION))); + mkdirSync(join(destination, tarballName(VERSION))); + }, + ], + // A link to a file with the right bytes is not an ordinary tarball either. + [ + 'TARBALL_DRIFT', + (destination) => { + if (!posixModes) return rmSync(join(destination, tarballName(VERSION))); + const target = fresh('linked-tarball-target'); + writeFileSync(target, TARBALL); + rmSync(join(destination, tarballName(VERSION))); + symlinkSync(target, join(destination, tarballName(VERSION))); + }, + ], + ]) { + const { root, destination } = places(); + change(destination); + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ + scope: synthetic, + container: built.synthetic, + destination, + privateRoot: root, + run, + }), + refuses(code), + code, + ); + assert.deepEqual(calls, [], code); + } + // Only a normalized absolute path names the destination and the private root, so a trailing separator + // or `.` cannot make lstat follow a link; a link whose target holds a receipt refuses as a link. + if (posixModes) { + const target = fresh('linked-target'); + mkdirSync(target); + writeFileSync(join(target, tarballName(VERSION)), TARBALL); + const link = fresh('linked-destination'); + symlinkSync(target, link); + const sealedTarget = fresh('linked-sealed-target'); + mkdirSync(sealedTarget); + writeFileSync(join(sealedTarget, tarballName(VERSION)), TARBALL); + writeFileSync(join(sealedTarget, RELEASE_BINDING_FILE), '{}'); + const sealedLink = fresh('linked-sealed-destination'); + symlinkSync(sealedTarget, sealedLink); + for (const destination of [`${link}/`, `${link}/.`, sealedLink]) { + const { root } = places(); + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ + scope: synthetic, + container: built.synthetic, + destination, + privateRoot: root, + run, + }), + refuses('PLACEMENT'), + destination, + ); + assert.deepEqual(calls, [], destination); + assert.deepEqual(readdirSync(target), [tarballName(VERSION)]); + } + } + { + const { root, destination } = places(); + for (const [code, options] of [ + ['PLACEMENT', { destination: relative(process.cwd(), destination), privateRoot: root }], + [ + 'PLACEMENT', + { destination: `${destination}/../${basename(destination)}`, privateRoot: root }, + ], + ['PRIVATE_ROOT', { destination, privateRoot: `${root}/` }], + ]) { + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ scope: synthetic, container: built.synthetic, ...options, run }), + refuses(code), + JSON.stringify(options), + ); + assert.deepEqual(calls, []); + } + } + // A tarball whose name differs only in case is not the tarball, even where the volume would open it. + { + const { root, destination } = places(); + rmSync(join(destination, tarballName(VERSION))); + writeFileSync(join(destination, tarballName(VERSION).toUpperCase()), TARBALL); + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ + scope: synthetic, + container: built.synthetic, + destination, + privateRoot: root, + run, + }), + refuses('CANDIDATE_ENTRIES'), + ); + assert.deepEqual(calls, []); + } + // A malformed call refuses with a fixed code, never a host error. + for (const options of [undefined, null, 'options']) { + await assert.rejects(encryptCandidate(options), refuses('SCOPE')); + } + await assert.rejects( + encryptCandidate({ + get scope() { + throw new Error('/private/host/path'); + }, + }), + (error) => refuses('UNEXPECTED')(error) && !error.message.includes('/private/host/path'), + ); + // The recipient is read once too: the brand check sees the value encrypted to. + { + let reads = 0; + const counted = { + kind: 'production', + get recipient() { + reads += 1; + return { file: ephemeral.file, current: ephemeral.pins, history: [] }; + }, + }; + const { root, destination } = places(); + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ + scope: counted, + container: built.clean, + destination, + privateRoot: root, + run, + }), + refuses('RECIPIENT_SOURCE'), + ); + assert.deepEqual(calls, []); + assert.equal(reads, 1); + } + // The scope is read once: a kind that changes after its first read cannot skip the brand check. + { + let reads = 0; + const flipping = { + get kind() { + reads += 1; + return reads === 1 ? 'production' : 'synthetic'; + }, + recipient: { file: ephemeral.file, current: ephemeral.pins, history: [] }, + }; + const { root, destination } = places(); + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ + scope: flipping, + container: built.clean, + destination, + privateRoot: root, + run, + }), + refuses('RECIPIENT_SOURCE'), + ); + assert.deepEqual(calls, []); + assert.equal(reads, 1); + } + // A destination that already holds a receipt is sealed: nothing is encrypted into it. + for (const kind of ['file', 'dangling link']) { + const { root, destination } = places(); + if (kind === 'file') writeFileSync(join(destination, RELEASE_BINDING_FILE), '{}'); + else if (posixModes) + symlinkSync(join(destination, 'missing'), join(destination, RELEASE_BINDING_FILE)); + else continue; + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ + scope: synthetic, + container: built.synthetic, + destination, + privateRoot: root, + run, + }), + refuses('ALREADY_SEALED'), + ); + assert.deepEqual(calls, []); + } + // A host failure in the gate refuses with a fixed code, never the host's error. + { + const { root } = places(); + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ + scope: synthetic, + container: built.synthetic, + destination: undefined, + privateRoot: root, + run, + }), + refuses('UNEXPECTED'), + ); + assert.deepEqual(calls, []); + } + // A container that is not one refuses as encryptOnce would, before any call. + const { root, destination } = places(); + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ + scope: synthetic, + container: Buffer.from('x'), + destination, + privateRoot: root, + run, + }), + (error) => error instanceof CustodyError && error.code.startsWith('CONTAINER_'), + ); + assert.deepEqual(calls, []); +}); + +/** Encrypt the synthetic container in fresh places and return everything the sealer tests need. */ +async function encrypted(scope = synthetic, container = built.synthetic) { + const { root, destination } = places(); + const record = await encryptCandidate({ scope, container, destination, privateRoot: root }); + return { root, destination, record }; +} + +test('a synthetic candidate is encrypted, then sealed once beside its ciphertext and tarball', async () => { + const { root, destination, record } = await encrypted(); + assert.deepEqual(record.recipient, PINS.good); + assert.deepEqual(readdirSync(root), []); + const sealed = sealReleaseBinding(record, { origin: origin(SYNTHETIC_SOURCE.commit) }); + assert.deepEqual( + readdirSync(destination).sort(), + [ciphertextFileName(VERSION), RELEASE_BINDING_FILE, tarballName(VERSION)].sort(), + ); + const bytes = readFileSync(join(destination, RELEASE_BINDING_FILE)); + assert.deepEqual(bytes, sealed.bytes); + assert.deepEqual(serializeReleaseBinding(sealed.receipt), bytes); + const manifest = readPrivateContainer(built.synthetic).manifest; + const expected = { scope: 'synthetic', pins: PINS.good, packageVersion: VERSION }; + assert.deepEqual(parseReleaseBinding(bytes, expected), sealed.receipt); + assert.deepEqual(sealed.receipt.source, manifest.source); + assert.deepEqual(sealed.receipt.tarball, { + name: tarballName(VERSION), + bytes: TARBALL.length, + sha256: sha256(TARBALL), + }); + assert.deepEqual(sealed.receipt.ciphertext, { + name: record.name, + bytes: record.bytes, + sha256: record.sha256, + }); + assert.deepEqual(sealed.receipt.recipient, PINS.good); + assert.equal(sealed.receipt.toolchain.gnupg, record.gnupg); + assert.equal(sealed.receipt.toolchain.node, manifest.toolchain.node); + assert.deepEqual(sealed.receipt.inputs, manifest.inputs); + assert.equal(sealed.receipt.target, manifest.target); + // The sealed candidate is reusable as it stands, and a record seals only once. + verifyRetainedCandidate({ directory: destination, expected }); + assert.throws( + () => sealReleaseBinding(record, { origin: origin(SYNTHETIC_SOURCE.commit) }), + refuses('RECORD_SOURCE'), + ); +}); + +test('only a record the gate returned seals, and an existing receipt refuses before anything is read', async () => { + const { destination, record } = await encrypted(); + const direct = places(); + const unscoped = await encryptOnce({ + container: built.synthetic, + recipient: synthetic.recipient, + destination: direct.destination, + privateRoot: direct.root, + }); + for (const value of [ + { ...record }, + globalThis.structuredClone(record), + unscoped, + undefined, + null, + 'record', + ]) { + assert.throws( + () => sealReleaseBinding(value, { origin: origin(SYNTHETIC_SOURCE.commit) }), + refuses('RECORD_SOURCE'), + ); + } + // A planted receipt wins over the unreadable ciphertext: nothing was read. + writeFileSync(join(destination, RELEASE_BINDING_FILE), 'planted'); + rmSync(join(destination, record.name)); + assert.throws( + () => sealReleaseBinding(record, { origin: origin(SYNTHETIC_SOURCE.commit) }), + refuses('ALREADY_SEALED'), + ); + assert.equal(readFileSync(join(destination, RELEASE_BINDING_FILE), 'utf8'), 'planted'); + // Calling without options, or with a throwing origin, refuses with a fixed code, never a host error. + const again = await encrypted(); + assert.throws(() => sealReleaseBinding(again.record), refuses('BINDING_ORIGIN')); + assert.throws(() => sealReleaseBinding(again.record, null), refuses('BINDING_ORIGIN')); + assert.throws( + () => + sealReleaseBinding(again.record, { + get origin() { + throw new Error(`at ${again.destination}`); + }, + }), + (error) => refuses('UNEXPECTED')(error) && !error.message.includes(again.destination), + ); + noReceipt(again.destination); +}); + +test('the sealer refuses a candidate directory that is not exactly its tarball and the ciphertext encrypted', async () => { + const flip = (path) => { + const bytes = readFileSync(path); + bytes[bytes.length - 1] ^= 1; + writeFileSync(path, bytes); + }; + const cases = [ + ['CANDIDATE_ENTRIES', (destination) => writeFileSync(join(destination, 'notes.txt'), 'x')], + ['CANDIDATE_ENTRIES', (destination) => rmSync(join(destination, tarballName(VERSION)))], + ['CANDIDATE_ENTRIES', (destination, record) => rmSync(join(destination, record.name))], + // Same count, another name: the listing is compared by name, not by size. + [ + 'CANDIDATE_ENTRIES', + (destination) => + renameSync( + join(destination, tarballName(VERSION)), + join(destination, tarballName(VERSION).toUpperCase()), + ), + ], + [ + 'CIPHERTEXT_DRIFT', + (destination, record) => appendFileSync(join(destination, record.name), 'x'), + ], + ['CIPHERTEXT_DRIFT', (destination, record) => flip(join(destination, record.name))], + [ + 'CIPHERTEXT_DRIFT', + (destination, record) => { + const bytes = readFileSync(join(destination, record.name)); + writeFileSync(join(destination, record.name), bytes.subarray(1)); + }, + ], + ['TARBALL_DRIFT', (destination) => flip(join(destination, tarballName(VERSION)))], + [ + 'TARBALL_DRIFT', + (destination) => + writeFileSync(join(destination, tarballName(VERSION)), Buffer.alloc(16 * 1024 * 1024 + 1)), + ], + ]; + if (posixModes) { + cases.push( + [ + 'CIPHERTEXT_DRIFT', + (destination, record) => { + const target = fresh('linked-ciphertext'); + writeFileSync(target, readFileSync(join(destination, record.name))); + rmSync(join(destination, record.name)); + symlinkSync(target, join(destination, record.name)); + }, + ], + [ + 'TARBALL_DRIFT', + (destination) => { + const target = fresh('linked-tarball'); + writeFileSync(target, TARBALL); + rmSync(join(destination, tarballName(VERSION))); + symlinkSync(target, join(destination, tarballName(VERSION))); + }, + ], + ); + } + for (const [code, change] of cases) { + const { destination, record } = await encrypted(); + change(destination, record); + assert.throws( + () => sealReleaseBinding(record, { origin: origin(SYNTHETIC_SOURCE.commit) }), + refuses(code), + code, + ); + noReceipt(destination); + } + // A malformed origin refuses in the self-check, before anything is written. + for (const value of [ + { kind: 'local' }, + origin('f'.repeat(40)), + { ...origin(SYNTHETIC_SOURCE.commit), event: 'pull_request' }, + ]) { + const { destination, record } = await encrypted(); + assert.throws(() => sealReleaseBinding(record, { origin: value }), refuses('BINDING_ORIGIN')); + noReceipt(destination); + } +}); + +test('the receipt is published once and atomically, whatever happens to the write', async () => { + // A receipt that appears while sealing is never overwritten. + const raced = await encrypted(); + assert.throws( + () => + sealReleaseBinding(raced.record, { + origin: origin(SYNTHETIC_SOURCE.commit), + write: (path, bytes) => { + writeFileSync(join(raced.destination, RELEASE_BINDING_FILE), 'raced'); + writeFileSync(path, bytes); + }, + }), + refuses('ALREADY_SEALED'), + ); + assert.equal(readFileSync(join(raced.destination, RELEASE_BINDING_FILE), 'utf8'), 'raced'); + assert.deepEqual( + readdirSync(raced.destination).sort(), + [ciphertextFileName(VERSION), RELEASE_BINDING_FILE, tarballName(VERSION)].sort(), + ); + // A write that fails midway leaves nothing under the final name and no partial sibling. + const failed = await encrypted(); + assert.throws( + () => + sealReleaseBinding(failed.record, { + origin: origin(SYNTHETIC_SOURCE.commit), + write: (path, bytes) => { + writeFileSync(path, bytes.subarray(0, 10)); + throw new Error(`disk full at ${path}`); + }, + }), + (error) => refuses('UNEXPECTED')(error) && !error.message.includes(failed.destination), + ); + assert.deepEqual( + readdirSync(failed.destination).sort(), + [ciphertextFileName(VERSION), tarballName(VERSION)].sort(), + ); + // The record survives a failed seal, so the candidate can still be sealed. + sealReleaseBinding(failed.record, { origin: origin(SYNTHETIC_SOURCE.commit) }); + assert.ok(readdirSync(failed.destination).includes(RELEASE_BINDING_FILE)); + // A link that fails for another reason publishes nothing, reports no host text and keeps the record. + const unlinked = await encrypted(); + const fs = createRequire(import.meta.url)('node:fs'); + const link = fs.linkSync; + fs.linkSync = (existing) => { + throw Object.assign(new Error(`EPERM: ${existing}`), { code: 'EPERM' }); + }; + syncBuiltinESMExports(); + try { + assert.throws( + () => sealReleaseBinding(unlinked.record, { origin: origin(SYNTHETIC_SOURCE.commit) }), + (error) => refuses('UNEXPECTED')(error) && !error.message.includes(unlinked.destination), + ); + } finally { + fs.linkSync = link; + syncBuiltinESMExports(); + } + assert.deepEqual( + readdirSync(unlinked.destination).sort(), + [ciphertextFileName(VERSION), tarballName(VERSION)].sort(), + ); + sealReleaseBinding(unlinked.record, { origin: origin(SYNTHETIC_SOURCE.commit) }); + assert.ok(readdirSync(unlinked.destination).includes(RELEASE_BINDING_FILE)); +}); + +test( + 'a sibling that cannot be removed after the link leaves the receipt published, never a false refusal', + { skip: !posixModes || process.getuid?.() === 0 }, + async () => { + const { destination, record } = await encrypted(); + const fs = createRequire(import.meta.url)('node:fs'); + const link = fs.linkSync; + fs.linkSync = (existing, created) => { + link(existing, created); + chmodSync(destination, 0o500); + }; + syncBuiltinESMExports(); + try { + const sealed = sealReleaseBinding(record, { origin: origin(SYNTHETIC_SOURCE.commit) }); + assert.deepEqual(readFileSync(join(destination, RELEASE_BINDING_FILE)), sealed.bytes); + assert.equal(readdirSync(destination).filter((name) => name.endsWith('.partial')).length, 1); + } finally { + fs.linkSync = link; + syncBuiltinESMExports(); + chmodSync(destination, 0o700); + } + // Published, but not reusable until the sibling is removed: reuse admits exactly three entries. + assert.throws( + () => + verifyRetainedCandidate({ + directory: destination, + expected: { scope: 'synthetic', pins: PINS.good, packageVersion: VERSION }, + }), + refuses('CANDIDATE_ENTRIES'), + ); + }, +); + +test('a destination that stops resolving refuses before any GnuPG call, with a fixed code and no path', async () => { + const { root, destination } = places(); + const fs = createRequire(import.meta.url)('node:fs'); + const realpath = fs.realpathSync; + let calls = 0; + // The gate resolves the destination once, before encryption: nothing after publication can fail on it. + fs.realpathSync = Object.assign( + (path, ...rest) => { + if (path === destination && (calls += 1) === 1) throw new Error(`vanished: ${destination}`); + return realpath(path, ...rest); + }, + { native: realpath.native }, + ); + syncBuiltinESMExports(); + const recorded = recording(); + try { + await assert.rejects( + encryptCandidate({ + scope: synthetic, + container: built.synthetic, + destination, + privateRoot: root, + run: recorded.run, + }), + (error) => refuses('PLACEMENT')(error) && !error.message.includes(destination), + ); + } finally { + fs.realpathSync = realpath; + syncBuiltinESMExports(); + } + assert.equal(calls, 1); + assert.deepEqual(recorded.calls, []); +}); + +test("the caller's container changing during encryption changes nothing encrypted or sealed", async () => { + const ephemeral = ephemeralKey(); + const scope = { kind: 'synthetic', recipient: { file: ephemeral.file, pins: ephemeral.pins } }; + const mine = Buffer.from(built.synthetic); + // Overwritten in place while GnuPG runs, before the gate's copy is written for encryption. + const { run } = recording((args) => { + if (args.includes('--show-keys')) mine.fill(0x20); + }); + const { root, destination } = places(); + const record = await encryptCandidate({ + scope, + container: mine, + destination, + privateRoot: root, + run, + }); + assert.equal(mine[0], 0x20); + const out = fresh('release-copy'); + mkdirSync(out); + ephemeral.decrypt(join(destination, record.name), join(out, 'container.tar')); + assert.deepEqual(readFileSync(join(out, 'container.tar')), built.synthetic); + sealReleaseBinding(record, { origin: origin(SYNTHETIC_SOURCE.commit) }); +}); + +test('the gate forwards the clock and the GnuPG installation to encryptOnce', async () => { + // A clock before the fixture key's creation makes its subkey one created in the future. + const early = places(); + await assert.rejects( + encryptCandidate({ + scope: synthetic, + container: built.synthetic, + destination: early.destination, + privateRoot: early.root, + now: 1, + }), + refuses('RECIPIENT_UNUSABLE'), + ); + const missing = places(); + const absent = { command: join(fresh('no-gpg'), 'gpg'), prefix: [] }; + await assert.rejects( + encryptCandidate({ + scope: synthetic, + container: built.synthetic, + destination: missing.destination, + privateRoot: missing.root, + gpg: { ...absent, gpgconf: absent }, + }), + refuses('GPG_UNAVAILABLE'), + ); +}); + +test('a production candidate round-trips: the receipt binds what the ciphertext decrypts to', async () => { + const ephemeral = ephemeralKey(); + // A rotated history: an earlier recipient retired, the ephemeral key current. + const earlier = { primary: '1'.repeat(40), subkey: '2'.repeat(40) }; + const recipient = loadRecipientForTests({ + root: recipientRoot(ephemeral.file, [ + { ...earlier, status: 'retired' }, + { ...ephemeral.pins, status: 'current' }, + ]), + }); + const { destination, record } = await encrypted({ kind: 'production', recipient }, built.clean); + assert.deepEqual(record.recipient, ephemeral.pins); + const sealed = sealReleaseBinding(record, { origin: origin(CLEAN.commit) }); + const out = fresh('release-round-trip'); + mkdirSync(out); + ephemeral.decrypt(join(destination, record.name), join(out, 'container.tar')); + const decrypted = readFileSync(join(out, 'container.tar')); + assert.deepEqual(decrypted, built.clean); + const { manifest } = readPrivateContainer(decrypted); + const { receipt } = sealed; + assert.equal(receipt.packageVersion, manifest.packageVersion); + assert.deepEqual(receipt.source, manifest.source); + assert.equal(receipt.tarball.sha256, manifest.tarballSha256); + const { gnupg, ...tools } = receipt.toolchain; + assert.deepEqual(tools, manifest.toolchain); + assert.equal(gnupg, record.gnupg); + assert.equal(receipt.target, manifest.target); + assert.deepEqual(receipt.inputs, manifest.inputs); + const expected = { + scope: 'production', + recipient, + packageVersion: VERSION, + source: CLEAN.commit, + }; + const reused = verifyRetainedCandidate({ directory: destination, expected }); + assert.deepEqual(reused.receipt, receipt); + // The production receipt never verifies as synthetic, nor under a history without its recipient. + assert.throws( + () => + verifyRetainedCandidate({ + directory: destination, + expected: { scope: 'synthetic', pins: ephemeral.pins, packageVersion: VERSION }, + }), + refuses('BINDING_SCOPE'), + ); + const rotatedAway = loadRecipientForTests({ + root: recipientRoot(ephemeral.file, [{ ...earlier, status: 'current' }]), + }); + assert.throws( + () => + verifyRetainedCandidate({ + directory: destination, + expected: { ...expected, recipient: rotatedAway }, + }), + refuses('BINDING_RECIPIENT'), + ); +}); diff --git a/tools/cli-package/release-fixture.mjs b/tools/cli-package/release-fixture.mjs new file mode 100644 index 00000000..a2695d94 --- /dev/null +++ b/tools/cli-package/release-fixture.mjs @@ -0,0 +1,86 @@ +/** + * Test-only release candidates (Step 9b.2). The synthetic sealed staging of `candidate-fixture.mjs`, sealed + * with a real tarball digest, gives a synthetic-scope container; a copy whose manifest records a clean, + * non-synthetic source gives a production-scope one (the container checks structure, not provenance). + * Production-scope tests load a temporary recipient root, never the committed path, and never a fixture + * key. Nothing here is a product; only tests import it. + */ +import { Buffer } from 'node:buffer'; +import { chmodSync, copyFileSync, cpSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { candidate, digest, fresh, seal } from './candidate-fixture.mjs'; +import { buildPrivateContainer } from './container.mjs'; + +/** A synthetic tarball: its bytes are never inspected, only named and hashed. */ +export const TARBALL = Buffer.from('a synthetic tarball, never a package\n'); +export const VERSION = '0.1.1'; +/** The clean, non-synthetic source the production-scope container records. */ +export const CLEAN = Object.freeze({ commit: 'd'.repeat(40), tree: 'e'.repeat(40) }); + +/** + * A container of a copy of `staging` whose manifest records `source`. + * @param {string} staging @param {{commit: string, tree: string, dirty: boolean}} source + */ +export function containerWith(staging, source) { + const copy = fresh('release-staging'); + cpSync(staging, copy, { recursive: true }); + const manifest = JSON.parse(readFileSync(join(copy, 'manifest.json'), 'utf8')); + manifest.source = source; + writeFileSync(join(copy, 'manifest.json'), JSON.stringify(manifest)); + return buildPrivateContainer(copy).bytes; +} + +/** + * The sealed synthetic staging, its container, and a clean one, all naming `TARBALL`'s digest. + * @returns {Promise<{staging: string, synthetic: Buffer, clean: Buffer}>} + */ +export async function containers() { + const fixture = await candidate(); + fixture.archive.sha256 = digest(TARBALL); + await seal(fixture); + writeFileSync(join(fixture.staging, 'metafile-esm.json'), '{"inputs":{},"outputs":{}}'); + return { + staging: fixture.staging, + synthetic: buildPrivateContainer(fixture.staging).bytes, + clean: containerWith(fixture.staging, { ...CLEAN, dirty: false }), + }; +} + +/** A fresh owner-only private root and a destination holding the candidate's tarball. */ +export function places() { + const root = fresh('release-root'); + mkdirSync(root); + chmodSync(root, 0o700); + const destination = fresh('release-destination'); + mkdirSync(destination); + writeFileSync(join(destination, `relavium-${VERSION}.tgz`), TARBALL); + return { root, destination }; +} + +/** A synthetic originating GitHub push run for `source`. @param {string} source */ +export const origin = (source) => ({ + kind: 'github', + repository: 'example/relavium', + workflow: 'release', + runId: '1', + runAttempt: '1', + event: 'push', + source, +}); + +/** + * A temporary production recipient root holding `file` and the given history. + * @param {string} file + * @param {{primary: string, subkey: string, status: string}[]} recipients + */ +export function recipientRoot(file, recipients) { + const root = fresh('release-recipient'); + mkdirSync(root); + copyFileSync(file, join(root, 'recipient.asc')); + // The loader admits only the canonical serialization. + writeFileSync( + join(root, 'pins.json'), + `${JSON.stringify({ version: 1, recipients }, null, 2)}\n`, + ); + return root; +} diff --git a/tools/cli-package/release-guard-core.mjs b/tools/cli-package/release-guard-core.mjs new file mode 100644 index 00000000..5a81890e --- /dev/null +++ b/tools/cli-package/release-guard-core.mjs @@ -0,0 +1,45 @@ +/** + * Interim fail-closed publication guard (Wave 3 Step 9a.1, the Interim release guard ruling of 2026-10-19). + * + * npm publication requires a release candidate: the exact tarball, its original ciphertext and a + * release binding receipt verified against the release custody contract + * (docs/reference/cli/packaging.md). Wave 3 Step 9d implements that custody graph and Wave 4b + * supplies the production recipient; until both exist no artifact can be a release candidate, so + * this guard refuses every artifact. A package-test artifact is never promoted to a release. + * + * It depends on Node built-ins only (the publish job checks out nothing else) and reads directory + * entry names only — never artifact contents. `release-guard.mjs` is the publish job's entry point. + */ +import { readdirSync } from 'node:fs'; + +export class ReleaseGuardError extends Error { + /** @param {string} code */ + constructor(code) { + super(`CLI publication refused: ${code}`); + this.name = 'ReleaseGuardError'; + this.code = code; + } +} + +/** + * @param {string} directory the downloaded artifact directory + * @returns {never} until Step 9d's release custody validator replaces this interim guard + * @throws {ReleaseGuardError} for every artifact + */ +export function requireReleaseCandidate(directory) { + let names; + try { + names = readdirSync(directory); + } catch { + throw new ReleaseGuardError('ARTIFACT_UNREADABLE'); + } + if (!names.includes('release-binding-receipt.json')) { + throw new ReleaseGuardError( + names.includes('package-test-receipt.json') + ? 'PACKAGE_TEST_ARTIFACT' + : 'RELEASE_CANDIDATE_REQUIRED', + ); + } + // No release binding receipt is trusted before the Step 9d custody validator exists. + throw new ReleaseGuardError('RELEASE_CUSTODY_UNAVAILABLE'); +} diff --git a/tools/cli-package/release-guard.mjs b/tools/cli-package/release-guard.mjs new file mode 100644 index 00000000..9fc2be77 --- /dev/null +++ b/tools/cli-package/release-guard.mjs @@ -0,0 +1,26 @@ +/** + * The release workflow's publish-job entry point for the interim publication guard + * (`release-guard-core.mjs`; Wave 3 Step 9a.1, the Interim release guard ruling of 2026-10-19). + * + * It refuses before it evaluates anything: the exit code is 1 from the first statement, so a module + * that fails to load, an unexpected throw or a guard that returned would still refuse, and nothing + * about how the script is invoked (a symlinked or relative path) can skip the refusal. Until Step 9d's + * custody validator replaces it, no artifact passes. It prints one fixed line and never artifact + * contents. + */ +import { resolve } from 'node:path'; + +process.exitCode = 1; +let code = 'GUARD_FAILURE'; +try { + const { ReleaseGuardError, requireReleaseCandidate } = await import('./release-guard-core.mjs'); + try { + if (process.argv.length !== 3) throw new ReleaseGuardError('USAGE'); + requireReleaseCandidate(resolve(process.argv[2])); + } catch (error) { + if (error instanceof ReleaseGuardError) code = error.code; + } +} catch { + // A guard that cannot load its rule still refuses. +} +console.error(`::error::CLI publication refused: ${code}`); diff --git a/tools/cli-package/release-guard.test.mjs b/tools/cli-package/release-guard.test.mjs new file mode 100644 index 00000000..6b3c5e00 --- /dev/null +++ b/tools/cli-package/release-guard.test.mjs @@ -0,0 +1,139 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { + copyFileSync, + mkdirSync, + mkdtempSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, relative } from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { ReleaseGuardError, requireReleaseCandidate } from './release-guard-core.mjs'; + +const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-release-guard-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); +const guard = fileURLToPath(new URL('./release-guard.mjs', import.meta.url)); +let count = 0; + +function artifact(files) { + count += 1; + const directory = join(scratch, `artifact-${count}`); + mkdirSync(directory); + for (const [name, contents] of Object.entries(files)) + writeFileSync(join(directory, name), contents); + return directory; +} + +/** Run the guard entry as the publish job does, from `cwd` with the given script path and arguments. */ +function runGuard(script, args, cwd = scratch) { + return spawnSync(process.execPath, [script, ...args], { cwd, encoding: 'utf8', timeout: 10_000 }); +} + +const PACKAGE_TEST = { + 'relavium-0.1.1.tgz': 'tarball', + 'package-test-receipt.json': JSON.stringify({ mode: 'package_test' }), +}; + +for (const [label, files, code] of [ + ['a package-test artifact', PACKAGE_TEST, 'PACKAGE_TEST_ARTIFACT'], + [ + 'a package-test artifact whose receipt claims release mode', + { ...PACKAGE_TEST, 'package-test-receipt.json': JSON.stringify({ mode: 'release' }) }, + 'PACKAGE_TEST_ARTIFACT', + ], + ['a bare tarball', { 'relavium-0.1.1.tgz': 'tarball' }, 'RELEASE_CANDIDATE_REQUIRED'], + ['an empty artifact', {}, 'RELEASE_CANDIDATE_REQUIRED'], + [ + 'a self-declared release candidate', + { + 'relavium-0.1.1.tgz': 'tarball', + 'relavium-0.1.1.tgz.gpg': 'ciphertext', + 'release-binding-receipt.json': JSON.stringify({ mode: 'release' }), + }, + 'RELEASE_CUSTODY_UNAVAILABLE', + ], +]) { + test(`the interim publication guard refuses ${label}`, () => { + const directory = artifact(files); + assert.throws( + () => requireReleaseCandidate(directory), + (error) => error instanceof ReleaseGuardError && error.code === code, + ); + const run = runGuard(guard, [directory]); + assert.equal(run.status, 1); + assert.equal(run.stdout, ''); + assert.equal(run.stderr, `::error::CLI publication refused: ${code}\n`); + }); +} + +test('the guard refuses an unreadable artifact and wrong usage with one fixed line', () => { + const missing = join(scratch, 'absent'); + assert.throws( + () => requireReleaseCandidate(missing), + (error) => error instanceof ReleaseGuardError && error.code === 'ARTIFACT_UNREADABLE', + ); + for (const args of [[], [scratch, scratch]]) { + const run = runGuard(guard, args); + assert.equal(run.status, 1); + assert.equal(run.stderr, '::error::CLI publication refused: USAGE\n'); + } +}); + +test('the guard refuses however its entry point is reached', () => { + const directory = artifact(PACKAGE_TEST); + const expected = '::error::CLI publication refused: PACKAGE_TEST_ARTIFACT\n'; + // A symlinked checkout directory makes the entry's argv path differ from its loaded real path. + const linked = join(scratch, 'linked-tools'); + symlinkSync(dirname(guard), linked, process.platform === 'win32' ? 'junction' : 'dir'); + // A relative, non-canonical path reaches the same file another way. + const relativePath = relative(scratch, join(linked, '..', 'linked-tools', 'release-guard.mjs')); + for (const script of [join(linked, 'release-guard.mjs'), relativePath]) { + const run = runGuard(script, [directory]); + assert.equal(run.status, 1, script); + assert.equal(run.stderr, expected, script); + } +}); + +test('the guard refuses when its rule cannot be loaded', () => { + // The entry point alone, without the core module beside it, still exits 1 with a fixed line. + const lonely = join(scratch, 'lonely'); + mkdirSync(lonely); + copyFileSync(guard, join(lonely, 'release-guard.mjs')); + const run = runGuard(join(lonely, 'release-guard.mjs'), [artifact(PACKAGE_TEST)]); + assert.equal(run.status, 1); + assert.equal(run.stdout, ''); + assert.equal(run.stderr, '::error::CLI publication refused: GUARD_FAILURE\n'); +}); + +for (const [label, rule, stderr] of [ + ['returns', '', '::error::CLI publication refused: GUARD_FAILURE\n'], + [ + 'throws an error that is not its own', + "const error = new Error('x'); error.code = 'ENOENT'; throw error;", + '::error::CLI publication refused: GUARD_FAILURE\n', + ], + // The exit code was set to 1 before the rule ran, so a bare exit keeps it. + ['exits the process itself', 'process.exit();', ''], +]) { + test(`the guard refuses when its rule ${label}`, () => { + const directory = join(scratch, `rule-${label.replaceAll(' ', '-')}`); + mkdirSync(directory); + copyFileSync(guard, join(directory, 'release-guard.mjs')); + writeFileSync( + join(directory, 'release-guard-core.mjs'), + 'export class ReleaseGuardError extends Error {\n' + + ' constructor(code) { super(code); this.code = code; }\n' + + '}\n' + + `export function requireReleaseCandidate() { ${rule} }\n`, + ); + const run = runGuard(join(directory, 'release-guard.mjs'), [artifact(PACKAGE_TEST)]); + assert.equal(run.status, 1); + assert.equal(run.stdout, ''); + assert.equal(run.stderr, stderr); + }); +} diff --git a/tools/cli-package/release-recipient.mjs b/tools/cli-package/release-recipient.mjs new file mode 100644 index 00000000..adaae979 --- /dev/null +++ b/tools/cli-package/release-recipient.mjs @@ -0,0 +1,187 @@ +/** + * The production recipient and the release scopes (Step 9b.2; packaging.md, "Recipient custody + * implementation boundary — Step 9b.2"). One committed directory holds the production recipient: + * `recipient.asc`, the `gpg --armor --export '!'` export, and `pins.json`, an append-only history in + * which exactly one recipient is current and none is ever removed, so an earlier candidate still verifies + * after a rotation. There is no environment variable, flag or fallback. In Wave 3 the directory does not + * exist, so every production use refuses; Wave 4b commits the files. A loaded recipient is branded: only the + * value this module returned counts as a production recipient, never a copy of it. + */ +import { Buffer } from 'node:buffer'; +import { lstatSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { isDeepStrictEqual } from 'node:util'; +import { CustodyError, refuseCustody } from './custody-error.mjs'; +import { readOrdinaryFile, byCodeUnit } from './files.mjs'; + +/** The committed production recipient directory, without a trailing separator so a link is never followed. */ +export const PRODUCTION_RECIPIENT_ROOT = fileURLToPath( + new URL('./release-recipient', import.meta.url), +); +/** The committed synthetic fixtures' pins: no production history may name one of their keys. */ +export const FIXTURE_PINS_FILE = fileURLToPath( + new URL('./custody-fixtures/pins.json', import.meta.url), +); +/** The source every synthetic candidate the fixtures seal records: never a real commit or tree. */ +export const SYNTHETIC_SOURCE = Object.freeze({ commit: 'c'.repeat(40), tree: 'b'.repeat(40) }); + +const FILES = Object.freeze(['pins.json', 'recipient.asc']); +const FILE_BYTES = 64 * 1024; +const MAX_RECIPIENTS = 32; +const FINGERPRINT = /^[0-9A-F]{40}$/u; +/** @type {WeakSet} */ +const loaded = new WeakSet(); + +/** + * @typedef {{primary: string, subkey: string}} Pins + * @typedef {{file: string, current: Pins, history: readonly (Pins & {status: 'current' | 'retired'})[]}} + * ProductionRecipient + */ + +/** @param {unknown} value @returns {value is Record} */ +const isRecord = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); +/** @param {unknown} value */ +const isFingerprint = (value) => typeof value === 'string' && FINGERPRINT.test(value); + +/** Whether `source` is the synthetic candidates' sentinel source. @param {{commit: string, tree: string}} source */ +export const isSyntheticSource = (source) => + source.commit === SYNTHETIC_SOURCE.commit && source.tree === SYNTHETIC_SOURCE.tree; + +/** Whether `value` is a recipient this module loaded: a copy or a hand-built object is not. */ +export const isProductionRecipient = (value) => loaded.has(value); + +/** @param {Buffer} bytes */ +function parseHistory(bytes) { + let value; + try { + value = JSON.parse(bytes.toString('utf8')); + } catch { + return refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + } + // The canonical comparison below fixes the version and every key set; these checks make the history + // safe to build and state its rules. + const recipients = isRecord(value) && Array.isArray(value.recipients) ? value.recipients : []; + if (recipients.length > MAX_RECIPIENTS) refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + const history = recipients.map((entry) => { + const { primary, subkey, status } = isRecord(entry) ? entry : {}; + if ( + !isFingerprint(primary) || + !isFingerprint(subkey) || + (status !== 'current' && status !== 'retired') + ) { + refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + } + return Object.freeze({ primary, subkey, status }); + }); + if ( + history.filter((entry) => entry.status === 'current').length !== 1 || + new Set(history.map((entry) => entry.subkey)).size !== history.length + ) { + refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + } + // Exactly its canonical serialization, so the file a reviewer reads is the history loaded: a + // duplicated key, which JSON.parse would resolve silently to its last value, or another order or + // layout refuses. + const canonical = Buffer.from( + `${JSON.stringify({ version: 1, recipients: history }, null, 2)}\n`, + ); + if (!canonical.equals(bytes)) refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + return Object.freeze(history); +} + +/** + * Every fingerprint the committed fixtures' pins file lists. A file that is not exactly a non-empty map of + * named primary and subkey pins refuses rather than yield a weaker denylist. + * @param {string} file + */ +function fixtureFingerprints(file) { + let pins; + try { + pins = JSON.parse(readOrdinaryFile(file, FILE_BYTES).toString('utf8')); + } catch { + return refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + } + const entries = isRecord(pins) ? Object.values(pins) : []; + if ( + entries.length === 0 || + !entries.every( + (entry) => + isRecord(entry) && + isDeepStrictEqual(Object.keys(entry).sort(byCodeUnit), ['primary', 'subkey']) && + isFingerprint(entry.primary) && + isFingerprint(entry.subkey), + ) + ) { + refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + } + return new Set(entries.flatMap((entry) => [entry.primary, entry.subkey])); +} + +/** + * Load the production recipient from the committed directory. It takes no argument: there is no other + * place a production recipient can come from. + * @returns {ProductionRecipient} + */ +export function loadProductionRecipient() { + return guarded(PRODUCTION_RECIPIENT_ROOT, FIXTURE_PINS_FILE); +} + +/** + * The tests' seam: load a recipient from a temporary directory. No product or tooling code may call it; + * release-scope.test.mjs pins that permanently. + * @param {{root: string, fixtures?: string}} options + * @returns {ProductionRecipient} + */ +export function loadRecipientForTests({ root, fixtures = FIXTURE_PINS_FILE } = {}) { + return guarded(root, fixtures); +} + +/** @param {string} root @param {string} fixtures @returns {ProductionRecipient} */ +function guarded(root, fixtures) { + try { + return load(root, fixtures); + } catch (error) { + // A host error's message names a path: only fixed custody codes leave. + if (error instanceof CustodyError) throw error; + return refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + } +} + +/** @param {string} root @param {string} fixtures @returns {ProductionRecipient} */ +function load(root, fixtures) { + let info; + try { + info = lstatSync(root); + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'ENOENT') { + refuseCustody('PRODUCTION_RECIPIENT_ABSENT'); + } + throw error; + } + // lstat: a linked directory is not the committed one. + if (!info.isDirectory() || !isDeepStrictEqual(readdirSync(root).sort(), FILES)) { + refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + } + const file = join(root, 'recipient.asc'); + let pins; + try { + readOrdinaryFile(file, FILE_BYTES); + pins = readOrdinaryFile(join(root, 'pins.json'), FILE_BYTES); + } catch { + refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + } + const history = parseHistory(pins); + const forbidden = fixtureFingerprints(fixtures); + if (history.some((entry) => forbidden.has(entry.primary) || forbidden.has(entry.subkey))) { + refuseCustody('FIXTURE_RECIPIENT'); + } + const current = history.find((entry) => entry.status === 'current'); + const recipient = Object.freeze({ + file, + current: Object.freeze({ primary: current.primary, subkey: current.subkey }), + history, + }); + loaded.add(recipient); + return recipient; +} diff --git a/tools/cli-package/release-recipient.test.mjs b/tools/cli-package/release-recipient.test.mjs new file mode 100644 index 00000000..a33a3622 --- /dev/null +++ b/tools/cli-package/release-recipient.test.mjs @@ -0,0 +1,300 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { Buffer } from 'node:buffer'; +import { chmodSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join } from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { CustodyError } from './custody-error.mjs'; +import { PINS } from './gpg-fixture.mjs'; +import { + FIXTURE_PINS_FILE, + PRODUCTION_RECIPIENT_ROOT, + SYNTHETIC_SOURCE, + isProductionRecipient, + isSyntheticSource, + loadProductionRecipient, + loadRecipientForTests, +} from './release-recipient.mjs'; + +const scratch = mkdtempSync(join(tmpdir(), 'relavium-release-recipient-')); +after(() => rmSync(scratch, { recursive: true, force: true })); +let count = 0; +const fresh = (name) => join(scratch, `${name}-${(count += 1)}`); +const refuses = (code) => (error) => + error instanceof CustodyError && + error.code === code && + /^PACKED_CLI_CUSTODY_[A-Z_]+$/u.test(error.message) && + !error.message.includes(scratch); +const posix = process.platform !== 'win32'; + +const A = { primary: '1'.repeat(40), subkey: '2'.repeat(40) }; +const B = { primary: '1'.repeat(40), subkey: '3'.repeat(40) }; + +/** A recipient root with the given files; `pins` may be any value or raw text. */ +function root({ + pins = { version: 1, recipients: [{ ...A, status: 'current' }] }, + key = 'key', +} = {}) { + const path = fresh('root'); + mkdirSync(path); + writeFileSync(join(path, 'recipient.asc'), key); + writeFileSync( + join(path, 'pins.json'), + typeof pins === 'string' ? pins : `${JSON.stringify(pins, null, 2)}\n`, + ); + return path; +} +const history = (...recipients) => ({ version: 1, recipients }); + +test('the committed recipient directory is absent in Wave 3, so every production use refuses', () => { + // Interim: Wave 4b commits the directory and turns this into a load of the real recipient. + assert.equal(basename(PRODUCTION_RECIPIENT_ROOT), 'release-recipient'); + // No trailing separator: lstat of `dir/` would follow a link to another directory. + assert.equal(/[\\/]$/u.test(PRODUCTION_RECIPIENT_ROOT), false); + assert.equal( + dirname(PRODUCTION_RECIPIENT_ROOT), + fileURLToPath(new URL('.', import.meta.url)).replace(/[\\/]$/u, ''), + ); + assert.throws(() => loadProductionRecipient(), refuses('PRODUCTION_RECIPIENT_ABSENT')); + // It takes no argument: a root passed to it is ignored, never an override. + assert.throws( + () => loadProductionRecipient({ root: root() }), + refuses('PRODUCTION_RECIPIENT_ABSENT'), + ); + assert.throws( + () => loadRecipientForTests({ root: join(fresh('missing'), 'release-recipient') }), + refuses('PRODUCTION_RECIPIENT_ABSENT'), + ); + // Only a missing root is absent: any other failure to reach it, such as a parent that is a file, is not. + const parentFile = fresh('parent-file'); + writeFileSync(parentFile, ''); + assert.throws( + () => loadRecipientForTests({ root: join(parentFile, 'release-recipient') }), + refuses('PRODUCTION_RECIPIENT_INVALID'), + ); +}); + +test('a loaded recipient is frozen, branded, and its current pins come from an append-only history', () => { + const path = root({ pins: history({ ...A, status: 'retired' }, { ...B, status: 'current' }) }); + const recipient = loadRecipientForTests({ root: path }); + assert.equal(recipient.file, join(path, 'recipient.asc')); + assert.deepEqual(recipient.current, B); + assert.deepEqual(recipient.history, [ + { ...A, status: 'retired' }, + { ...B, status: 'current' }, + ]); + assert.ok(Object.isFrozen(recipient) && Object.isFrozen(recipient.current)); + assert.ok(Object.isFrozen(recipient.history) && recipient.history.every(Object.isFrozen)); + assert.equal(isProductionRecipient(recipient), true); + // Only the loaded value is a production recipient: never a copy or a hand-built object. + for (const value of [ + { ...recipient }, + globalThis.structuredClone(recipient), + { file: recipient.file, current: recipient.current, history: recipient.history }, + null, + undefined, + 'recipient', + ]) { + assert.equal(isProductionRecipient(value), false); + } + // Each load brands its own value. + const again = loadRecipientForTests({ root: path }); + assert.notEqual(again, recipient); + assert.equal(isProductionRecipient(again), true); +}); + +test('a recipient directory that is not exactly the two committed files refuses', () => { + const valid = root(); + const cases = [ + () => { + const file = fresh('file-root'); + writeFileSync(file, ''); + return file; + }, + () => { + const path = root(); + writeFileSync(join(path, 'extra'), ''); + return path; + }, + () => { + const path = root(); + rmSync(join(path, 'pins.json')); + return path; + }, + () => { + const path = root(); + rmSync(join(path, 'recipient.asc')); + return path; + }, + () => root({ key: Buffer.alloc(64 * 1024 + 1) }), + () => + root({ + pins: JSON.stringify(history({ ...A, status: 'current' })).padEnd(64 * 1024 + 1, ' '), + }), + ]; + if (posix) { + cases.push( + () => { + const link = fresh('linked-root'); + symlinkSync(valid, link); + return link; + }, + () => { + const path = root(); + rmSync(join(path, 'recipient.asc')); + symlinkSync(join(valid, 'recipient.asc'), join(path, 'recipient.asc')); + return path; + }, + () => { + const path = root(); + rmSync(join(path, 'pins.json')); + assert.equal(spawnSync('mkfifo', [join(path, 'pins.json')]).status, 0); + return path; + }, + ); + } + for (const [index, make] of cases.entries()) { + assert.throws( + () => loadRecipientForTests({ root: make() }), + refuses('PRODUCTION_RECIPIENT_INVALID'), + String(index), + ); + } +}); + +test('the pins history is exactly one current recipient among at most 32 distinct ones', () => { + const current = { ...A, status: 'current' }; + for (const pins of [ + 'not json', + '[]', + history(), + { version: 2, recipients: [current] }, + { version: 1, recipients: [current], extra: true }, + { recipients: [current] }, + { version: 1, recipients: {} }, + history({ ...A, status: 'retired' }), + history(current, { ...B, status: 'current' }), + history(current, { primary: B.primary, subkey: A.subkey, status: 'retired' }), + history({ ...A, status: 'pending' }), + history(current, { ...B, status: 'pending' }), + history({ ...A, status: 'current', extra: true }), + history({ primary: A.primary, status: 'current' }), + history({ primary: 'a'.repeat(40), subkey: A.subkey, status: 'current' }), + history({ primary: [A.primary], subkey: A.subkey, status: 'current' }), + history({ primary: A.primary, subkey: `${A.subkey}!`, status: 'current' }), + history({ primary: `X${A.primary}`, subkey: A.subkey, status: 'current' }), + // Only the canonical serialization: a compact layout, a missing final newline, another key order or a + // duplicated key, which JSON.parse would resolve silently to its last value. + JSON.stringify(history({ ...A, status: 'current' })), + JSON.stringify(history({ ...A, status: 'current' }), null, 2), + `${JSON.stringify({ recipients: [{ ...A, status: 'current' }], version: 1 }, null, 2)}\n`, + `${JSON.stringify(history({ status: 'current', ...A }), null, 2)}\n`, + `${JSON.stringify(history({ ...A, status: 'retired' }), null, 2)}\n`.replace( + '"status": "retired"', + '"status": "retired",\n "status": "current"', + ), + history( + current, + ...Array.from({ length: 32 }, (_, index) => ({ + primary: A.primary, + subkey: index.toString(16).toUpperCase().padStart(40, 'F'), + status: 'retired', + })), + ), + ]) { + assert.throws( + () => loadRecipientForTests({ root: root({ pins }) }), + refuses('PRODUCTION_RECIPIENT_INVALID'), + typeof pins === 'string' ? pins : JSON.stringify(pins).slice(0, 120), + ); + } + // Exactly 32 entries is within the bound. + const full = history( + current, + ...Array.from({ length: 31 }, (_, index) => ({ + primary: A.primary, + subkey: index.toString(16).toUpperCase().padStart(40, 'F'), + status: 'retired', + })), + ); + assert.equal(loadRecipientForTests({ root: root({ pins: full }) }).history.length, 32); +}); + +test('a history naming a committed fixture key refuses, whichever fingerprint it is', () => { + const fixture = PINS.good; + for (const entry of [ + { primary: fixture.primary, subkey: A.subkey }, + { primary: A.primary, subkey: fixture.subkey }, + ]) { + assert.throws( + () => + loadRecipientForTests({ + root: root({ pins: history({ ...entry, status: 'current' }) }), + }), + refuses('FIXTURE_RECIPIENT'), + ); + } + // A retired fixture entry refuses as well. + assert.throws( + () => + loadRecipientForTests({ + root: root({ + pins: history({ ...A, status: 'current' }, { ...PINS.other, status: 'retired' }), + }), + }), + refuses('FIXTURE_RECIPIENT'), + ); + // The denylist is the committed fixtures file; an unreadable or malformed one fails closed. + assert.equal( + FIXTURE_PINS_FILE, + fileURLToPath(new URL('./custody-fixtures/pins.json', import.meta.url)), + ); + const fixtureFiles = [join(fresh('none'), 'pins.json')]; + for (const content of [ + '[]', + '{}', + 'not json', + JSON.stringify({ good: { primary: A.primary } }), + JSON.stringify({ good: { ...A, extra: 'x' } }), + JSON.stringify({ good: { primary: 'a'.repeat(40), subkey: A.subkey } }), + JSON.stringify({ good: { primary: A.primary, subkey: 'a'.repeat(40) } }), + JSON.stringify({ good: A, bad: null }), + ]) { + const file = fresh('fixtures.json'); + writeFileSync(file, content); + fixtureFiles.push(file); + } + for (const fixtures of fixtureFiles) { + assert.throws( + () => loadRecipientForTests({ root: root(), fixtures }), + refuses('PRODUCTION_RECIPIENT_INVALID'), + ); + } +}); + +test( + 'a recipient directory that cannot be read refuses with a fixed code and no path', + { skip: !posix || process.getuid?.() === 0 }, + () => { + const path = root(); + chmodSync(path, 0o000); + try { + assert.throws( + () => loadRecipientForTests({ root: path }), + refuses('PRODUCTION_RECIPIENT_INVALID'), + ); + } finally { + chmodSync(path, 0o700); + } + }, +); + +test('the synthetic source is the fixtures sentinel, never a clean source', () => { + assert.deepEqual(SYNTHETIC_SOURCE, { commit: 'c'.repeat(40), tree: 'b'.repeat(40) }); + assert.ok(Object.isFrozen(SYNTHETIC_SOURCE)); + assert.equal(isSyntheticSource({ commit: 'c'.repeat(40), tree: 'b'.repeat(40) }), true); + assert.equal(isSyntheticSource({ commit: 'c'.repeat(40), tree: 'e'.repeat(40) }), false); + assert.equal(isSyntheticSource({ commit: 'd'.repeat(40), tree: 'b'.repeat(40) }), false); +}); diff --git a/tools/cli-package/release-scope.test.mjs b/tools/cli-package/release-scope.test.mjs new file mode 100644 index 00000000..6c37ac81 --- /dev/null +++ b/tools/cli-package/release-scope.test.mjs @@ -0,0 +1,381 @@ +import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join, posix, relative } from 'node:path'; +import { test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { COMMIT, TREE, fresh } from './candidate-fixture.mjs'; +import { CONTAINER_LIMITS } from './container.mjs'; +import { ciphertextName } from './custody.mjs'; +import { sha256 } from './files.mjs'; +import { + CIPHERTEXT_BYTES, + RELEASE_BINDING_FILE, + ciphertextFileName, + serializeReleaseBinding, + tarballName, + validateReleaseBinding, + verifyRetainedCandidate, +} from './release-binding.mjs'; +import { ReleaseGuardError, requireReleaseCandidate } from './release-guard-core.mjs'; +import { PRODUCTION_RECIPIENT_ROOT, SYNTHETIC_SOURCE } from './release-recipient.mjs'; + +const repository = fileURLToPath(new URL('../../', import.meta.url)); +const here = fileURLToPath(new URL('.', import.meta.url)); +/** Tracked and new, unignored source files, as repository-relative POSIX paths. */ +const tracked = spawnSync( + 'git', + [ + 'ls-files', + '-z', + '--cached', + '--others', + '--exclude-standard', + '--', + '*.mjs', + '*.cjs', + '*.js', + '*.ts', + '*.tsx', + '*.mts', + '*.yml', + '*.yaml', + // Package scripts can name a module too. + 'package.json', + '*/package.json', + ], + { cwd: repository, encoding: 'utf8' }, +) + .stdout.split('\0') + .filter(Boolean); +/** Test, fixture and maintainer-only files: everything else is product or tooling code. */ +const isTestOnly = (path) => + /\.test\.[cm]?[jt]sx?$/u.test(path) || + /-fixture\.mjs$/u.test(path) || + /(?:^|\/)(?:test|tests|__tests__|lint-fixtures)\//u.test(path) || + path.startsWith('tools/cli-package/custody-fixtures/'); +const code = tracked.filter((path) => !isTestOnly(path)); +const read = (path) => readFileSync(join(repository, path), 'utf8'); +/** The code files whose text matches `pattern`. @param {RegExp} pattern */ +const naming = (pattern) => code.filter((path) => pattern.test(read(path))).sort(); + +test('the scan sees the custody modules and leaves the tests out', () => { + for (const path of [ + 'tools/cli-package/custody.mjs', + 'tools/cli-package/release-custody.mjs', + 'tools/cli-package/release-guard-core.mjs', + '.github/workflows/release.yml', + ]) { + assert.ok(code.includes(path), path); + } + for (const path of [ + 'tools/cli-package/release-custody.test.mjs', + 'tools/cli-package/release-fixture.mjs', + 'tools/cli-package/custody-fixtures/generate.mjs', + ]) { + assert.ok(tracked.includes(path) && !code.includes(path), path); + } +}); + +test('no product or tooling code reaches a test fixture, and only the loader reads the fixture pins', () => { + assert.deepEqual(naming(/[A-Za-z0-9_-]+-fixture\.mjs/u), []); + assert.deepEqual(naming(/custody-fixtures/u), ['tools/cli-package/release-recipient.mjs']); +}); + +/** + * Whether `text` mentions a file named `name` anywhere, quoted or not, comments included: a package script + * runs in its package and a workflow step in its working directory, and a template literal can build the + * rest of the path, so no resolution is attempted. JavaScript and percent escapes are decoded, separators + * normalized and case ignored first. Only one tracked file has the name, so any mention is a reference. + * @param {string} text @param {string} name + */ +function namesFile(text, name) { + const decoded = text + .replace(/\\\r?\n/gu, '') + .replace(/\\x([0-9a-f]{2})|\\u\{([0-9a-f]+)\}|\\u([0-9a-f]{4})/giu, (_, x, braced, u) => + String.fromCodePoint(Number.parseInt(x ?? braced ?? u, 16)), + ) + .replace(/%([0-9a-f]{2})/giu, (_, hex) => String.fromCharCode(Number.parseInt(hex, 16))) + .replaceAll('\\', '/') + .toLowerCase(); + const escaped = name.toLowerCase().replaceAll('.', '\\.'); + return new RegExp(`(? { + for (const text of [ + "import { encryptOnce } from '../custody.mjs';", + 'const m = await import("../../cli-package/custody.mjs?v=1");', + "import { encryptOnce } from './custody.mjs#fragment';", + "import { encryptOnce } from './custody%2Emjs';", + "import { encryptOnce } from './%63ustody.mjs';", + "import { encryptOnce } from './custody\\u002emjs';", + "import { encryptOnce } from './custody\\x2emjs';", + "import { encryptOnce } from './cus\\u{74}ody.mjs';", + "const url = new URL('custody.mjs', import.meta.url);", + "const url = new URL('.\\\\custody.mjs', import.meta.url);", + "const url = new URL('Custody.mjs', import.meta.url);", + 'const m = (here, base) => import(`${join(here, base)}/custody.mjs`);', + 'run: node -e "import(\'./tools/cli-package/custody.mjs\')"', + '"a": "node --import ../../tools/cli-package/custody.mjs run.mjs"', + '"b": "node -e \\"import(process.argv[1])\\" ../tools/cli-package/custody.mjs"', + 'run: node tools/cli-package/custody.mjs', + '// encryptOnce lives in custody.mjs', + "export * as custody from '../../tools/cli-package/custody.mjs';", + 'require(`./custody.mjs`)', + ]) { + assert.equal(namesFile(text, 'custody.mjs'), true, text); + } + for (const text of [ + "import { refuseCustody } from './custody-error.mjs';", + "import { encryptCandidate } from './release-custody.mjs';", + "import { x } from './custody.mjs.bak';", + "import { x } from './my-custody.mjs';", + "import { x } from './custodyxmjs';", + ]) { + assert.equal(namesFile(text, 'custody.mjs'), false, text); + } +}); + +test('only the release gate refers to custody.mjs, by any spelling', () => { + assert.equal( + tracked.filter((path) => posix.basename(path).toLowerCase() === 'custody.mjs').length, + 1, + ); + assert.deepEqual(code.filter((path) => namesFile(read(path), 'custody.mjs')).sort(), [ + 'tools/cli-package/release-custody.mjs', + ]); +}); + +test('the loader takes no argument, and no product or tooling code reaches its test seam', () => { + // Permanent: these controls stay when Step 9d updates the interim one below. + assert.deepEqual(naming(/\bloadRecipientForTests\b/u), [ + 'tools/cli-package/release-recipient.mjs', + ]); + assert.deepEqual(naming(/\bloadProductionRecipient\s*\(\s*[^)\s]/u), []); +}); + +test('no product path or workflow names a release module yet', () => { + // Interim: Step 9d wires the release modules into the originating job and updates this control. + assert.deepEqual(naming(/release-(?:custody|binding|recipient)\.mjs/u), [ + 'tools/cli-package/release-binding.mjs', + 'tools/cli-package/release-custody.mjs', + ]); +}); + +/** + * Whether every dynamic import in `source` takes exactly one string literal, so that a closure scan + * reads every module the source can load. + * @param {string} source + */ +function dynamicImportsAreLiteral(source) { + for (const match of source.matchAll(/\bimport\s*\(/gu)) { + const rest = source.slice(match.index + match[0].length); + if (!/^\s*(['"])[^'"\n]+\1\s*\)/u.test(rest)) return false; + } + return true; +} + +test('a dynamic import is admitted only as one string literal', () => { + for (const source of [ + "await import('./archive.mjs');", + 'import("node:fs")', + "import ( './x.mjs' )", + ]) { + assert.equal(dynamicImportsAreLiteral(source), true, source); + } + for (const source of [ + "import('node:child' + '_process')", + "import('./' + name)", + 'import(`./x.mjs`)', + 'import(name)', + "import('./x.mjs', { with: {} })", + ]) { + assert.equal(dynamicImportsAreLiteral(source), false, source); + } +}); + +/** + * Every module specifier `source` names: each string literal right after `from` or `import`, with or + * without a parenthesis, whatever the spacing, so a compact, spaced or multi-line import is read too. + * @param {string} source + */ +function specifiersOf(source) { + // A keyword, never a member call such as `Buffer.from('…')`. + return [...source.matchAll(/(? match[2], + ); +} + +test('the specifier scan reads every spelling of an import', () => { + assert.deepEqual( + specifiersOf( + [ + "import { a } from './a.mjs';", + "import{b}from'./b.mjs';", + "import 'node:fs';", + "import {\n c, // the container's\n} from './c.mjs';", + "export * as d from './d.mjs';", + "export { e } from './e.mjs';", + "const f = await import ( 'node:child_process' );", + 'const g = import("./g.mjs");', + "const magic = Buffer.from('ustar\\0');", + ].join('\n'), + ), + [ + './a.mjs', + './b.mjs', + 'node:fs', + './c.mjs', + './d.mjs', + './e.mjs', + 'node:child_process', + './g.mjs', + ], + ); +}); + +/** Every specifier a module's import closure reaches, relative files followed. @param {string} entry */ +function closure(entry) { + const seen = new Set(); + const specifiers = new Set(); + const visit = (path) => { + if (seen.has(path)) return; + seen.add(path); + const source = readFileSync(path, 'utf8'); + for (const specifier of specifiersOf(source)) { + specifiers.add(specifier); + if (specifier.startsWith('.')) visit(join(dirname(path), specifier)); + } + }; + visit(join(here, entry)); + return { + files: [...seen].map((path) => relative(here, path).split('\\').join('/')).sort(), + specifiers: [...specifiers].sort(), + }; +} + +test("the binding receipt's closure starts no process, so reuse makes no GnuPG call", () => { + const { files, specifiers } = closure('release-binding.mjs'); + assert.deepEqual(files, [ + 'archive.mjs', + 'custody-error.mjs', + 'files.mjs', + 'release-binding.mjs', + 'release-recipient.mjs', + 'ustar.mjs', + ]); + // Every specifier is one of those files or an allowed builtin: no package, no process module. + const builtins = [ + 'node:assert/strict', + 'node:buffer', + 'node:crypto', + 'node:fs', + 'node:path', + 'node:url', + 'node:util', + 'node:zlib', + ]; + for (const specifier of specifiers) { + assert.ok(specifier.startsWith('./') || builtins.includes(specifier), specifier); + } + // Nor can a closure file reach a process or a module without a literal import. + for (const file of files) { + const source = readFileSync(join(here, file), 'utf8'); + assert.doesNotMatch( + source, + /\b(?:getBuiltinModule|createRequire|process\.binding|process\.dlopen|process\.execve)\b|\brequire\s*\(/u, + file, + ); + assert.equal(dynamicImportsAreLiteral(source), true, file); + } + // The scan reads the imports it claims to: the gate's closure reaches GnuPG. + const gate = closure('release-custody.mjs'); + for (const reached of ['custody.mjs', 'gnupg.mjs', 'process.mjs', 'container.mjs']) { + assert.ok(gate.files.includes(reached), reached); + } + assert.ok(gate.specifiers.includes('node:child_process')); +}); + +test('no production recipient is committed in Wave 3', () => { + // Interim: Wave 4b commits the recipient directory and updates this control. + assert.equal(existsSync(PRODUCTION_RECIPIENT_ROOT), false); + const listed = spawnSync( + 'git', + ['ls-files', '--', posix.join('tools/cli-package', 'release-recipient')], + { cwd: repository, encoding: 'utf8' }, + ); + assert.equal(listed.status, 0); + assert.equal(listed.stdout, ''); +}); + +test('the interim release guard still refuses a complete, valid synthetic candidate', () => { + const directory = fresh('guarded-candidate'); + mkdirSync(directory); + const tarball = Buffer.from('a synthetic tarball, never a package\n'); + const ciphertext = Buffer.from('synthetic ciphertext bytes'); + const pins = { primary: '4'.repeat(40), subkey: '5'.repeat(40) }; + const expected = { scope: 'synthetic', pins, packageVersion: '0.1.1' }; + const receipt = validateReleaseBinding( + { + version: 1, + kind: 'release_binding', + packageVersion: '0.1.1', + source: { ...SYNTHETIC_SOURCE, dirty: true }, + tarball: { name: tarballName('0.1.1'), bytes: tarball.length, sha256: sha256(tarball) }, + ciphertext: { + name: ciphertextFileName('0.1.1'), + bytes: ciphertext.length, + sha256: sha256(ciphertext), + }, + recipient: pins, + toolchain: { + node: 'v22.23.1', + pnpm: '9.12.3', + typescript: '5.6.3', + tsup: '8.3.5', + esbuild: '0.24.0', + gnupg: '2.5.21', + }, + target: 'node22', + inputs: { 'package.json': 'a'.repeat(64) }, + mapCheck: { status: 'complete' }, + origin: { + kind: 'github', + repository: 'example/relavium', + workflow: 'release', + runId: '1', + runAttempt: '1', + event: 'push', + source: SYNTHETIC_SOURCE.commit, + }, + }, + expected, + ); + writeFileSync(join(directory, RELEASE_BINDING_FILE), serializeReleaseBinding(receipt)); + writeFileSync(join(directory, tarballName('0.1.1')), tarball); + writeFileSync(join(directory, ciphertextFileName('0.1.1')), ciphertext); + // The candidate is complete and reusable as it stands, and the guard still refuses it. + verifyRetainedCandidate({ directory, expected }); + assert.throws( + () => requireReleaseCandidate(directory), + (error) => error instanceof ReleaseGuardError && error.code === 'RELEASE_CUSTODY_UNAVAILABLE', + ); +}); + +test('the constants the release modules restate equal their sources', () => { + for (const version of ['0.1.1', '1.2.3-rc.1', '2.0.0+build.5']) { + assert.equal(ciphertextFileName(version), ciphertextName(version)); + } + // custody.mjs keeps its overhead private: read it from the source, so both halves have a source. + const overhead = /^const CIPHERTEXT_OVERHEAD = (\d+) \* (\d+);$/mu.exec( + read('tools/cli-package/custody.mjs'), + ); + assert.ok(overhead, 'custody.mjs no longer declares CIPHERTEXT_OVERHEAD as a product'); + assert.equal( + CIPHERTEXT_BYTES, + CONTAINER_LIMITS.totalBytes + Number(overhead[1]) * Number(overhead[2]), + ); + assert.deepEqual(SYNTHETIC_SOURCE, { commit: COMMIT, tree: TREE }); +}); diff --git a/tools/cli-package/seal.mjs b/tools/cli-package/seal.mjs index 1af581b2..9837d8cc 100644 --- a/tools/cli-package/seal.mjs +++ b/tools/cli-package/seal.mjs @@ -1,17 +1,20 @@ -/** Bind the sole prepack result to independently inspected bytes. No build or archive extraction. */ +/** + * Bind the sole prepack result to independently inspected bytes. No build or archive extraction. + * Every input is explicit: the owned candidate root, its private staging directory and the source + * identity the producer captured from the original checkout. Nothing resolves a repository from + * this module's own location, so a candidate copy cannot be sealed against different files. + */ import assert from 'node:assert/strict'; import { createRequire } from 'node:module'; -import { readFileSync, readdirSync, writeFileSync } from 'node:fs'; -import { basename, join, resolve } from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; +import { lstatSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { basename, isAbsolute, join } from 'node:path'; import { inspectPackageArchive } from './archive.mjs'; -import { originFromEnvironment, validatePackageReceipt } from './binding.mjs'; -import { flatFiles, oneTarball, readOrdinaryFile } from './files.mjs'; -import { isolatedEnvironment, runCommand } from './process.mjs'; +import { PACKAGE_TEST_MODE, originFromEnvironment, validatePackageReceipt } from './binding.mjs'; +import { flatFiles, oneTarball, readOrdinaryFile, sha256, byCodeUnit } from './files.mjs'; +import { MANIFEST_FILE, RECORD_BYTES, sealPrivateManifest, verifySealedDist } from './manifest.mjs'; +import { symbolicateCandidate } from './symbolicate.mjs'; -const repository = fileURLToPath(new URL('../../', import.meta.url)); - -export function producerNodeMajor(root = repository) { +export function producerNodeMajor(root) { const declared = readOrdinaryFile(join(root, '.nvmrc'), 20).toString('utf8').trim(); assert.match(declared, /^[1-9]\d*$/u, 'PACKED_CLI_INVALID_PRODUCER_NODE'); const major = Number(declared); @@ -22,7 +25,7 @@ export function producerNodeMajor(root = repository) { return major; } -export function expectedPackManifest(root = repository) { +export function expectedPackManifest(root) { const require = createRequire(join(root, 'apps/cli/package.json')); const { parse } = require('yaml'); const catalog = parse(readFileSync(join(root, 'pnpm-workspace.yaml'), 'utf8')).catalog; @@ -48,21 +51,40 @@ export function expectedPackManifest(root = repository) { return manifest; } -export function finalizedPublicFiles(root = repository) { +/** + * The finalized public files of a candidate: exactly the runtime outputs named by its staged + * metafile, byte-identical to the finalizer's private hash record, plus canonical migrations and + * the public documents. The finalized dist holds no map and no build metadata. + */ +export function finalizedPublicFiles(root, staging) { const cli = join(root, 'apps/cli'); const files = new Map(); - const graph = JSON.parse(readOrdinaryFile(join(cli, 'dist/metafile-esm.json'), 4 * 1024 * 1024)); - const outputs = Object.keys(graph.outputs).map((path) => path.replaceAll('\\', '/')); + const graph = JSON.parse( + readOrdinaryFile(join(staging, 'metafile-esm.json'), RECORD_BYTES['metafile-esm.json']), + ); + const outputs = Object.keys(graph.outputs) + .map((path) => path.replaceAll('\\', '/')) + .filter((path) => !path.endsWith('.map')); assert.ok(outputs.length > 0 && outputs.length <= 256); + const actual = []; for (const path of outputs) { assert.match(path, /^dist\/[A-Za-z0-9_-]+\.js$/u, 'PACKED_CLI_UNREVIEWED_GRAPH_OUTPUT'); - files.set(`package/${path}`, readOrdinaryFile(join(cli, path), 8 * 1024 * 1024)); + const bytes = readOrdinaryFile(join(cli, path), 8 * 1024 * 1024); + files.set(`package/${path}`, bytes); + actual.push({ path, bytes: bytes.length, sha256: sha256(bytes) }); } assert.deepEqual( - readdirSync(join(cli, 'dist')).sort(), - [...outputs.map((path) => basename(path)), 'metafile-esm.json'].sort(), + readdirSync(join(cli, 'dist')).sort(byCodeUnit), + outputs.map((path) => basename(path)).sort(byCodeUnit), 'PACKED_CLI_EXTRA_FINAL_OUTPUT', ); + const record = JSON.parse(readOrdinaryFile(join(staging, 'finalized.json'), 1024 * 1024)); + const byPath = (left, right) => byCodeUnit(left.path, right.path); + assert.deepEqual( + [...record.outputs].sort(byPath), + [...actual].sort(byPath), + 'PACKED_CLI_FINALIZED_RECORD_DRIFT', + ); const canonical = flatFiles(join(root, 'packages/db/drizzle'), 'package/drizzle'); const copied = flatFiles(join(cli, 'drizzle'), 'package/drizzle'); assert.deepEqual(copied, canonical, 'PACKED_CLI_MIGRATION_COPY_CHANGED'); @@ -73,32 +95,77 @@ export function finalizedPublicFiles(root = repository) { return files; } -export async function sealPackage(artifactDirectory) { +/** Whether anything exists at `path`, without following a link. */ +function present(path) { + try { + lstatSync(path); + return true; + } catch (error) { + if (error?.code === 'ENOENT') return false; + throw error; + } +} + +/** + * Inspect the tarball, then seal the candidate's private map manifest, then symbolicate actual packed + * frames against it, then verify that `dist` and the tarball are still the manifest's, and only then + * write the receipt: a refusal at any step leaves no receipt. An artifact directory that already holds + * a receipt, or a staging that already holds a manifest, is never sealed again: either refuses before + * the inspection and any capture. + * @param {string} artifactDirectory the public artifact directory holding exactly one tarball + * @param {{repository: string, staging: string, source: string, dirty: boolean, checkout: string, + * work: string, ports?: {manifest?: typeof sealPrivateManifest, + * symbolicate?: typeof symbolicateCandidate, verify?: typeof verifySealedDist}}} candidate the + * candidate root and its private staging, the original checkout's identity and object store, and a + * private work directory + */ +export async function sealPackage( + artifactDirectory, + { repository, staging, source, dirty, checkout, work, ports = {} }, +) { + const { + manifest = sealPrivateManifest, + symbolicate = symbolicateCandidate, + verify = verifySealedDist, + } = ports; const tarball = oneTarball(artifactDirectory); - const source = ( - await runCommand('git', ['--no-replace-objects', 'rev-parse', 'HEAD'], { - cwd: repository, - env: isolatedEnvironment(repository), - }) - ).stdout.trim(); - assert.match(source, /^[0-9a-f]{40}$/u); - const dirty = - ( - await runCommand('git', ['status', '--porcelain'], { - cwd: repository, - env: isolatedEnvironment(repository), - }) - ).stdout !== ''; + assert.match(source, /^[0-9a-f]{40}$/u, 'PACKED_CLI_SOURCE_IDENTITY'); + assert.equal(typeof dirty, 'boolean', 'PACKED_CLI_SOURCE_IDENTITY'); + assert.ok( + typeof checkout === 'string' && + isAbsolute(checkout) && + typeof work === 'string' && + isAbsolute(work), + 'PACKED_CLI_SEAL_INPUTS', + ); + // ADR-0111's post-receipt rebuild refusal (Step 9a.2b): a sealed candidate is never sealed again. + assert.ok( + readdirSync(artifactDirectory).length === 1 && !present(join(staging, MANIFEST_FILE)), + 'PACKED_CLI_ALREADY_SEALED', + ); const archive = inspectPackageArchive(readOrdinaryFile(tarball, 16 * 1024 * 1024), { - expectedFiles: finalizedPublicFiles(), - expectedPackage: expectedPackManifest(), + expectedFiles: finalizedPublicFiles(repository, staging), + expectedPackage: expectedPackManifest(repository), }); const origin = originFromEnvironment(process.env); const pnpm = JSON.parse(readFileSync(join(repository, 'package.json'), 'utf8')).packageManager; assert.equal(pnpm, 'pnpm@9.12.3'); + // Only now, with the packed bytes proven to be the finalized ones, the private manifest is sealed; + // actual packed frames are then symbolicated against it before any receipt exists. + const sealed = await manifest({ + candidate: repository, + staging, + checkout, + source, + dirty, + archive, + }); + await symbolicate({ candidate: repository, staging, checkout, work, archive }); + // The capture ran the candidate: its dist and the tarball must still be the ones just sealed. + await verify({ candidate: repository, manifest: sealed, tarball }); const receipt = { version: 1, - mode: 'compiled_package_test', + mode: PACKAGE_TEST_MODE, source, dirty, packageVersion: archive.package.version, @@ -113,18 +180,19 @@ export async function sealPackage(artifactDirectory) { validatePackageReceipt(receipt, { source, origin, - nodeMajor: producerNodeMajor(), + nodeMajor: producerNodeMajor(repository), allowLocal: origin.kind === 'local', }); - writeFileSync( - join(artifactDirectory, 'package-test-receipt.json'), - JSON.stringify(receipt, null, 2) + '\n', - { flag: 'wx' }, - ); + try { + writeFileSync( + join(artifactDirectory, 'package-test-receipt.json'), + JSON.stringify(receipt, null, 2) + '\n', + { flag: 'wx' }, + ); + } catch (error) { + // Another seal wrote its receipt meanwhile. + if (error?.code === 'EEXIST') assert.fail('PACKED_CLI_ALREADY_SEALED'); + throw error; + } return receipt; } - -if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { - assert.equal(process.argv.length, 3, 'one artifact directory required'); - console.log(JSON.stringify(await sealPackage(resolve(process.argv[2])))); -} diff --git a/tools/cli-package/seal.test.mjs b/tools/cli-package/seal.test.mjs index 822d181f..ecd47622 100644 --- a/tools/cli-package/seal.test.mjs +++ b/tools/cli-package/seal.test.mjs @@ -1,21 +1,58 @@ import assert from 'node:assert/strict'; -import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { Buffer } from 'node:buffer'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { test } from 'node:test'; -import { finalizedPublicFiles, producerNodeMajor } from './seal.mjs'; +import { fileURLToPath } from 'node:url'; +import { gzipSync } from 'node:zlib'; +import { ManifestError } from './manifest.mjs'; +import { finalizedPublicFiles, producerNodeMajor, sealPackage } from './seal.mjs'; +import { ustar } from './ustar-fixture.mjs'; +/** A finalized candidate: public JS only in dist, its metafile and hash record in private staging. */ function withFinalizedFixture(body) { const root = mkdtempSync(join(tmpdir(), 'relavium-seal-fixture-')); + const staging = join(root, 'staging'); const write = (path, bytes) => { mkdirSync(dirname(join(root, path)), { recursive: true }); writeFileSync(join(root, path), bytes); }; - const graph = (outputs) => write('apps/cli/dist/metafile-esm.json', JSON.stringify({ outputs })); + const graph = (outputs) => write('staging/metafile-esm.json', JSON.stringify({ outputs })); + const record = (paths) => { + const outputs = paths.map((path) => { + const bytes = readFileSync(join(root, 'apps/cli', path)); + return { + path, + bytes: bytes.length, + sha256: createHash('sha256').update(bytes).digest('hex'), + }; + }); + write( + 'staging/finalized.json', + JSON.stringify({ version: 1, outputs, maps: [], upstreamMaps: 0 }), + ); + }; try { - graph({ 'dist/index.js': {}, 'dist/chunk-ABC.js': {} }); + graph({ + 'dist/index.js': {}, + 'dist/index.js.map': {}, + 'dist/chunk-ABC.js': {}, + 'dist/chunk-ABC.js.map': {}, + }); write('apps/cli/dist/index.js', '#!/usr/bin/env node\nimport "./chunk-ABC.js";\n'); write('apps/cli/dist/chunk-ABC.js', 'export {};\n'); + record(['dist/index.js', 'dist/chunk-ABC.js']); for (const path of ['0000_first.sql', 'meta/_journal.json', 'meta/0000_snapshot.json']) { write(`packages/db/drizzle/${path}`, `canonical ${path}\n`); write(`apps/cli/drizzle/${path}`, `canonical ${path}\n`); @@ -23,7 +60,14 @@ function withFinalizedFixture(body) { for (const name of ['package.json', 'README.md', 'CHANGELOG.md']) write(`apps/cli/${name}`, name); write('LICENSE', 'canonical repository license\n'); - return body({ root, write, graph, remove: (path) => rmSync(join(root, path)) }); + return body({ + root, + staging, + write, + graph, + record, + remove: (path) => rmSync(join(root, path)), + }); } finally { rmSync(root, { recursive: true, force: true }); } @@ -42,9 +86,9 @@ test('producer runtime follows the declared toolchain independently of the fixed }); }); -test('finalized seal inventory contains exact graph bytes, canonical migrations and public documents', () => { - withFinalizedFixture(({ root }) => { - const files = finalizedPublicFiles(root); +test('finalized seal inventory contains exact recorded JS bytes, canonical migrations and public documents', () => { + withFinalizedFixture(({ root, staging }) => { + const files = finalizedPublicFiles(root, staging); assert.deepEqual( [...files.keys()].sort(), [ @@ -69,6 +113,7 @@ test('finalized seal inventory contains exact graph bytes, canonical migrations ); assert.equal(files.get('package/LICENSE').toString(), 'canonical repository license\n'); assert.equal(files.has('package/dist/metafile-esm.json'), false); + assert.equal(files.has('package/dist/index.js.map'), false); }); }); @@ -94,7 +139,6 @@ for (const [name, mutate, refusal] of [ undefined, ], ...[ - 'dist/index.js.map', 'dist/nested/index.js', '../outside.js', '/dist/index.js', @@ -112,10 +156,27 @@ for (const [name, mutate, refusal] of [ /PACKED_CLI_EXTRA_FINAL_OUTPUT/, ], [ - 'unlisted emitted map', + 'a map left in the public package', ({ write }) => write('apps/cli/dist/index.js.map', '{}'), /PACKED_CLI_EXTRA_FINAL_OUTPUT/, ], + [ + 'build metadata left in the public package', + ({ write }) => write('apps/cli/dist/metafile-esm.json', '{}'), + /PACKED_CLI_EXTRA_FINAL_OUTPUT/, + ], + [ + 'JS changed after the finalizer recorded it', + ({ write }) => write('apps/cli/dist/chunk-ABC.js', 'export const late = 1;\n'), + /PACKED_CLI_FINALIZED_RECORD_DRIFT/, + ], + [ + 'a hash record missing an output', + ({ record }) => record(['dist/index.js']), + /PACKED_CLI_FINALIZED_RECORD_DRIFT/, + ], + ['a missing hash record', ({ remove }) => remove('staging/finalized.json'), undefined], + ['a missing staged metafile', ({ remove }) => remove('staging/metafile-esm.json'), undefined], [ 'changed migration', ({ write }) => write('apps/cli/drizzle/0000_first.sql', 'changed'), @@ -156,7 +217,7 @@ for (const [name, mutate, refusal] of [ test(`finalized seal refuses ${name}`, () => { withFinalizedFixture((fixture) => { mutate(fixture); - assert.throws(() => finalizedPublicFiles(fixture.root), refusal); + assert.throws(() => finalizedPublicFiles(fixture.root, fixture.staging), refusal); }); }); } @@ -165,10 +226,612 @@ test( 'finalized seal refuses a symlink masquerading as an emitted file', { skip: process.platform === 'win32' }, () => { - withFinalizedFixture(({ root, remove }) => { + withFinalizedFixture(({ root, staging, remove }) => { remove('apps/cli/dist/chunk-ABC.js'); symlinkSync(join(root, 'LICENSE'), join(root, 'apps/cli/dist/chunk-ABC.js')); - assert.throws(() => finalizedPublicFiles(root), /PACKED_CLI_FILE_KIND_OR_BOUND/); + assert.throws(() => finalizedPublicFiles(root, staging), /PACKED_CLI_FILE_KIND_OR_BOUND/); }); }, ); + +const repository = fileURLToPath(new URL('../..', import.meta.url)); +const SOURCE = 'a'.repeat(40); +const ORIGIN_VARIABLES = [ + 'GITHUB_ACTIONS', + 'GITHUB_REPOSITORY', + 'GITHUB_WORKFLOW', + 'GITHUB_RUN_ID', + 'GITHUB_RUN_ATTEMPT', + 'GITHUB_EVENT_NAME', + 'GITHUB_SHA', + 'RELAVIUM_PACKAGE_SOURCE_SHA', +]; +const GITHUB_ORIGIN = { + GITHUB_ACTIONS: 'true', + GITHUB_REPOSITORY: 'owner/name', + GITHUB_WORKFLOW: 'CI', + GITHUB_RUN_ID: '42', + GITHUB_RUN_ATTEMPT: '1', + GITHUB_EVENT_NAME: 'push', + GITHUB_SHA: 'b'.repeat(40), + RELAVIUM_PACKAGE_SOURCE_SHA: SOURCE, +}; + +/** Run `body` with exactly `variables` as the origin environment, restoring the job's own after. */ +async function withOrigin(variables, body) { + const saved = ORIGIN_VARIABLES.map((name) => [name, process.env[name]]); + for (const name of ORIGIN_VARIABLES) delete process.env[name]; + Object.assign(process.env, variables); + try { + return await body(); + } finally { + for (const [name, value] of saved) + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } +} + +/** + * A finalized candidate that is not the repository holding seal.mjs: its own toolchain pins, + * catalog, CLI manifest, migrations and documents, a private staging directory outside the + * candidate root, and pnpm's tarball of exactly the finalized public files. + */ +async function withCandidate(body, { packageManager = 'pnpm@9.12.3' } = {}) { + const temporary = mkdtempSync(join(tmpdir(), 'relavium-seal-candidate-')); + const root = join(temporary, 'candidate'); + const staging = join(temporary, 'private-staging'); + const artifact = join(temporary, 'artifact'); + const write = (path, bytes) => { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, bytes); + }; + const published = { + name: 'relavium', + version: '0.1.1', + type: 'module', + bin: { relavium: 'dist/index.js' }, + engines: { node: '>=22' }, + scripts: { start: 'node dist/index.js' }, + dependencies: { commander: '^14.0.0' }, + }; + const js = { + 'dist/index.js': '#!/usr/bin/env node\nimport "./chunk-ABC.js";\n', + 'dist/chunk-ABC.js': 'export {};\n', + }; + const migrations = ['0000_first.sql', 'meta/_journal.json', 'meta/0000_snapshot.json']; + try { + write(join(root, '.nvmrc'), `${process.versions.node.split('.')[0]}\n`); + write(join(root, 'package.json'), JSON.stringify({ packageManager })); + write(join(root, 'pnpm-workspace.yaml'), 'packages: []\ncatalog:\n commander: ^14.0.0\n'); + write(join(root, 'LICENSE'), 'candidate license\n'); + write( + join(root, 'apps/cli/package.json'), + JSON.stringify({ + ...published, + scripts: { ...published.scripts, prepack: 'node prepack.mjs' }, + dependencies: { commander: 'catalog:' }, + }), + ); + // The copied tree's own installation resolves `yaml` for the catalog read. + symlinkSync( + join(repository, 'apps/cli/node_modules'), + join(root, 'apps/cli/node_modules'), + 'junction', + ); + write(join(root, 'apps/cli/README.md'), 'candidate readme\n'); + write(join(root, 'apps/cli/CHANGELOG.md'), 'candidate changes\n'); + for (const [path, text] of Object.entries(js)) write(join(root, 'apps/cli', path), text); + for (const path of migrations) { + write(join(root, 'packages/db/drizzle', path), `candidate ${path}\n`); + write(join(root, 'apps/cli/drizzle', path), `candidate ${path}\n`); + } + write( + join(staging, 'metafile-esm.json'), + JSON.stringify({ + outputs: Object.fromEntries( + Object.keys(js).flatMap((path) => [ + [path, {}], + [`${path}.map`, {}], + ]), + ), + }), + ); + write( + join(staging, 'finalized.json'), + JSON.stringify({ + version: 1, + outputs: Object.entries(js).map(([path, text]) => ({ + path, + bytes: Buffer.byteLength(text), + sha256: createHash('sha256').update(text).digest('hex'), + })), + maps: [], + upstreamMaps: 0, + }), + ); + const files = new Map([ + ...Object.entries(js).map(([path, text]) => [`package/${path}`, Buffer.from(text)]), + ...migrations.map((path) => [`package/drizzle/${path}`, Buffer.from(`candidate ${path}\n`)]), + ['package/package.json', Buffer.from(JSON.stringify(published))], + ['package/README.md', Buffer.from('candidate readme\n')], + ['package/CHANGELOG.md', Buffer.from('candidate changes\n')], + ['package/LICENSE', Buffer.from('candidate license\n')], + ]); + const tarball = gzipSync(ustar(files)); + write(join(artifact, 'relavium-0.1.1.tgz'), tarball); + return await body({ root, staging, artifact, tarball, files, published }); + } finally { + rmSync(temporary, { recursive: true, force: true }); + } +} + +const receiptPath = (artifact) => join(artifact, 'package-test-receipt.json'); + +/** + * The seal with recording stand-ins for the private manifest and symbolication, whose own tests + * (manifest.test.mjs, symbolicate.test.mjs) run them for real. + */ +/** What the manifest stand-in returns as the sealed manifest. */ +const SEALED = Object.freeze({ sealed: 'manifest' }); + +function sealWith(artifact, candidate, calls = []) { + return sealPackage(artifact, { + checkout: candidate.repository, + work: join(dirname(candidate.staging), 'work'), + ...candidate, + ports: { + manifest: async (options) => { + calls.push(['manifest', options]); + return SEALED; + }, + symbolicate: async (options) => void calls.push(['symbolicate', options]), + verify: async (options) => void calls.push(['verify', options]), + }, + }); +} + +test('the seal binds a locally produced candidate it is given, never the repository holding seal.mjs', async () => { + await withCandidate(async ({ root, staging, artifact, tarball, files, published }) => { + const receipt = await withOrigin({}, () => + sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: true }), + ); + assert.equal(receipt.mode, 'package_test'); + assert.equal(receipt.source, SOURCE); + assert.equal(receipt.dirty, true); + assert.equal(receipt.packageVersion, '0.1.1'); + assert.equal(receipt.tarball, 'relavium-0.1.1.tgz'); + assert.equal(receipt.tarballSha256, createHash('sha256').update(tarball).digest('hex')); + assert.equal(receipt.archiveBytes, tarball.length); + assert.deepEqual(receipt.files.map((file) => file.path).sort(), [...files.keys()].sort()); + // The catalog dependency resolved from the candidate's own workspace; prepack removed. + assert.deepEqual(receipt.package, published); + assert.deepEqual(receipt.origin, { kind: 'local' }); + assert.deepEqual(receipt.toolchain, { node: process.version, pnpm: '9.12.3' }); + assert.deepEqual(JSON.parse(readFileSync(receiptPath(artifact), 'utf8')), receipt); + assert.deepEqual(readdirSync(artifact).sort(), [ + 'package-test-receipt.json', + 'relavium-0.1.1.tgz', + ]); + }); +}); + +test('the seal records the trusted job origin and refuses a dirty source there', async () => { + await withCandidate(async ({ root, staging, artifact }) => { + await withOrigin(GITHUB_ORIGIN, async () => { + await assert.rejects( + sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: true }), + /DIRTY_SOURCE/, + ); + assert.equal(existsSync(receiptPath(artifact)), false); + const receipt = await sealWith(artifact, { + repository: root, + staging, + source: SOURCE, + dirty: false, + }); + assert.deepEqual(receipt.origin, { + kind: 'github', + repository: 'owner/name', + workflow: 'CI', + runId: '42', + runAttempt: '1', + event: 'push', + source: SOURCE, + }); + }); + }); +}); + +test('the seal refuses a candidate that pins a different pnpm, writing no receipt', async () => { + await withCandidate( + async ({ root, staging, artifact }) => { + await withOrigin({}, () => + assert.rejects( + sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: false }), + /pnpm@9\.12\.4/, + ), + ); + assert.equal(existsSync(receiptPath(artifact)), false); + }, + { packageManager: 'pnpm@9.12.4' }, + ); +}); + +test('the seal refuses an invalid source identity before it reads the candidate', async () => { + await withCandidate(async ({ staging, artifact }) => { + const missing = join(artifact, 'no-candidate'); + for (const identity of [ + { source: 'A'.repeat(40), dirty: false }, + { source: SOURCE.slice(1), dirty: false }, + { source: SOURCE, dirty: 'false' }, + ]) + await assert.rejects( + sealWith(artifact, { repository: missing, staging, ...identity }), + /PACKED_CLI_SOURCE_IDENTITY/, + ); + assert.equal(existsSync(receiptPath(artifact)), false); + }); +}); + +test('the seal refuses archive bytes that differ from the finalized candidate', async () => { + await withCandidate(async ({ root, staging, artifact, files }) => { + files.set('package/LICENSE', Buffer.from('repository license\n')); + writeFileSync(join(artifact, 'relavium-0.1.1.tgz'), gzipSync(ustar(files))); + await withOrigin({}, () => + assert.rejects( + sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: false }), + /FILE_DRIFT/, + ), + ); + assert.equal(existsSync(receiptPath(artifact)), false); + }); +}); + +test('the seal seals the private manifest after the inspection and symbolicates before any receipt', async () => { + await withCandidate(async ({ root, staging, artifact }) => { + const calls = []; + const work = join(dirname(staging), 'work'); + const receipt = await withOrigin({}, () => + sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: false }, calls), + ); + assert.deepEqual( + calls.map(([step]) => step), + ['manifest', 'symbolicate', 'verify'], + ); + const [[, manifest], [, symbolicate], [, verify]] = calls; + // The dist and tarball check receives the manifest just sealed, in memory, and the inspected tarball. + assert.deepEqual(verify, { + candidate: root, + manifest: SEALED, + tarball: join(artifact, 'relavium-0.1.1.tgz'), + }); + assert.equal(verify.manifest, SEALED); + // Both steps receive the inspected archive: the bytes the receipt then records. + assert.equal(manifest.archive.sha256, receipt.tarballSha256); + assert.deepEqual(manifest.archive.files, receipt.files); + assert.deepEqual( + { ...manifest, archive: undefined }, + { + candidate: root, + staging, + checkout: root, + source: SOURCE, + dirty: false, + archive: undefined, + }, + ); + assert.deepEqual( + { ...symbolicate, archive: undefined }, + { candidate: root, staging, checkout: root, work, archive: undefined }, + ); + assert.equal(symbolicate.archive, manifest.archive); + }); +}); + +test('a refused manifest, symbolication or dist check leaves no receipt, and an unproven archive reaches none', async () => { + const order = ['manifest', 'symbolicate', 'verify']; + for (const failing of order) { + await withCandidate(async ({ root, staging, artifact }) => { + const steps = []; + const refusal = new Error(`synthetic ${failing} refusal`); + const step = (name) => async () => { + steps.push(name); + if (name === failing) throw refusal; + }; + await withOrigin({}, () => + assert.rejects( + sealPackage(artifact, { + repository: root, + staging, + source: SOURCE, + dirty: false, + checkout: root, + work: join(dirname(staging), 'work'), + ports: { + manifest: step('manifest'), + symbolicate: step('symbolicate'), + verify: step('verify'), + }, + }), + (error) => error === refusal, + ), + ); + assert.deepEqual(steps, order.slice(0, order.indexOf(failing) + 1)); + assert.equal(existsSync(receiptPath(artifact)), false); + }); + } + await withCandidate(async ({ root, staging, artifact, files }) => { + files.set('package/README.md', Buffer.from('changed\n')); + writeFileSync(join(artifact, 'relavium-0.1.1.tgz'), gzipSync(ustar(files))); + const calls = []; + await withOrigin({}, () => + assert.rejects( + sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: false }, calls), + /FILE_DRIFT/, + ), + ); + assert.deepEqual(calls, []); + }); +}); + +test('a sealed candidate is never sealed again, and nothing is read or run first', async () => { + // A receipt beside the tarball: the candidate was sealed. + await withCandidate(async ({ root, staging, artifact }) => { + await withOrigin({}, () => + sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: false }), + ); + const receipt = readFileSync(receiptPath(artifact)); + const calls = []; + await withOrigin({}, () => + assert.rejects( + sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: false }, calls), + { message: 'PACKED_CLI_ALREADY_SEALED' }, + ), + ); + assert.deepEqual(calls, []); + assert.deepEqual(readFileSync(receiptPath(artifact)), receipt); + }); + // A manifest in the staging, even a dangling link, refuses the same way. + for (const plant of [ + (path) => writeFileSync(path, '{}'), + (path) => symlinkSync(join(dirname(path), 'absent'), path), + ]) { + await withCandidate(async ({ root, staging, artifact }) => { + plant(join(staging, 'manifest.json')); + const calls = []; + await withOrigin({}, () => + assert.rejects( + sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: false }, calls), + { message: 'PACKED_CLI_ALREADY_SEALED' }, + ), + ); + assert.deepEqual(calls, []); + assert.equal(existsSync(receiptPath(artifact)), false); + }); + } + // Another seal that writes its receipt meanwhile wins, and this one refuses with the same code. + await withCandidate(async ({ root, staging, artifact }) => { + await withOrigin({}, () => + assert.rejects( + sealPackage(artifact, { + repository: root, + staging, + source: SOURCE, + dirty: false, + checkout: root, + work: join(dirname(staging), 'work'), + ports: { + manifest: async () => SEALED, + symbolicate: async () => writeFileSync(receiptPath(artifact), 'the other seal'), + verify: async () => {}, + }, + }), + { message: 'PACKED_CLI_ALREADY_SEALED' }, + ), + ); + assert.equal(readFileSync(receiptPath(artifact), 'utf8'), 'the other seal'); + }); +}); + +/** The manifest a real seal would hold for the fixture's bytes: its sorted outputs and the tarball. */ +function sealedManifest(root, artifact) { + const digest = (bytes) => createHash('sha256').update(bytes).digest('hex'); + const dist = join(root, 'apps/cli/dist'); + return { + outputs: readdirSync(dist) + .sort() + .map((name) => ({ path: `dist/${name}`, sha256: digest(readFileSync(join(dist, name))) })), + tarballSha256: digest(readFileSync(join(artifact, 'relavium-0.1.1.tgz'))), + }; +} +const sealInputs = (root, staging) => ({ + repository: root, + staging, + source: SOURCE, + dirty: false, + checkout: root, + work: join(dirname(staging), 'work'), +}); + +test('without a verify port the seal checks dist and the tarball before the receipt', async () => { + // Unchanged, they verify and the receipt is written. + await withCandidate(async ({ root, staging, artifact }) => { + const manifest = sealedManifest(root, artifact); + await withOrigin({}, () => + sealPackage(artifact, { + ...sealInputs(root, staging), + ports: { manifest: async () => manifest, symbolicate: async () => {} }, + }), + ); + assert.equal(existsSync(receiptPath(artifact)), true); + }); + // A chunk or the tarball changed during the capture refuses, and no receipt is written. + for (const [change, code] of [ + [ + (root) => writeFileSync(join(root, 'apps/cli/dist/chunk-ABC.js'), 'export {}; \n'), + 'DIST_DRIFT', + ], + [ + (_root, artifact) => writeFileSync(join(artifact, 'relavium-0.1.1.tgz'), 'other'), + 'TARBALL_CHANGED', + ], + ]) { + await withCandidate(async ({ root, staging, artifact }) => { + const manifest = sealedManifest(root, artifact); + await withOrigin({}, () => + assert.rejects( + sealPackage(artifact, { + ...sealInputs(root, staging), + ports: { + manifest: async () => manifest, + symbolicate: async () => change(root, artifact), + }, + }), + (error) => error instanceof ManifestError && error.code === code, + ), + ); + assert.equal(existsSync(receiptPath(artifact)), false); + }); + } +}); + +test('the already-sealed refusal precedes the inspection, and other extras keep their own code', async () => { + // A staged manifest beside an archive the inspection would refuse: the seal refusal comes first. + await withCandidate(async ({ root, staging, artifact, files }) => { + writeFileSync(join(staging, 'manifest.json'), '{}'); + files.set('package/LICENSE', Buffer.from('changed\n')); + writeFileSync(join(artifact, 'relavium-0.1.1.tgz'), gzipSync(ustar(files))); + await withOrigin({}, () => + assert.rejects(sealPackage(artifact, sealInputs(root, staging)), { + message: 'PACKED_CLI_ALREADY_SEALED', + }), + ); + }); + // A receipt beside the tarball and no candidate at all: still the seal refusal. + await withCandidate(async ({ staging, artifact }) => { + writeFileSync(receiptPath(artifact), '{}'); + await withOrigin({}, () => + assert.rejects(sealPackage(artifact, sealInputs(join(artifact, 'no-candidate'), staging)), { + message: 'PACKED_CLI_ALREADY_SEALED', + }), + ); + }); + // Any other extra file is the public artifact check's refusal, never ALREADY_SEALED. + await withCandidate(async ({ root, staging, artifact }) => { + writeFileSync(join(artifact, 'notes.txt'), 'extra'); + await withOrigin({}, () => + assert.rejects(sealPackage(artifact, sealInputs(root, staging)), { + message: 'PACKED_CLI_UNEXPECTED_PUBLIC_ARTIFACT', + }), + ); + }); +}); + +test('a receipt write failing other than EEXIST keeps its own error and leaves no receipt', async () => { + await withCandidate(async ({ root, staging, artifact }) => { + await withOrigin({}, () => + assert.rejects( + sealPackage(artifact, { + ...sealInputs(root, staging), + ports: { + manifest: async () => SEALED, + symbolicate: async () => {}, + // The artifact directory disappears before the receipt is written. + verify: async () => rmSync(artifact, { recursive: true, force: true }), + }, + }), + (error) => error?.code === 'ENOENT', + ), + ); + assert.equal(existsSync(receiptPath(artifact)), false); + }); +}); + +test('both steps receive the candidate, the original checkout and the dirty flag as given', async () => { + await withCandidate(async ({ root, staging, artifact }) => { + const calls = []; + const checkout = join(dirname(staging), 'original-checkout'); + await withOrigin({}, () => + sealPackage(artifact, { + repository: root, + staging, + source: SOURCE, + dirty: true, + checkout, + work: join(dirname(staging), 'work'), + ports: { + manifest: async (options) => void calls.push(options), + symbolicate: async (options) => void calls.push(options), + verify: async () => {}, + }, + }), + ); + const [manifest, symbolicate] = calls; + assert.deepEqual( + [manifest.candidate, manifest.checkout, manifest.dirty], + [root, checkout, true], + ); + assert.deepEqual([symbolicate.candidate, symbolicate.checkout], [root, checkout]); + }); +}); + +test('without ports the seal runs the real private manifest and the real symbolication', async () => { + // The real manifest refuses this candidate's record, which stages no maps. + await withCandidate(async ({ root, staging, artifact }) => { + await withOrigin({}, () => + assert.rejects( + sealPackage(artifact, { + repository: root, + staging, + source: SOURCE, + dirty: false, + checkout: root, + work: join(dirname(staging), 'work'), + }), + (error) => error instanceof ManifestError && error.code === 'RECORD', + ), + ); + assert.equal(existsSync(receiptPath(artifact)), false); + }); + // The real symbolication reads the manifest that a stand-in never sealed. + await withCandidate(async ({ root, staging, artifact }) => { + await withOrigin({}, () => + assert.rejects( + sealPackage(artifact, { + repository: root, + staging, + source: SOURCE, + dirty: false, + checkout: root, + work: join(dirname(staging), 'work'), + ports: { manifest: async () => {} }, + }), + (error) => error instanceof ManifestError && error.code === 'UNREADABLE', + ), + ); + assert.equal(existsSync(receiptPath(artifact)), false); + }); +}); + +test('the seal requires the checkout and a private work directory before it reads the candidate', async () => { + await withCandidate(async ({ staging, artifact }) => { + const missing = join(artifact, 'no-candidate'); + for (const inputs of [ + { checkout: undefined, work: '/private/work' }, + { checkout: 'relative', work: '/private/work' }, + { checkout: '/checkout', work: undefined }, + { checkout: '/checkout', work: 'relative' }, + ]) { + await assert.rejects( + sealPackage(artifact, { + repository: missing, + staging, + source: SOURCE, + dirty: false, + ...inputs, + }), + /PACKED_CLI_SEAL_INPUTS/, + ); + } + }); +}); diff --git a/tools/cli-package/smoke/README.md b/tools/cli-package/smoke/README.md index 11fe2333..347113bf 100644 --- a/tools/cli-package/smoke/README.md +++ b/tools/cli-package/smoke/README.md @@ -46,7 +46,10 @@ aggregate. Each result has `id`, `passed`, and evidence or an error. Overall - Replay a public offline `agent run --fixture` with `git_status` through the actual native process host. A private copied Node executable named `git`/`git.exe` runs the fixed extensionless fixture script, which creates only its recorded child. - Inject a synthetic error on the real target after readiness, then observe actual + The fixture's processes also end once the smoke worker that owns their case is + gone, so a consumer interrupted mid-case leaves no orphaned target; while the + worker lives, their owner watch never ends them: the product's own cleanup does, or + on failure the smoke's watchdog and exact-PID cleanup, which remain failure evidence. Inject a synthetic error on the real target after readiness, then observe actual target and descendant closure. Delay delivery of real close events to distinguish an error/request from the owned native acknowledgement. POSIX additionally covers natural leader exit while a descendant holds inherited pipes. Windows runs the diff --git a/tools/cli-package/smoke/index.mjs b/tools/cli-package/smoke/index.mjs index 16ce07a3..bc1f1f08 100644 --- a/tools/cli-package/smoke/index.mjs +++ b/tools/cli-package/smoke/index.mjs @@ -1,9 +1,38 @@ import assert from 'node:assert/strict'; +import { mkdtempSync, readdirSync, rmSync } from 'node:fs'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { readOrdinaryFile } from '../files.mjs'; import { isolatedEnvironment, runCommand } from '../process.mjs'; +/** The prefix of the worker's own temporary root under the support directory. */ +export const INSTALLED_SMOKE_ROOT_PREFIX = 'installed-smoke-'; + +/** + * Create the worker's own temporary root under `supportRoot`, named so that + * `abandonInstalledSmoke` finds it. + * @param {string} supportRoot + */ +export function createInstalledSmokeRoot(supportRoot) { + // `supportRoot` is the CI job's own absolute directory, already resolved and checked by the worker; + // no model supplies it, which is what Sonar's S8707 taint rule assumes of a command-line argument. + return mkdtempSync(join(supportRoot, INSTALLED_SMOKE_ROOT_PREFIX)); // NOSONAR — see above +} + +/** + * For a caller that an interrupt ends while the worker runs: the worker is killed before its own + * removal, so remove every worker root under `supportRoot`. The native fixture targets the installed + * CLI started in their own process groups end on their own once that worker is gone. + * @param {string} supportRoot + */ +export function abandonInstalledSmoke(supportRoot) { + for (const name of readdirSync(supportRoot)) { + if (!name.startsWith(INSTALLED_SMOKE_ROOT_PREFIX)) continue; + // Retries absorb a killed process that was still finishing a write. + rmSync(join(supportRoot, name), { recursive: true, force: true, maxRetries: 3 }); + } +} + /** Join native module lifetime before the caller removes the owned installation. */ export async function runInstalledSmoke({ packageRoot, supportRoot, artifactSha256, source }) { const result = await runCommand( diff --git a/tools/cli-package/smoke/installation.mjs b/tools/cli-package/smoke/installation.mjs new file mode 100644 index 00000000..f4989ce7 --- /dev/null +++ b/tools/cli-package/smoke/installation.mjs @@ -0,0 +1,19 @@ +import { basename, dirname, sep } from 'node:path'; + +/** + * The consumer's node_modules directory that owns an installed package's real path. With pnpm's + * virtual store that path is `/.pnpm//node_modules/`, so the owner is the + * directory holding the store, not the store's per-package node_modules; without a virtual store + * it is the nearest node_modules ancestor. An unrelated node_modules further up never owns it. + * @param {string} packageRoot the installed package's absolute real path + * @returns {string | undefined} + */ +export function installedModulesOwner(packageRoot) { + const store = `${sep}node_modules${sep}.pnpm${sep}`; + const index = packageRoot.lastIndexOf(store); + if (index >= 0) return packageRoot.slice(0, index + `${sep}node_modules`.length); + for (let parent = dirname(packageRoot); parent !== dirname(parent); parent = dirname(parent)) { + if (basename(parent) === 'node_modules') return parent; + } + return undefined; +} diff --git a/tools/cli-package/smoke/installation.test.mjs b/tools/cli-package/smoke/installation.test.mjs new file mode 100644 index 00000000..0766531b --- /dev/null +++ b/tools/cli-package/smoke/installation.test.mjs @@ -0,0 +1,43 @@ +import assert from 'node:assert/strict'; +import { join, parse } from 'node:path'; +import { test } from 'node:test'; +import { installedModulesOwner } from './installation.mjs'; + +const root = parse(process.cwd()).root; +const path = (...parts) => join(root, ...parts); + +test('a pnpm virtual-store installation is owned by the node_modules that holds the store', () => { + const home = path('tmp', 'consumer'); + const installed = join( + home, + 'node_modules', + '.pnpm', + 'relavium@0.1.1', + 'node_modules', + 'relavium', + ); + // The nearest node_modules is the store's per-package directory; dependencies resolve beside it. + assert.equal(installedModulesOwner(installed), join(home, 'node_modules')); +}); + +test('an unrelated node_modules above the installation never owns it', () => { + const home = path('work', 'node_modules', 'scratch', 'consumer'); + assert.equal( + installedModulesOwner( + join(home, 'node_modules', '.pnpm', 'relavium@0.1.1', 'node_modules', 'relavium'), + ), + join(home, 'node_modules'), + ); + assert.equal( + installedModulesOwner(join(home, 'node_modules', 'relavium')), + join(home, 'node_modules'), + ); +}); + +test('a flat installation is owned by its nearest node_modules; none is no owner', () => { + assert.equal( + installedModulesOwner(path('tmp', 'consumer', 'node_modules', 'relavium')), + path('tmp', 'consumer', 'node_modules'), + ); + assert.equal(installedModulesOwner(path('tmp', 'consumer', 'relavium')), undefined); +}); diff --git a/tools/cli-package/smoke/native-descendant.cjs b/tools/cli-package/smoke/native-descendant.cjs index 513c993d..1be988b4 100644 --- a/tools/cli-package/smoke/native-descendant.cjs +++ b/tools/cli-package/smoke/native-descendant.cjs @@ -2,7 +2,17 @@ const assert = require('node:assert/strict'); const { appendFileSync } = require('node:fs'); const { setInterval } = require('node:timers'); -assert.equal(process.argv.length, 3); +assert.equal(process.argv.length, 4); +const owner = Number(process.argv[3]); +assert.ok(Number.isSafeInteger(owner) && owner > 0); +// Like its parent, it never outlives the smoke worker that owns its case. +setInterval(() => { + try { + process.kill(owner, 0); + } catch (error) { + if (error.code === 'ESRCH') process.exit(0); + } +}, 100).unref(); appendFileSync( process.argv[2], JSON.stringify({ diff --git a/tools/cli-package/smoke/native-process.mjs b/tools/cli-package/smoke/native-process.mjs index f526bd6c..6a79248f 100644 --- a/tools/cli-package/smoke/native-process.mjs +++ b/tools/cli-package/smoke/native-process.mjs @@ -290,6 +290,22 @@ export function assertNativeBootstrapDiagnostic(diagnostics, mode) { ); } +/** + * The private configuration beside a fixture target's `status` script. Its owner is this smoke + * worker: the fixture's processes end once it is gone (an interrupted consumer). + * @param {{mode: string, log: string, release?: string}} options + */ +export function nativeFixtureConfig({ mode, log, release }) { + return { + mode, + log, + node: process.execPath, + descendant: fixture('native-descendant.cjs'), + owner: process.pid, + ...(release === undefined ? {} : { release }), + }; +} + export async function assertInstalledNativeProcesses(context, { bootstrap = false } = {}) { const { root, cli, environment, calls } = context; const home = join(root, bootstrap ? 'bootstrap-home' : 'native-home'); @@ -339,13 +355,7 @@ export async function assertInstalledNativeProcesses(context, { bootstrap = fals copyFileSync(fixture('native-status.cjs'), join(cwd, 'status')); writeFileSync( join(cwd, 'native-fixture.json'), - JSON.stringify({ - mode, - log, - node: process.execPath, - descendant: fixture('native-descendant.cjs'), - ...(bootstrap ? { release } : {}), - }), + JSON.stringify(nativeFixtureConfig({ mode, log, ...(bootstrap ? { release } : {}) })), { mode: 0o600 }, ); writeFileSync( diff --git a/tools/cli-package/smoke/native-process.test.mjs b/tools/cli-package/smoke/native-process.test.mjs index 5ceab75e..e33f5f7c 100644 --- a/tools/cli-package/smoke/native-process.test.mjs +++ b/tools/cli-package/smoke/native-process.test.mjs @@ -22,9 +22,11 @@ import { copyFixtureRuntimeSupport, createFixtureSentinel, InstalledNativeControlError, + nativeFixtureConfig, pendingNativeFixturePids, } from './native-process.mjs'; import { explicitEnvironment, ndjson, poll } from './process.mjs'; +import { setTimeout as sleep } from 'node:timers/promises'; function observedGroupRefusal() { return { @@ -206,6 +208,7 @@ test( log, node: descendantNode, descendant: fileURLToPath(new URL('./native-descendant.cjs', import.meta.url)), + owner: process.pid, }), ); const child = spawn(executable, ['status'], { @@ -313,6 +316,155 @@ test( }, ); +/** Whether `pid` still names a live process. */ +function alive(pid) { + try { + process.kill(pid, 0); + return true; + } catch (error) { + if (error.code === 'ESRCH') return false; + throw error; + } +} + +/** SIGKILL `pid` (a negative one names a process group) unless it is already gone. */ +function stop(pid) { + try { + process.kill(pid, 'SIGKILL'); + } catch (error) { + if (error.code !== 'ESRCH') throw error; + } +} + +/** + * Run the real fixture target in `mode` with a stand-in for the smoke worker as its owner. The target + * is detached, as the installed CLI starts it, so nothing stops its group but the fixture's own watch + * of its owner. Cleanup is registered with `t.after`, so it also runs when the test times out, and + * neither child keeps the test process alive. + */ +function ownedFixture(t, mode) { + const root = mkdtempSync(join(tmpdir(), `relavium-status-${mode}-test-`)); + const log = join(root, 'native.ndjson'); + const owner = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { + stdio: 'ignore', + }); + owner.unref(); + copyFileSync( + fileURLToPath(new URL('./native-status.cjs', import.meta.url)), + join(root, 'status'), + ); + writeFileSync( + join(root, 'native-fixture.json'), + JSON.stringify({ ...nativeFixtureConfig({ mode, log }), owner: owner.pid }), + ); + const target = spawn(process.execPath, [join(root, 'status')], { + cwd: root, + detached: process.platform !== 'win32', + stdio: ['ignore', 'pipe', 'ignore'], + }); + target.unref(); + const fixture = { root, log, owner, target, stdout: '', descendant: undefined }; + target.stdout.on('data', (chunk) => { + fixture.stdout += chunk.toString(); + }); + t.after(() => { + if (process.platform !== 'win32' && target.pid !== undefined) stop(-target.pid); + else if (target.exitCode === null && target.signalCode === null) target.kill('SIGKILL'); + if (fixture.descendant !== undefined) stop(fixture.descendant); + if (owner.pid !== undefined) stop(owner.pid); + target.stdout.destroy(); + rmSync(root, { recursive: true, force: true }); + }); + return fixture; +} + +/** The descendant the fixture target recorded. */ +function recordedDescendant(fixture) { + const pid = ndjson(readFileSync(fixture.log, 'utf8')).find( + (item) => item.operation === 'native-fixture-descendant', + )?.pid; + assert.ok(Number.isSafeInteger(pid) && pid > 0); + fixture.descendant = pid; + return pid; +} + +test( + 'the fixture target and its descendant end once the worker that owns their case is gone', + { timeout: 15000 }, + async (t) => { + const fixture = ownedFixture(t, 'injected-error'); + await poll( + () => fixture.stdout.includes('RELAVIUM_NATIVE_FIXTURE_READY\n'), + 5000, + 'Fixture descendant readiness missing', + ); + const pids = [fixture.target.pid, recordedDescendant(fixture)]; + // Both stay while their owner lives. + await sleep(400); + assert.ok(pids.every(alive)); + fixture.owner.kill('SIGKILL'); + await poll(() => !pids.some(alive), 5000, 'A fixture process outlived the worker that owns it'); + }, +); + +test( + 'a leader-exit descendant outlives its exited parent until the worker that owns it is gone', + { timeout: 15000, skip: process.platform === 'win32' }, + async (t) => { + // Leader-exit mode reparents the descendant by design; it must watch the worker, never its parent, + // or it would end on its own and the product's group reap would go unproven. + const fixture = ownedFixture(t, 'leader-exit'); + await poll(() => fixture.target.exitCode !== null, 5000, 'The leader did not exit on its own'); + assert.equal(fixture.target.exitCode, 0); + const descendant = recordedDescendant(fixture); + await sleep(600); + assert.equal(alive(descendant), true, 'the descendant ended with its parent, not its owner'); + fixture.owner.kill('SIGKILL'); + await poll(() => !alive(descendant), 5000, 'The descendant outlived the worker that owns it'); + }, +); + +test('a fixture target is owned by the smoke worker that writes its configuration', () => { + const descendant = fileURLToPath(new URL('./native-descendant.cjs', import.meta.url)); + const common = { log: '/case/native.ndjson', node: process.execPath, descendant }; + assert.deepEqual(nativeFixtureConfig({ mode: 'exit-fallback', log: common.log }), { + mode: 'exit-fallback', + ...common, + owner: process.pid, + }); + assert.deepEqual( + nativeFixtureConfig({ mode: 'bootstrap-fatal', log: common.log, release: '/case/release' }), + { mode: 'bootstrap-fatal', ...common, owner: process.pid, release: '/case/release' }, + ); +}); + +test('a fixture target without an owner refuses at once instead of running unwatched', async (t) => { + const root = mkdtempSync(join(tmpdir(), 'relavium-status-unowned-test-')); + const log = join(root, 'native.ndjson'); + copyFileSync( + fileURLToPath(new URL('./native-status.cjs', import.meta.url)), + join(root, 'status'), + ); + const unowned = nativeFixtureConfig({ mode: 'exit-fallback', log }); + delete unowned.owner; + writeFileSync(join(root, 'native-fixture.json'), JSON.stringify(unowned)); + const target = spawn(process.execPath, [join(root, 'status')], { + cwd: root, + stdio: ['ignore', 'ignore', 'ignore'], + }); + t.after(() => { + if (target.exitCode === null && target.signalCode === null) target.kill('SIGKILL'); + rmSync(root, { recursive: true, force: true }); + }); + const outcome = await Promise.race([ + new Promise((resolve) => target.once('exit', (code) => resolve(code))), + sleep(5000, 'still running', { ref: false }), + ]); + assert.notEqual(outcome, 'still running'); + assert.notEqual(outcome, 0); + assert.equal(existsSync(log), false); +}); + test('shared fixture runtime support reuses read-only identical bytes and refuses replacement content or links', () => { const root = mkdtempSync(join(tmpdir(), 'relavium-shared-runtime-test-')); const source = join(root, 'source.dylib'); diff --git a/tools/cli-package/smoke/native-status.cjs b/tools/cli-package/smoke/native-status.cjs index 07469538..59878492 100644 --- a/tools/cli-package/smoke/native-status.cjs +++ b/tools/cli-package/smoke/native-status.cjs @@ -19,12 +19,23 @@ assert.ok( 'bootstrap-output', ].includes(config.mode), ); +assert.ok(Number.isSafeInteger(config.owner) && config.owner > 0); +// The fixture never outlives the smoke worker that owns its case: an interrupted consumer kills that +// worker, and nothing else would stop a target in its own process group. Unreferenced, so it never +// keeps a fixture alive that would otherwise exit. +setInterval(() => { + try { + process.kill(config.owner, 0); + } catch (error) { + if (error.code === 'ESRCH') process.exit(0); + } +}, 100).unref(); record({ operation: 'native-fixture-root', pid: process.pid }); if (config.mode === 'exit-fallback') { process.stdout.write('RELAVIUM_NATIVE_FIXTURE_READY\n'); setInterval(() => {}, 1000); } else { - const descendant = spawn(config.node, [config.descendant, config.log], { + const descendant = spawn(config.node, [config.descendant, config.log, String(config.owner)], { shell: false, stdio: ['ignore', 'inherit', 'inherit', 'ipc'], }); diff --git a/tools/cli-package/smoke/worker.mjs b/tools/cli-package/smoke/worker.mjs index d5712326..5e7295a3 100644 --- a/tools/cli-package/smoke/worker.mjs +++ b/tools/cli-package/smoke/worker.mjs @@ -4,7 +4,6 @@ import { createHash } from 'node:crypto'; import { existsSync, mkdirSync, - mkdtempSync, readFileSync, readdirSync, realpathSync, @@ -20,6 +19,8 @@ import { assertInstalledMcp, assertInstalledWorkflows } from './workflows.mjs'; import { assertExit, explicitEnvironment, invoke, ndjson } from './process.mjs'; import { assertInstalledNativeProcesses, InstalledNativeControlError } from './native-process.mjs'; import { emitInstalledWorkerFailure } from './failure-diagnostic.mjs'; +import { installedModulesOwner } from './installation.mjs'; +import { createInstalledSmokeRoot } from './index.mjs'; const directory = dirname(fileURLToPath(import.meta.url)); const repository = resolve(directory, '../../..'); @@ -60,11 +61,9 @@ async function exerciseInstalledSmoke({ packageRoot, supportRoot, artifactSha256 const require = createRequire(join(packageRoot, 'package.json')); const entry = join(packageRoot, 'dist/index.js'); const before = snapshot(packageRoot); - let installedModules; - for (let parent = dirname(packageRoot); parent !== dirname(parent); parent = dirname(parent)) - if (parent.endsWith(`${sep}node_modules`)) installedModules = parent; + const installedModules = installedModulesOwner(packageRoot); assert.ok(installedModules, 'Installed package must have an explicit node_modules owner'); - const root = mkdtempSync(join(supportRoot, 'installed-smoke-')); + const root = createInstalledSmokeRoot(supportRoot); const log = join(root, 'guard.ndjson'); const report = { schemaVersion: 1, diff --git a/tools/cli-package/source-map.mjs b/tools/cli-package/source-map.mjs new file mode 100644 index 00000000..214776ba --- /dev/null +++ b/tools/cli-package/source-map.mjs @@ -0,0 +1,187 @@ +/** + * Strict Source Map v3 `mappings` decoding for the candidate finalizer and symbolication (ADR-0111, + * Step 9a.2a). Node's built-in `node:module` SourceMap is lenient — it decodes an invalid character as + * zero and extrapolates a position it has no segment for — so it serves only as an independent test + * oracle, never as the validator. A refusal names a fixed code and never quotes the mappings. + */ + +export class SourceMapError extends Error { + /** @param {string} code */ + constructor(code) { + super(`source map refused: ${code}`); + this.name = 'SourceMapError'; + this.code = code; + } +} + +/** @param {string} code @returns {never} */ +function refuse(code) { + throw new SourceMapError(code); +} + +const BASE64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; +const DIGITS = new Int8Array(128).fill(-1); +for (let index = 0; index < BASE64.length; index += 1) DIGITS[BASE64.charCodeAt(index)] = index; +/** The largest unsigned VLQ value (a signed 32-bit magnitude and its sign bit). */ +const VLQ_MAX = 0xffffffff; +/** The most base64 digits a value up to `VLQ_MAX` can take (five bits each). */ +const MAX_DIGITS = 7; + +/** + * One decoded segment: a generated column, and for a mapped segment its source index and 0-based + * original line and column, and optionally a name index. + * @typedef {{generatedColumn: number, source?: number, originalLine?: number, + * originalColumn?: number, name?: number}} Segment + */ + +/** + * Decode `mappings` into one array of segments per generated line. Refuses a character outside the + * base64 alphabet, a truncated, oversized or over-long value, an empty segment, a segment of other + * than 1, 4 or 5 fields, a generated column that does not increase along its line, a negative line or + * column, and an out-of-range source or name index. Upper bounds on positions are the range checks' + * (`assertGeneratedRange`, `assertOriginalRange`). + * @param {string} mappings + * @param {{sources: number, names: number}} counts the lengths of the map's `sources` and `names` + * @returns {Segment[][]} + */ +export function decodeMappings(mappings, { sources, names }) { + if (typeof mappings !== 'string') refuse('MAPPINGS_TYPE'); + /** @type {Segment[][]} */ + const lines = [[]]; + let column = 0; + let source = 0; + let originalLine = 0; + let originalColumn = 0; + let name = 0; + /** @type {number[]} */ + let fields = []; + let position = 0; + const finish = () => { + if (fields.length !== 1 && fields.length !== 4 && fields.length !== 5) + refuse('MAPPINGS_FIELDS'); + const line = lines.at(-1); + column += fields[0]; + if (column < 0) refuse('MAPPINGS_RANGE'); + if (line.length > 0 && column <= line[line.length - 1].generatedColumn) + refuse('MAPPINGS_ORDER'); + /** @type {Segment} */ + const segment = { generatedColumn: column }; + if (fields.length >= 4) { + source += fields[1]; + originalLine += fields[2]; + originalColumn += fields[3]; + if (source < 0 || source >= sources || originalLine < 0 || originalColumn < 0) { + refuse('MAPPINGS_RANGE'); + } + segment.source = source; + segment.originalLine = originalLine; + segment.originalColumn = originalColumn; + if (fields.length === 5) { + name += fields[4]; + if (name < 0 || name >= names) refuse('MAPPINGS_RANGE'); + segment.name = name; + } + } + line.push(segment); + fields = []; + }; + while (position < mappings.length) { + const character = mappings.charCodeAt(position); + if (character === 0x3b /* ; */ || character === 0x2c /* , */) { + // An empty segment is malformed; an empty line is not. + if (fields.length > 0) finish(); + else if (character === 0x2c || mappings.charCodeAt(position - 1) === 0x2c) { + refuse('MAPPINGS_EMPTY_SEGMENT'); + } + if (character === 0x3b) { + lines.push([]); + column = 0; + } + position += 1; + continue; + } + let value = 0; + let scale = 1; + for (let digits = 1; ; digits += 1) { + if (position >= mappings.length) refuse('MAPPINGS_TRUNCATED'); + const code = mappings.charCodeAt(position); + const digit = code < 128 ? DIGITS[code] : -1; + if (digit < 0) refuse('MAPPINGS_CHARACTER'); + position += 1; + value += (digit & 31) * scale; + if (value > VLQ_MAX) refuse('MAPPINGS_RANGE'); + if ((digit & 32) === 0) break; + // A 32-bit value never needs more than seven digits; zero-valued padding beyond them would + // otherwise grow the scale to Infinity and the value to NaN, which no comparison refuses. + if (digits === MAX_DIGITS) refuse('MAPPINGS_RANGE'); + scale *= 32; + } + const magnitude = Math.floor(value / 2); + fields.push(value % 2 === 1 ? -magnitude : magnitude); + } + if (fields.length > 0) finish(); + else if (mappings.charCodeAt(mappings.length - 1) === 0x2c) refuse('MAPPINGS_EMPTY_SEGMENT'); + return lines; +} + +/** + * The segment that starts exactly at a 0-based generated position, or `undefined`. Never the + * nearest preceding one: a frame that is off by a column must not map. + * @param {Segment[][]} lines @param {number} line @param {number} column + */ +export function segmentAt(lines, line, column) { + const segments = lines[line]; + if (segments === undefined) return undefined; + let low = 0; + let high = segments.length - 1; + while (low <= high) { + const middle = (low + high) >>> 1; + const start = segments[middle].generatedColumn; + if (start === column) return segments[middle]; + if (start < column) low = middle + 1; + else high = middle - 1; + } + return undefined; +} + +/** The line terminators ECMAScript and TypeScript both count. */ +const LINE_TERMINATOR = /\r\n|[\n\r\u2028\u2029]/u; + +/** + * The UTF-16 length of each line of `text`, as V8 and TypeScript count lines and columns. + * @param {string} text + */ +export function lineLengths(text) { + return text.split(LINE_TERMINATOR).map((line) => line.length); +} + +/** + * Require every generated position to lie inside the generated text: no segment on a line the text + * does not have, and no segment past its line's end (a position at the end is a caret position, as + * esbuild maps an empty line at column 0). + * @param {Segment[][]} lines @param {number[]} generated the generated text's line lengths + */ +export function assertGeneratedRange(lines, generated) { + for (let line = 0; line < lines.length; line += 1) { + const segments = lines[line]; + if (segments.length === 0) continue; + if (line >= generated.length) refuse('GENERATED_RANGE'); + if (segments[segments.length - 1].generatedColumn > generated[line]) refuse('GENERATED_RANGE'); + } +} + +/** + * Require every original position to lie inside its source text: an existing line, and a column no + * further than that line's end. + * @param {Segment[][]} lines @param {readonly number[][]} originals line lengths per source index + */ +export function assertOriginalRange(lines, originals) { + for (const segments of lines) { + for (const segment of segments) { + if (segment.source === undefined) continue; + const lengths = originals[segment.source]; + const length = lengths?.[segment.originalLine ?? -1]; + if (length === undefined || (segment.originalColumn ?? 0) > length) refuse('ORIGINAL_RANGE'); + } + } +} diff --git a/tools/cli-package/source-map.test.mjs b/tools/cli-package/source-map.test.mjs new file mode 100644 index 00000000..ea1f6b81 --- /dev/null +++ b/tools/cli-package/source-map.test.mjs @@ -0,0 +1,329 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs'; +import { SourceMap, createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { basename, join } from 'node:path'; +import { after, test } from 'node:test'; +import { pathToFileURL } from 'node:url'; +import { + SourceMapError, + assertGeneratedRange, + assertOriginalRange, + decodeMappings, + lineLengths, + segmentAt, +} from './source-map.mjs'; + +// The exact esbuild behind the installed tsup builds the fixtures; no product build runs here. +const cliRequire = createRequire(new URL('../../apps/cli/package.json', import.meta.url)); +const { build } = createRequire(cliRequire.resolve('tsup/package.json'))('esbuild'); +const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-source-map-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); + +const BASE64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; + +/** Encode one signed value as base64 VLQ, as a source map writer does. */ +function vlq(value) { + let rest = value < 0 ? -value * 2 + 1 : value * 2; + let out = ''; + do { + let digit = rest % 32; + rest = Math.floor(rest / 32); + if (rest > 0) digit += 32; + out += BASE64[digit]; + } while (rest > 0); + return out; +} + +/** Encode absolute segments ([column] or [column, source, line, column, name?]) per line. */ +function encode(lines) { + let source = 0; + let line = 0; + let column = 0; + let name = 0; + return lines + .map((segments) => { + let generated = 0; + return segments + .map((segment) => { + let out = vlq(segment[0] - generated); + generated = segment[0]; + if (segment.length >= 4) { + out += vlq(segment[1] - source) + vlq(segment[2] - line) + vlq(segment[3] - column); + [source, line, column] = [segment[1], segment[2], segment[3]]; + } + if (segment.length === 5) { + out += vlq(segment[4] - name); + name = segment[4]; + } + return out; + }) + .join(','); + }) + .join(';'); +} + +const refuses = (code) => (error) => error instanceof SourceMapError && error.code === code; +const COUNTS = { sources: 2, names: 2 }; + +test('decodes every segment exactly as Node’s own source map does', () => { + // A deterministic map with long lines, empty lines, unmapped and named segments and backward + // original moves, decoded strictly and compared with node:module at every segment start. + let seed = 7; + const next = (bound) => { + seed = (seed * 1103515245 + 12345) % 2 ** 31; + return seed % bound; + }; + const lines = []; + for (let line = 0; line < 40; line += 1) { + const segments = []; + let column = next(3); + const count = next(30); + for (let index = 0; index < count; index += 1) { + column += 1 + next(5000); + const kind = next(5); + if (kind === 0) segments.push([column]); + else if (kind === 1) segments.push([column, next(2), next(400), next(300)]); + else segments.push([column, next(2), next(400), next(300), next(2)]); + } + lines.push(segments); + } + const map = { version: 3, sources: ['a.ts', 'b.ts'], names: ['x', 'y'], mappings: encode(lines) }; + const decoded = decodeMappings(map.mappings, COUNTS); + const oracle = new SourceMap(map); + let checked = 0; + decoded.forEach((segments, line) => { + assert.equal(segments.length, lines[line].length); + segments.forEach((segment, index) => { + const expected = lines[line][index]; + assert.equal(segment.generatedColumn, expected[0]); + assert.equal(segmentAt(decoded, line, segment.generatedColumn), segment); + if (expected.length === 1) { + assert.equal(segment.source, undefined); + return; + } + const entry = oracle.findEntry(line, segment.generatedColumn); + assert.equal(entry.generatedLine, line); + assert.equal(entry.generatedColumn, segment.generatedColumn); + assert.equal(entry.originalSource, map.sources[segment.source]); + assert.equal(entry.originalLine, segment.originalLine); + assert.equal(entry.originalColumn, segment.originalColumn); + assert.equal(entry.name, segment.name === undefined ? undefined : map.names[segment.name]); + checked += 1; + }); + }); + assert.ok(checked > 200, 'the fixture exercises many mapped segments'); +}); + +test('maps a real esbuild frame below a hashbang and past astral characters to its original column', async () => { + const source = [ + '#!/usr/bin/env node', + "const label: string = '😀 ✓ — ü';", + 'const template = `one', + '', + 'three`;', + 'export function fail(reason: string): never {', + ' const note = reason;', + " void '😀😀'; throw new Error(note + label + template);", + '}', + "fail('x');", + '', + ].join('\n'); + writeFileSync(join(scratch, 'frame.ts'), source); + const built = await build({ + absWorkingDir: scratch, + entryPoints: ['frame.ts'], + outdir: 'out', + format: 'esm', + platform: 'node', + target: 'node22', + bundle: true, + minify: true, + sourcemap: 'external', + sourcesContent: false, + write: false, + logLevel: 'silent', + }); + const js = built.outputFiles.find((file) => file.path.endsWith('.js')).text; + const map = JSON.parse(built.outputFiles.find((file) => file.path.endsWith('.map')).text); + writeFileSync(join(scratch, 'frame.mjs'), js); + const decoded = decodeMappings(map.mappings, { + sources: map.sources.length, + names: map.names.length, + }); + // The hashbang stays on line 1 with no mapping, so code mappings start on line 2. + assert.match(js, /^#!\/usr\/bin\/env node\n/u); + assert.deepEqual(decoded[0], []); + assertGeneratedRange(decoded, lineLengths(js)); + assertOriginalRange(decoded, [lineLengths(source)]); + // Run it and read the throw site's 1-based line and column from V8's own structured call site. + const run = spawnSync( + process.execPath, + [ + '--input-type=module', + '--eval', + `Error.prepareStackTrace = (_, sites) => sites; +try { await import(${JSON.stringify(pathToFileURL(join(scratch, 'frame.mjs')).href)}); } +catch (error) { const site = error.stack.find((s) => s.getFileName()?.endsWith('frame.mjs')); + process.stdout.write(JSON.stringify([site.getLineNumber(), site.getColumnNumber()])); }`, + ], + { encoding: 'utf8', timeout: 20_000 }, + ); + assert.equal(run.status, 0, run.stderr); + const [line, column] = JSON.parse(run.stdout); + assert.ok(line >= 2); + const segment = segmentAt(decoded, line - 1, column - 1); + assert.ok(segment !== undefined, 'the throw site starts an exact segment'); + // `new` on source line 8, in UTF-16 code units: the two astral characters before it count twice. + assert.deepEqual( + [basename(map.sources[segment.source]), segment.originalLine + 1, segment.originalColumn + 1], + ['frame.ts', 8, 22], + ); + // One column off is no mapping at all, never the nearest preceding segment. + assert.equal(segmentAt(decoded, line - 1, column), undefined); +}); + +test('accepts empty maps, empty lines and unmapped segments', () => { + assert.deepEqual(decodeMappings('', COUNTS), [[]]); + assert.deepEqual(decodeMappings(';;', COUNTS), [[], [], []]); + assert.deepEqual(decodeMappings('A;C', COUNTS), [ + [{ generatedColumn: 0 }], + [{ generatedColumn: 1 }], + ]); + assert.deepEqual(decodeMappings('AAAA,CCCCC;', COUNTS), [ + [ + { generatedColumn: 0, source: 0, originalLine: 0, originalColumn: 0 }, + { generatedColumn: 1, source: 1, originalLine: 1, originalColumn: 1, name: 1 }, + ], + [], + ]); +}); + +test('refuses every malformed mapping with its own code', () => { + const largest = vlq(2 ** 31 - 1); + assert.equal(decodeMappings(largest, COUNTS)[0][0].generatedColumn, 2 ** 31 - 1); + for (const [mappings, code, counts = COUNTS] of [ + ['AAAA!', 'MAPPINGS_CHARACTER'], + ['AAéA', 'MAPPINGS_CHARACTER'], + ['A A', 'MAPPINGS_CHARACTER'], + ['g', 'MAPPINGS_TRUNCATED'], + ['AAAg', 'MAPPINGS_TRUNCATED'], + ['AA', 'MAPPINGS_FIELDS'], + ['AAA', 'MAPPINGS_FIELDS'], + ['AAAAAA', 'MAPPINGS_FIELDS'], + [',', 'MAPPINGS_EMPTY_SEGMENT'], + ['A,', 'MAPPINGS_EMPTY_SEGMENT'], + ['A,,A', 'MAPPINGS_EMPTY_SEGMENT'], + ['A,;A', 'MAPPINGS_EMPTY_SEGMENT'], + [';,A', 'MAPPINGS_EMPTY_SEGMENT'], + ['D', 'MAPPINGS_RANGE'], + ['ADAA', 'MAPPINGS_RANGE'], + ['AEAA', 'MAPPINGS_RANGE'], + ['AADA', 'MAPPINGS_RANGE'], + ['AAAD', 'MAPPINGS_RANGE'], + ['AAAAE', 'MAPPINGS_RANGE'], + ['AAAAD', 'MAPPINGS_RANGE'], + ['AAAA', 'MAPPINGS_RANGE', { sources: 0, names: 0 }], + [vlq(2 ** 32), 'MAPPINGS_RANGE'], + ['C,A', 'MAPPINGS_ORDER'], + ['E,D', 'MAPPINGS_ORDER'], + ]) { + assert.throws(() => decodeMappings(mappings, counts), refuses(code), mappings); + } + assert.throws(() => decodeMappings(undefined, COUNTS), refuses('MAPPINGS_TYPE')); + assert.throws(() => decodeMappings('!', COUNTS), { + name: 'SourceMapError', + code: 'MAPPINGS_CHARACTER', + message: 'source map refused: MAPPINGS_CHARACTER', + }); + // A refusal never quotes the mappings it read. + assert.throws( + () => decodeMappings('secretAAAA!', COUNTS), + (error) => !error.message.includes('secret'), + ); +}); + +test('bounds every value at exactly 0xffffffff in at most seven digits, and only base64 decodes', () => { + // The largest admitted value, 0xffffffff, is an original-line delta of -(2^31 - 1). + const back = `AA${vlq(2 ** 31 - 1)}A,CA${vlq(-(2 ** 31 - 1))}A`; + assert.equal(decodeMappings(back, COUNTS)[0][1].originalLine, 0); + // Seven digits decode; zero-payload padding past them refuses rather than growing the scale to + // Infinity, which would decode NaN and pass every later comparison. + assert.equal(decodeMappings(`${'g'.repeat(6)}A`, COUNTS)[0][0].generatedColumn, 0); + for (const [label, mappings, counts = COUNTS] of [ + ['one past the bound', vlq(2 ** 31)], + ['eight digits', `${'g'.repeat(7)}A`], + ['seven full digits and more', `${'/'.repeat(7)}${'g'.repeat(198)}A`], + ['a padded generated column', `${'g'.repeat(205)}A`], + ['a padded original column', `AAA${'g'.repeat(205)}A`], + ['a padded name index', `AAAA${'g'.repeat(205)}A`, { sources: 1, names: 0 }], + ]) { + assert.throws(() => decodeMappings(mappings, counts), refuses('MAPPINGS_RANGE'), label); + } + // DEL and the first non-ASCII code unit are no digits. + for (const code of [0x7f, 0x80]) { + assert.throws( + () => decodeMappings(String.fromCharCode(code), COUNTS), + refuses('MAPPINGS_CHARACTER'), + ); + } + // A column behind the previous segment refuses even when it is ahead of the line's first. + assert.throws(() => decodeMappings('A,K,F', COUNTS), refuses('MAPPINGS_ORDER')); +}); + +test('finds only the segment that starts exactly at a position', () => { + const lines = decodeMappings('A,EAAE,KAEC;;GAAA', COUNTS); + assert.equal(segmentAt(lines, 0, 0).generatedColumn, 0); + assert.equal(segmentAt(lines, 0, 2).originalColumn, 2); + assert.equal(segmentAt(lines, 0, 7).originalLine, 2); + for (const [line, column] of [ + [0, 1], + [0, 3], + [0, 8], + [1, 0], + [2, 0], + [2, 2], + [3, 0], + [-1, 0], + ]) { + assert.equal(segmentAt(lines, line, column), undefined, `${line}:${column}`); + } + // A line's only segment is found exactly. + assert.equal(segmentAt(decodeMappings('A;;GAAA', COUNTS), 2, 3).generatedColumn, 3); +}); + +test('counts lines and UTF-16 columns as V8 and TypeScript do', () => { + assert.deepEqual(lineLengths('ab\ncd\r\nefg\rh
😀
'), [2, 2, 3, 1, 2, 0]); + assert.deepEqual(lineLengths(''), [0]); + // U+2028 and U+2029 end a line; NEL (U+0085) does not. + const [nel, ls, ps] = [0x85, 0x2028, 0x2029].map((code) => String.fromCharCode(code)); + assert.deepEqual(lineLengths(`a${ls}b${ps}c`), [1, 1, 1]); + assert.deepEqual(lineLengths(`a${nel}b`), [3]); +}); + +test('requires generated and original positions inside their texts', () => { + const lines = decodeMappings('A,EAAE;CACA', COUNTS); + assertGeneratedRange(lines, [2, 1]); + assert.throws(() => assertGeneratedRange(lines, [1, 1]), refuses('GENERATED_RANGE')); + assert.throws(() => assertGeneratedRange(lines, [2]), refuses('GENERATED_RANGE')); + // Original positions (0, 2) and (1, 2) of source 0; a column at a line's end is a caret position. + assertOriginalRange(lines, [[2, 2], []]); + assert.throws(() => assertOriginalRange(lines, [[2, 1], []]), refuses('ORIGINAL_RANGE')); + assert.throws(() => assertOriginalRange(lines, [[1, 2], []]), refuses('ORIGINAL_RANGE')); + assert.throws(() => assertOriginalRange(lines, [[2], []]), refuses('ORIGINAL_RANGE')); + assert.throws(() => assertOriginalRange(lines, [[], []]), refuses('ORIGINAL_RANGE')); + // Unmapped segments name no source and pass. + assertOriginalRange(decodeMappings('A,C', COUNTS), []); + // Each original position is checked against its own source's lines, and a source with none + // refuses. + assertOriginalRange(decodeMappings('ACAA,CAAE', COUNTS), [[], [2]]); + assert.throws( + () => assertOriginalRange(decodeMappings('ACAE', COUNTS), [[5], [1]]), + refuses('ORIGINAL_RANGE'), + ); + assert.throws( + () => assertOriginalRange(decodeMappings('AAAA', COUNTS), []), + refuses('ORIGINAL_RANGE'), + ); +}); diff --git a/tools/cli-package/symbolicate.mjs b/tools/cli-package/symbolicate.mjs new file mode 100644 index 00000000..42f03f76 --- /dev/null +++ b/tools/cli-package/symbolicate.mjs @@ -0,0 +1,377 @@ +/** + * Symbolication of actual packed CLI and engine frames (ADR-0111; packaging.md "Candidate + * finalization — Step 9" item 5; Step 9a.2a). After the archive inspection and the private manifest, + * the producer runs the finalized `dist` — byte-equal to the tarball, re-hashed per frame — under the + * tooling-only preload (frame-capture.mjs) while it imports a canary workflow, and maps the captured + * frames through the staged final maps. Each anchor below must be reached by a synchronous frame that + * lands exactly on a mapping segment and resolves to the call site that the matching source + * revision's syntax tree gives: the candidate copy for a dirty local build, the checkout's Git objects + * at the recorded commit otherwise, both bound to the manifest's per-source digests. Each frame file is + * re-hashed after the run. A refusal names a fixed code; no frame, path, position or source text is + * ever printed. + */ +import { Buffer } from 'node:buffer'; +import { randomBytes } from 'node:crypto'; +import { lstatSync, mkdirSync, realpathSync, writeFileSync } from 'node:fs'; +import nodeModule, { createRequire } from 'node:module'; +import { basename, dirname, join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { readPrivateManifest } from './manifest.mjs'; +import { readOrdinaryFile, sha256 } from './files.mjs'; +import { isolatedEnvironment, isolatedGitEnvironment, runCommand } from './process.mjs'; +import { SourceMapError, decodeMappings, segmentAt } from './source-map.mjs'; + +export class SymbolicationError extends Error { + /** @param {string} code */ + constructor(code) { + super(`PACKED_CLI_SYMBOLICATION_${code}`); + this.name = 'SymbolicationError'; + this.code = code; + } +} + +/** @param {string} code @returns {never} */ +function refuse(code) { + throw new SymbolicationError(code); +} + +/** + * The named call sites the canary import passes through: three in the CLI and two in the inlined + * `@relavium/core`. Each is the one call to `call` inside the function `within`; its expected line and + * column come from the source revision, never from this table. + */ +export const ANCHORS = Object.freeze([ + { source: 'packages/core/src/yaml-decode.ts', within: 'decodeHardenedYaml', call: 'parseYaml' }, + { source: 'packages/core/src/parser.ts', within: 'parseWorkflow', call: 'decodeHardenedYaml' }, + { + source: 'apps/cli/src/authoring/authoring.ts', + within: 'parseAsWorkflow', + call: 'parseWorkflow', + }, + { + source: 'apps/cli/src/authoring/authoring.ts', + within: 'detectAndParse', + call: 'parseAsWorkflow', + }, + { source: 'apps/cli/src/commands/import.ts', within: 'importCommand', call: 'detectAndParse' }, +]); + +const PRELOAD = fileURLToPath(new URL('./frame-capture.mjs', import.meta.url)); + +/** + * Whether this Node can run the capture: the preload confines module resolution through + * `module.registerHooks` (Node 22.15 and later), which the workspace's Node 22.13 floor lacks. + * @param {{registerHooks?: unknown}} [module] + */ +export function captureSupported(module = nodeModule) { + return typeof module.registerHooks === 'function'; +} +const FRAME_LIMIT = 64; + +/** A workflow the CLI accepts, whose id carries the canary the preload watches for. */ +function canaryDocument(canary) { + return [ + "schema_version: '1.0'", + 'workflow:', + ` id: ${canary}`, + ' name: Symbolication canary', + ' nodes:', + ' - { id: start, type: input }', + ' - { id: out, type: output }', + ' edges:', + ' - { from: start, to: out }', + '', + ].join('\n'); +} + +/** + * @typedef {{file: string | null, line: number, column: number, async: boolean}} Frame + */ + +/** + * Run the finalized CLI's `import` of a canary workflow under the preload, with its home, project and + * temporary files in `work` (owner-only, outside the candidate), and return the one captured stack. + * The project's own `.relavium/` exists before the run, so the CLI's upward search for a project stops + * there and never adopts an ancestor's — a developer's real one on Windows, where the temporary + * directory lies below the home; the run must have imported the canary into it. + * @param {{candidate: string, work: string, run?: typeof runCommand}} options + * @returns {Promise} + */ +export async function captureFrames({ candidate, work, run = runCommand }) { + const canary = `frames-canary-${randomBytes(6).toString('hex')}`; + const home = join(work, 'home'); + const project = join(work, 'project'); + const catalog = join(project, '.relavium'); + for (const directory of [home, project, catalog]) mkdirSync(directory, { mode: 0o700 }); + const document = join(work, `${canary}.relavium.yaml`); + writeFileSync(document, canaryDocument(canary), { mode: 0o600, flag: 'wx' }); + const log = join(work, 'frames.json'); + try { + await run( + process.execPath, + [ + '--import', + pathToFileURL(PRELOAD).href, + join(candidate, 'apps', 'cli', 'dist', 'index.js'), + 'import', + document, + '--cwd', + project, + ], + { + cwd: work, + env: { + ...isolatedEnvironment(home), + RELAVIUM_FRAME_CAPTURE_LOG: log, + RELAVIUM_FRAME_CAPTURE_PACKAGE: join(candidate, 'apps', 'cli', 'package.json'), + RELAVIUM_FRAME_CAPTURE_CANARY: canary, + }, + timeoutMs: 120_000, + }, + ); + } catch { + // The child's own output is never printed: it can quote the candidate. + refuse('CAPTURE_FAILED'); + } + let imported = false; + try { + imported = lstatSync(join(catalog, 'workflows', `${canary}.relavium.yaml`)).isFile(); + } catch { + imported = false; + } + if (!imported) refuse('CAPTURE_PROJECT'); + let record; + try { + record = JSON.parse(readOrdinaryFile(log, 1024 * 1024).toString('utf8')); + } catch { + refuse('CAPTURE_FAILED'); + } + if (record?.version !== 1 || !Array.isArray(record.captures)) refuse('CAPTURE_SHAPE'); + if (record.captures.length !== 1) refuse('CAPTURE_COUNT'); + const [frames] = record.captures; + if ( + !Array.isArray(frames) || + frames.length === 0 || + frames.length > FRAME_LIMIT || + !frames.every( + (frame) => + frame !== null && + typeof frame === 'object' && + (frame.file === null || typeof frame.file === 'string') && + typeof frame.async === 'boolean' && + (frame.file === null || + (Number.isSafeInteger(frame.line) && + frame.line > 0 && + Number.isSafeInteger(frame.column) && + frame.column > 0)), + ) + ) { + refuse('CAPTURE_SHAPE'); + } + return frames; +} + +/** + * The text of each path at the matching source revision, bound to the manifest's digest: the + * candidate copy for a dirty local build, otherwise the blob at the recorded commit, read from the + * original checkout's object store by Git without home, global or system configuration, without + * filters or text conversion (`cat-file blob`) and without replace refs. + * @param {{candidate: string, checkout: string, manifest: import('./manifest.mjs').PrivateManifest, + * paths: readonly string[], run?: typeof runCommand}} options + * @returns {Promise>} + */ +export async function readSourceRevision({ + candidate, + checkout, + manifest, + paths, + run = runCommand, +}) { + const texts = new Map(); + for (const path of paths) { + const digest = manifest.sources[path]; + if (digest === undefined) refuse('SOURCE_REVISION'); + let bytes; + try { + bytes = manifest.source.dirty + ? readOrdinaryFile(join(candidate, ...path.split('/')), 8 * 1024 * 1024) + : Buffer.from( + ( + await run( + 'git', + ['--no-replace-objects', 'cat-file', 'blob', `${manifest.source.commit}:${path}`], + { cwd: checkout, env: isolatedGitEnvironment(), timeoutMs: 60_000 }, + ) + ).stdout, + 'utf8', + ); + } catch { + refuse('SOURCE_REVISION'); + } + if (sha256(bytes) !== digest) refuse('SOURCE_REVISION'); + texts.set(path, bytes.toString('utf8')); + } + return texts; +} + +/** + * The 1-based line and UTF-16 column of an anchor's call site in `text`: the start of the callee's + * name in the one call to `call` inside the one function named `within`. + * @param {typeof import('typescript')} ts @param {string} text + * @param {{source: string, within: string, call: string}} anchor + */ +export function anchorPosition(ts, text, anchor) { + const file = ts.createSourceFile( + anchor.source, + text, + ts.ScriptTarget.Latest, + true, + anchor.source.endsWith('.tsx') ? ts.ScriptKind.TSX : ts.ScriptKind.TS, + ); + const owners = []; + const visit = (node) => { + if ( + (ts.isFunctionDeclaration(node) || ts.isMethodDeclaration(node)) && + node.body !== undefined && + node.name !== undefined && + node.name.getText(file) === anchor.within + ) { + owners.push(node); + } else if ( + ts.isVariableDeclaration(node) && + ts.isIdentifier(node.name) && + node.name.text === anchor.within && + node.initializer !== undefined && + (ts.isArrowFunction(node.initializer) || ts.isFunctionExpression(node.initializer)) + ) { + owners.push(node.initializer); + } + ts.forEachChild(node, visit); + }; + visit(file); + if (owners.length !== 1) refuse('ANCHOR_SOURCE'); + const callees = []; + const find = (node) => { + if (ts.isCallExpression(node)) { + const callee = node.expression; + if (ts.isIdentifier(callee) && callee.text === anchor.call) callees.push(callee); + else if (ts.isPropertyAccessExpression(callee) && callee.name.text === anchor.call) { + callees.push(callee.name); + } + } + ts.forEachChild(node, find); + }; + find(owners[0]); + if (callees.length !== 1) refuse('ANCHOR_SOURCE'); + const { line, character } = file.getLineAndCharacterOfPosition(callees[0].getStart(file)); + return { line: line + 1, column: character + 1 }; +} + +/** + * Map the captured frames and require every anchor, against the manifest sealed into `staging`. Each + * frame inside the finalized `dist` must be the very file the manifest and the tarball record; its + * map must be the staged bytes the manifest records and decode strictly; only an exact segment counts. + * @param {{candidate: string, staging: string, checkout: string, work: string, + * archive: {files: {path: string, sha256: string}[]}, run?: typeof runCommand, + * capture?: typeof captureFrames, revision?: typeof readSourceRevision, + * anchors?: readonly {source: string, within: string, call: string}[]}} options + * @returns {Promise<{anchors: number}>} + */ +export async function symbolicateCandidate({ + candidate, + staging, + checkout, + work, + archive, + run = runCommand, + capture = captureFrames, + revision = readSourceRevision, + anchors = ANCHORS, +}) { + // The maintainer's coverage ruling: at least two `@relavium/core` and three CLI call sites, counted + // as distinct call sites, so a repeated entry does not count twice. + const within = (root) => + new Set( + anchors + .filter((anchor) => anchor.source.startsWith(root)) + .map((anchor) => JSON.stringify([anchor.source, anchor.within, anchor.call])), + ).size; + if (within('packages/core/src/') < 2 || within('apps/cli/src/') < 3) refuse('ANCHORS'); + const manifest = readPrivateManifest(staging); + const frames = await capture({ candidate, work, run }); + const paths = [...new Set(anchors.map((anchor) => anchor.source))]; + const texts = await revision({ candidate, checkout, manifest, paths, run }); + let ts; + try { + ts = createRequire(join(candidate, 'apps', 'cli', 'package.json'))('typescript'); + } catch { + refuse('TOOLCHAIN'); + } + const outputs = new Map(manifest.outputs.map((output) => [output.path, output])); + const packed = new Map(archive.files.map((file) => [file.path, file.sha256])); + const dist = realpathSync.native(join(candidate, 'apps', 'cli', 'dist')); + const decoded = new Map(); + const mapsOf = (output) => { + let entry = decoded.get(output.path); + if (entry !== undefined) return entry; + const bytes = readOrdinaryFile( + join(staging, 'maps', `${basename(output.path)}.map`), + 16 * 1024 * 1024, + ); + if (sha256(bytes) !== output.map.sha256) refuse('MAP_DIGEST'); + try { + const map = JSON.parse(bytes.toString('utf8')); + entry = { + sources: map.sources, + lines: decodeMappings(map.mappings, { + sources: map.sources.length, + names: map.names.length, + }), + }; + } catch (error) { + if (error instanceof SourceMapError || error instanceof SyntaxError) refuse('MAP_DECODE'); + throw error; + } + decoded.set(output.path, entry); + return entry; + }; + const reached = []; + for (const frame of frames) { + if (frame.async || !frame.file?.startsWith('file:')) continue; + let path; + try { + path = realpathSync.native(fileURLToPath(frame.file)); + } catch { + refuse('FRAME_FILE'); + } + // Frames outside the finalized dist (the preload, yaml, commander) carry no anchor. + if (dirname(path) !== dist) continue; + const key = `dist/${basename(path)}`; + const output = outputs.get(key); + if (output === undefined) refuse('FRAME_FILE'); + const digest = sha256(readOrdinaryFile(path, 16 * 1024 * 1024)); + if (digest !== output.sha256 || packed.get(`package/${key}`) !== digest) refuse('FRAME_FILE'); + const { sources, lines } = mapsOf(output); + const segment = segmentAt(lines, frame.line - 1, frame.column - 1); + if (segment?.source === undefined) continue; + reached.push({ + source: sources[segment.source], + line: (segment.originalLine ?? 0) + 1, + column: (segment.originalColumn ?? 0) + 1, + }); + } + for (const anchor of anchors) { + const expected = anchorPosition(ts, texts.get(anchor.source), anchor); + if ( + !reached.some( + (frame) => + frame.source === anchor.source && + frame.line === expected.line && + frame.column === expected.column, + ) + ) { + refuse('ANCHOR_UNMAPPED'); + } + } + return { anchors: anchors.length }; +} diff --git a/tools/cli-package/symbolicate.test.mjs b/tools/cli-package/symbolicate.test.mjs new file mode 100644 index 00000000..eef63cad --- /dev/null +++ b/tools/cli-package/symbolicate.test.mjs @@ -0,0 +1,1050 @@ +import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { dirname, join } from 'node:path'; +import { test } from 'node:test'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { + COMMIT, + PRELOAD, + SOURCES, + candidate, + cliRequire, + coreChunk, + digest, + fresh, + posixModes, + put, + reseal, + scratch, + seal, + ts, +} from './candidate-fixture.mjs'; +import * as keyringStub from './frame-capture-keyring.mjs'; +import { ManifestError, sealPrivateManifest } from './manifest.mjs'; +import { isolatedEnvironment, isolatedGitEnvironment, runCommand } from './process.mjs'; +import { + ANCHORS, + SymbolicationError, + anchorPosition, + captureFrames, + captureSupported, + readSourceRevision, + symbolicateCandidate, +} from './symbolicate.mjs'; + +const refuses = (code) => (error) => error instanceof SymbolicationError && error.code === code; +const repository = fileURLToPath(new URL('../../', import.meta.url)); + +function symbolicate(fixture, options = {}) { + return symbolicateCandidate({ + candidate: fixture.root, + staging: fixture.staging, + checkout: fixture.root, + work: fixture.work, + archive: fixture.archive, + ...options, + }); +} + +/** A sealed fixture and the frames one real capture of it produced. */ +async function sealedWithFrames() { + const fixture = await candidate(); + await seal(fixture); + const frames = await captureFrames({ candidate: fixture.root, work: fixture.work }); + return { fixture, frames }; +} + +/** Re-record the manifest's map digest for `chunk` after a deliberate change to its staged map. */ +function rerecordManifestMap(fixture, chunk) { + const path = join(fixture.staging, 'manifest.json'); + const manifest = JSON.parse(readFileSync(path, 'utf8')); + const bytes = readFileSync(join(fixture.staging, 'maps', `${chunk}.map`)); + for (const output of manifest.outputs) { + if (output.path === `dist/${chunk}`) output.map.sha256 = digest(bytes); + } + writeFileSync(path, JSON.stringify(manifest, null, 2) + '\n'); +} + +/** Entry code that imports the canary into the run's project, as the real CLI does. */ +const IMPORTS = + "import { mkdirSync as m, writeFileSync as w } from 'node:fs'; import { join as j } from 'node:path';\n" + + 'const [, , , project] = process.argv.slice(2);\n' + + "m(j(project, '.relavium', 'workflows'), { recursive: true });\n" + + "w(j(project, '.relavium', 'workflows', `${process.env.RELAVIUM_FRAME_CAPTURE_CANARY}.relavium.yaml`), '');\n"; + +test('actual packed frames reach every anchor through the real preload and exact segments', async () => { + const fixture = await candidate(); + await seal(fixture); + let frames; + const result = await symbolicate(fixture, { + capture: async (options) => (frames = await captureFrames(options)), + }); + assert.deepEqual(result, { anchors: ANCHORS.length }); + // One stack crossed the entry and the lazy chunk that inlines the core. + const files = new Set( + frames.filter((frame) => frame.file?.includes('/dist/')).map((frame) => frame.file), + ); + assert.ok(files.size >= 2, 'the stack crosses more than one finalized chunk'); + // The canary was imported into the run's own project, never the candidate. + const imported = readdirSync(join(fixture.work, 'project', '.relavium', 'workflows')); + assert.equal(imported.length, 1); + assert.match(imported[0], /^frames-canary-[0-9a-f]{12}\.relavium\.yaml$/u); + assert.equal(existsSync(join(fixture.root, '.relavium')), false); + // The count is the table's, not a constant. + assert.deepEqual( + await symbolicate(fixture, { anchors: [...ANCHORS, ANCHORS[0]], capture: async () => frames }), + { anchors: ANCHORS.length + 1 }, + ); +}); + +test('an ancestor project of the work directory is never adopted by the canary import', async () => { + // A developer's real `.relavium/` above the temporary directory, as on Windows. + const ancestor = fresh('ancestor'); + mkdirSync(join(ancestor, '.relavium'), { recursive: true }); + const fixture = await candidate({ parent: ancestor }); + await seal(fixture); + assert.deepEqual(await symbolicate(fixture), { anchors: ANCHORS.length }); + assert.deepEqual(readdirSync(join(ancestor, '.relavium')), []); + assert.equal(readdirSync(join(fixture.work, 'project', '.relavium', 'workflows')).length, 1); + // A run that imports nothing into its project refuses. + const elsewhere = await candidate(); + writeFileSync( + join(elsewhere.cli, 'dist', 'index.js'), + "import { readFileSync } from 'node:fs'; import { parse } from 'yaml';\n" + + "parse(readFileSync(process.argv[3], 'utf8'));\n", + ); + await assert.rejects( + captureFrames({ candidate: elsewhere.root, work: elsewhere.work }), + refuses('CAPTURE_PROJECT'), + ); + // So does one that writes its project's catalog but not the canary. + const other = fresh('other-import'); + mkdirSync(other, { mode: 0o700 }); + await assert.rejects( + captureFrames({ + candidate: scratch, + work: other, + run: async (_command, args, options) => { + const workflows = join(args.at(-1), '.relavium', 'workflows'); + mkdirSync(workflows); + writeFileSync(join(workflows, 'another.relavium.yaml'), ''); + writeFileSync( + options.env.RELAVIUM_FRAME_CAPTURE_LOG, + JSON.stringify({ + version: 1, + captures: [[{ file: null, line: null, column: null, async: false }]], + }), + ); + return { stdout: '', stderr: '' }; + }, + }), + refuses('CAPTURE_PROJECT'), + ); +}); + +test('the anchors keep the coverage ruling and resolve in the real repository', async () => { + const core = ANCHORS.filter((anchor) => anchor.source.startsWith('packages/core/src/')); + const cli = ANCHORS.filter((anchor) => anchor.source.startsWith('apps/cli/src/')); + assert.ok(core.length >= 2, 'at least two core anchors'); + assert.ok(cli.length >= 3, 'at least three CLI anchors'); + assert.equal(Object.isFrozen(ANCHORS), true); + for (const anchor of ANCHORS) { + const text = readFileSync(join(repository, ...anchor.source.split('/')), 'utf8'); + const position = anchorPosition(ts, text, anchor); + const line = text.split(/\r\n|[\n\r\u2028\u2029]/u)[position.line - 1]; + assert.equal(line.slice(position.column - 1).startsWith(anchor.call), true, anchor.within); + } + // A table below the ruling refuses before anything runs. + const fixture = await candidate(); + await seal(fixture); + for (const anchors of [ + [], + ANCHORS.slice(1), + ANCHORS.slice(0, 4), + ANCHORS.filter((anchor) => !anchor.source.startsWith('packages/core/')), + // A repeated entry is one call site. + [ANCHORS[0], ANCHORS[0], ANCHORS[2], ANCHORS[2], ANCHORS[2]], + [...ANCHORS.slice(1), ANCHORS[1]], + // Call sites in other engine packages, or outside a package's sources, are not core anchors. + [ + ...cli, + { source: 'packages/shared/src/a.ts', within: 'f', call: 'g' }, + { source: 'packages/core/test/b.ts', within: 'f', call: 'g' }, + ], + ]) { + let ran = false; + await assert.rejects( + symbolicate(fixture, { + anchors, + capture: async () => { + ran = true; + return []; + }, + }), + refuses('ANCHORS'), + ); + assert.equal(ran, false); + } +}); + +test('the expected positions come from the source revision, independently of any map', () => { + for (const anchor of ANCHORS) { + const text = SOURCES[anchor.source].join('\n'); + const position = anchorPosition(ts, text, anchor); + const line = text.split('\n')[position.line - 1]; + assert.equal(line.slice(position.column - 1).startsWith(anchor.call), true, anchor.call); + } + const at = (text, within = 'f', call = 'g') => + anchorPosition(ts, text, { source: 'a.ts', within, call }); + // A member call is positioned at its property name. + assert.deepEqual(at('function f() {\n yaml.parse(x);\n}\n', 'f', 'parse'), { + line: 2, + column: 8, + }); + // Declaration, arrow, function expression and method owners; a call outside the owner is ignored. + assert.deepEqual(at('function f() { g(); }\nfunction h() { g(); }'), { line: 1, column: 16 }); + assert.deepEqual(at('const f = () => {\n g();\n};'), { line: 2, column: 3 }); + assert.deepEqual(at('const f = function () {\n g();\n};'), { line: 2, column: 3 }); + assert.deepEqual(at('class A {\n f() {\n g();\n }\n}'), { line: 3, column: 5 }); + assert.deepEqual(at('const f = () => g();\nconst h = () => g();'), { line: 1, column: 17 }); + assert.deepEqual(at('let f;\nfunction f() { g(); }'), { line: 2, column: 16 }); + assert.deepEqual(at('export default function () { g(); }\nfunction f() { g(); }'), { + line: 2, + column: 16, + }); + // Bodiless signatures (overloads, declarations) are not owners; the implementation is. + assert.deepEqual( + at( + 'function f(a: string): void;\nfunction f(a: number): void;\nfunction f(a: unknown) { g(); }', + ), + { line: 3, column: 26 }, + ); + assert.deepEqual(at('declare function f(): void;\nfunction f() { g(); }'), { + line: 2, + column: 16, + }); + // A missing owner, a second owner, no call or a second call is ambiguous. + for (const text of [ + 'function h() { g(); }', + 'function f() { g(); g(); }', + 'function f() { h(); }', + 'function f() { g(); }\nfunction f() { g(); }', + ]) { + assert.throws(() => at(text), refuses('ANCHOR_SOURCE'), text); + } + // A refusal names only its fixed code. + assert.throws(() => at('function h() {}'), { + name: 'SymbolicationError', + code: 'ANCHOR_SOURCE', + message: 'PACKED_CLI_SYMBOLICATION_ANCHOR_SOURCE', + }); +}); + +test('wrong JS, a wrong map, permuted sources and a wrong source revision cannot reach the anchors', async () => { + // Every digest is re-recorded after each change, so only the mapping layer can refuse it. + for (const [label, change] of [ + [ + 'JS shifted by one column', + (fixture) => { + const path = join(fixture.cli, 'dist', coreChunk(fixture)); + writeFileSync(path, readFileSync(path, 'utf8').replace(/^/gmu, ' ')); + }, + ], + [ + 'JS shifted by one line', + (fixture) => { + const path = join(fixture.cli, 'dist', coreChunk(fixture)); + writeFileSync(path, `\n${readFileSync(path, 'utf8')}`); + }, + ], + [ + 'a map from a build one line apart', + (fixture) => { + const path = join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`); + const map = JSON.parse(readFileSync(path, 'utf8')); + map.mappings = `;${map.mappings}`; + writeFileSync(path, JSON.stringify(map)); + }, + ], + [ + 'a map whose sources are permuted', + (fixture) => { + const path = join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`); + const map = JSON.parse(readFileSync(path, 'utf8')); + map.sources.reverse(); + writeFileSync(path, JSON.stringify(map)); + }, + ], + [ + 'a source revision with one more line', + (fixture) => { + const path = join(fixture.root, 'packages', 'core', 'src', 'parser.ts'); + writeFileSync(path, `// shifted\n${readFileSync(path, 'utf8')}`); + }, + ], + [ + 'a source revision one column apart', + (fixture) => { + const path = join(fixture.root, 'packages', 'core', 'src', 'parser.ts'); + const text = readFileSync(path, 'utf8'); + assert.ok(text.includes('const raw = decodeHardenedYaml')); + writeFileSync( + path, + text.replace('const raw = decodeHardenedYaml', 'const raw = decodeHardenedYaml'), + ); + }, + ], + ]) { + const fixture = await candidate(); + change(fixture); + await reseal(fixture); + await assert.rejects(symbolicate(fixture), refuses('ANCHOR_UNMAPPED'), label); + } + // Another chunk's map names other sources, so the resealed manifest no longer covers the anchors. + const fixture = await candidate(); + writeFileSync( + join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`), + readFileSync(join(fixture.staging, 'maps', 'index.js.map')), + ); + await reseal(fixture); + await assert.rejects(symbolicate(fixture), refuses('SOURCE_REVISION')); +}); + +test('a call after non-ASCII text on its line still reaches its anchor', async () => { + const sources = { + ...SOURCES, + 'packages/core/src/yaml-decode.ts': [ + "import { parse as parseYaml } from 'yaml';", + '', + 'export function decodeHardenedYaml(text: string): unknown {', + " const mark = 'é—😀'; return mark.length > 0 ? parseYaml(text, { version: '1.2' }) : null;", + '}', + '', + ], + }; + const fixture = await candidate({ sources }); + await seal(fixture); + assert.deepEqual(await symbolicate(fixture), { anchors: ANCHORS.length }); +}); + +test('files that are not the manifest’s refuse by their digest', async () => { + for (const [label, change, code] of [ + [ + 'finalized JS changed after the manifest', + (fixture) => { + const path = join(fixture.cli, 'dist', coreChunk(fixture)); + writeFileSync(path, `${readFileSync(path, 'utf8')} `); + }, + 'FRAME_FILE', + ], + [ + 'finalized JS the tarball agrees with but the manifest does not', + (fixture) => { + const chunk = coreChunk(fixture); + const path = join(fixture.cli, 'dist', chunk); + writeFileSync(path, `${readFileSync(path, 'utf8')} `); + fixture.archive.files.find((file) => file.path === `package/dist/${chunk}`).sha256 = digest( + readFileSync(path), + ); + }, + 'FRAME_FILE', + ], + [ + 'a tarball entry that is not the run file', + (fixture) => { + const key = `package/dist/${coreChunk(fixture)}`; + fixture.archive.files.find((file) => file.path === key).sha256 = 'e'.repeat(64); + }, + 'FRAME_FILE', + ], + [ + 'a staged map changed after the manifest', + (fixture) => { + const path = join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`); + writeFileSync(path, `${readFileSync(path, 'utf8')} `); + }, + 'MAP_DIGEST', + ], + [ + 'a source changed after the manifest', + (fixture) => { + const path = join(fixture.root, 'apps', 'cli', 'src', 'commands', 'import.ts'); + writeFileSync(path, `${readFileSync(path, 'utf8')}\n`); + }, + 'SOURCE_REVISION', + ], + ]) { + const fixture = await candidate(); + await seal(fixture); + change(fixture); + await assert.rejects(symbolicate(fixture), refuses(code), label); + } + // Without a manifest the candidate is never run. + const bare = await candidate(); + let ran = false; + await assert.rejects( + symbolicate(bare, { + capture: async () => { + ran = true; + return []; + }, + }), + (error) => error instanceof ManifestError && error.code === 'UNREADABLE', + ); + assert.equal(ran, false); +}); + +test('a recorded map that does not decode refuses with its own code', async () => { + for (const [label, change] of [ + ['not JSON', () => '{'], + ['malformed mappings', (map) => JSON.stringify({ ...map, mappings: 'A!' })], + [ + 'a source index out of range', + (map) => JSON.stringify({ ...map, sources: map.sources.slice(0, -1) }), + ], + [ + 'a name index out of range', + (map) => JSON.stringify({ ...map, names: map.names.slice(0, -1) }), + ], + ]) { + const fixture = await candidate(); + await seal(fixture); + const chunk = coreChunk(fixture); + const path = join(fixture.staging, 'maps', `${chunk}.map`); + const map = JSON.parse(readFileSync(path, 'utf8')); + assert.ok(map.names.length > 0, 'the minified chunk records names'); + writeFileSync(path, change(map)); + rerecordManifestMap(fixture, chunk); + await assert.rejects(symbolicate(fixture), refuses('MAP_DECODE'), label); + } +}); + +test('only synchronous frames inside the finalized dist count, and an unrecorded one refuses', async () => { + const { fixture, frames } = await sealedWithFrames(); + assert.deepEqual(await symbolicate(fixture, { capture: async () => frames }), { + anchors: ANCHORS.length, + }); + // Anchors reached only by asynchronous frames do not count. + await assert.rejects( + symbolicate(fixture, { + capture: async () => frames.map((frame) => ({ ...frame, async: true })), + }), + refuses('ANCHOR_UNMAPPED'), + ); + // Native, node-internal, preload and sibling-directory frames are skipped. + const sibling = join(fixture.cli, 'distx'); + mkdirSync(sibling); + writeFileSync(join(sibling, 'a.js'), ''); + const skipped = [ + { file: null, line: null, column: null, async: false }, + { file: 'node:internal/modules/run_main', line: 1, column: 1, async: false }, + { file: pathToFileURL(PRELOAD).href, line: 1, column: 1, async: false }, + { file: pathToFileURL(join(sibling, 'a.js')).href, line: 1, column: 1, async: false }, + ]; + assert.deepEqual(await symbolicate(fixture, { capture: async () => [...skipped, ...frames] }), { + anchors: ANCHORS.length, + }); + // A dist frame the manifest does not record, or that cannot be resolved, refuses. + writeFileSync(join(fixture.cli, 'dist', 'extra.js'), ''); + for (const name of ['extra.js', 'missing.js']) { + const frame = { + file: pathToFileURL(join(fixture.cli, 'dist', name)).href, + line: 1, + column: 1, + async: false, + }; + await assert.rejects( + symbolicate(fixture, { capture: async () => [frame, ...frames] }), + refuses('FRAME_FILE'), + name, + ); + } + // A symlinked candidate path reaches the same dist. + const link = fresh('linked-candidate'); + symlinkSync(fixture.root, link, 'junction'); + assert.deepEqual(await symbolicate(fixture, { candidate: link, capture: async () => frames }), { + anchors: ANCHORS.length, + }); + // Without the candidate's TypeScript the run refuses with its code. A fresh candidate: Node's + // resolution cache would still answer for one that already resolved it. + const untooled = await candidate(); + await seal(untooled); + rmSync(join(untooled.cli, 'node_modules'), { recursive: true, force: true }); + await assert.rejects( + symbolicate(untooled, { capture: async () => frames }), + refuses('TOOLCHAIN'), + ); +}); + +/** Git without the host's configuration, for building test repositories. */ +function gitIn(directory, args) { + const run = spawnSync('git', args, { + cwd: directory, + encoding: 'utf8', + env: { PATH: process.env.PATH, HOME: directory, GIT_CONFIG_NOSYSTEM: '1' }, + }); + assert.equal(run.status, 0, run.stderr); + return run.stdout.trim(); +} + +function commitAll(directory) { + gitIn(directory, ['add', '.']); + gitIn(directory, [ + '-c', + 'user.name=t', + '-c', + 'user.email=t@example.invalid', + 'commit', + '--quiet', + '-m', + 'x', + ]); + return gitIn(directory, ['rev-parse', 'HEAD']); +} + +test('a clean candidate reads the recorded commit from the checkout, never HEAD or its own copy', async () => { + const fixture = await candidate(); + const checkout = fresh('checkout'); + mkdirSync(checkout); + gitIn(checkout, ['init', '--quiet']); + for (const [path, lines] of Object.entries(SOURCES)) put(checkout, path, lines.join('\n')); + const recorded = commitAll(checkout); + put( + checkout, + 'packages/core/src/parser.ts', + `// later\n${SOURCES['packages/core/src/parser.ts'].join('\n')}`, + ); + commitAll(checkout); + await sealPrivateManifest({ + candidate: fixture.root, + staging: fixture.staging, + checkout, + source: recorded, + dirty: false, + archive: fixture.archive, + }); + // The candidate copy changes after the seal too: a clean build must not read it. + const copy = join(fixture.root, 'packages', 'core', 'src', 'yaml-decode.ts'); + writeFileSync(copy, `// later\n${readFileSync(copy, 'utf8')}`); + // The run port reaches the capture and each Git read, one per anchor source. + const commands = []; + const run = (command, args, options) => { + commands.push(command === 'git' ? args.slice(0, 3).join(' ') : command); + return runCommand(command, args, options); + }; + assert.deepEqual(await symbolicate(fixture, { checkout, run }), { anchors: ANCHORS.length }); + assert.deepEqual(commands, [ + process.execPath, + ...Array(new Set(ANCHORS.map((anchor) => anchor.source)).size).fill( + '--no-replace-objects cat-file blob', + ), + ]); +}); + +test('a dirty candidate reads its own copy, never the checkout', async () => { + const fixture = await candidate(); + await seal(fixture); + const empty = fresh('empty-checkout'); + mkdirSync(empty); + assert.deepEqual(await symbolicate(fixture, { checkout: empty }), { anchors: ANCHORS.length }); +}); + +test('the source revision is read without configuration and refuses with its fixed code', async () => { + const path = 'apps/cli/src/commands/import.ts'; + const text = 'é — 😀\nexport function f() {}\n'; + const clean = { + source: { commit: COMMIT, dirty: false }, + sources: { [path]: digest(Buffer.from(text, 'utf8')) }, + }; + const calls = []; + const texts = await readSourceRevision({ + candidate: fresh('nowhere'), + checkout: scratch, + manifest: clean, + paths: [path], + run: async (command, args, options) => { + calls.push({ command, args, options }); + return { stdout: text, stderr: '' }; + }, + }); + // Non-ASCII text round-trips. + assert.equal(texts.get(path), text); + assert.deepEqual( + calls.map(({ command, args, options }) => [command, args, options.cwd, options.env]), + [ + [ + 'git', + ['--no-replace-objects', 'cat-file', 'blob', `${COMMIT}:${path}`], + scratch, + isolatedGitEnvironment(), + ], + ], + ); + let unlistedRead = false; + for (const [label, options] of [ + [ + 'a missing dirty copy', + { + candidate: fresh('nowhere'), + manifest: { source: { commit: COMMIT, dirty: true }, sources: { [path]: 'f'.repeat(64) } }, + }, + ], + [ + 'a failed Git read', + { + candidate: scratch, + manifest: clean, + run: async () => { + throw new Error('git failed'); + }, + }, + ], + [ + 'a blob that is not the built source', + { + candidate: scratch, + manifest: { ...clean, sources: { [path]: digest(`${text}\n`) } }, + run: async () => ({ stdout: text, stderr: '' }), + }, + ], + [ + 'a path outside the manifest, refused before Git runs', + { + candidate: scratch, + manifest: clean, + paths: ['apps/cli/src/x.ts'], + run: async () => { + unlistedRead = true; + return { stdout: text, stderr: '' }; + }, + }, + ], + ]) { + await assert.rejects( + readSourceRevision({ checkout: scratch, paths: [path], ...options }), + refuses('SOURCE_REVISION'), + label, + ); + } + assert.equal(unlistedRead, false); +}); + +test('the CLI runs in the private work directory with an isolated environment', async () => { + const fixture = await candidate(); + process.env.RELAVIUM_SYMBOLICATION_SENTINEL = 'ambient-secret'; + const calls = []; + try { + await captureFrames({ + candidate: fixture.root, + work: fixture.work, + run: (command, args, options) => { + calls.push({ command, args, options }); + return runCommand(command, args, options); + }, + }); + } finally { + delete process.env.RELAVIUM_SYMBOLICATION_SENTINEL; + } + const [{ command, args, options }] = calls; + assert.equal(calls.length, 1); + assert.equal(command, process.execPath); + const canary = options.env.RELAVIUM_FRAME_CAPTURE_CANARY; + assert.match(canary, /^frames-canary-[0-9a-f]{12}$/u); + const home = join(fixture.work, 'home'); + const project = join(fixture.work, 'project'); + const document = join(fixture.work, `${canary}.relavium.yaml`); + assert.deepEqual(args, [ + '--import', + pathToFileURL(PRELOAD).href, + join(fixture.root, 'apps', 'cli', 'dist', 'index.js'), + 'import', + document, + '--cwd', + project, + ]); + assert.equal(options.cwd, fixture.work); + assert.deepEqual(options.env, { + ...isolatedEnvironment(home), + RELAVIUM_FRAME_CAPTURE_LOG: join(fixture.work, 'frames.json'), + RELAVIUM_FRAME_CAPTURE_PACKAGE: join(fixture.root, 'apps', 'cli', 'package.json'), + RELAVIUM_FRAME_CAPTURE_CANARY: canary, + }); + if (posixModes) { + for (const directory of [home, project, join(project, '.relavium')]) { + assert.equal(statSync(directory).mode & 0o777, 0o700, directory); + } + assert.equal(statSync(document).mode & 0o777, 0o600); + } +}); + +test('the capture validates the record it reads', async () => { + const valid = { file: 'file:///x.js', line: 1, column: 1, async: false }; + const cases = [ + ['no record', undefined, 'CAPTURE_FAILED'], + [ + 'an oversized record', + `${JSON.stringify({ version: 1, captures: [[valid]] })}${' '.repeat(1024 * 1024)}`, + 'CAPTURE_FAILED', + ], + ['null', 'null', 'CAPTURE_SHAPE'], + ['another version', { version: 2, captures: [[valid]] }, 'CAPTURE_SHAPE'], + ['captures not an array', { version: 1, captures: 'xy' }, 'CAPTURE_SHAPE'], + ['captures array-like', { version: 1, captures: { 0: [valid], length: 1 } }, 'CAPTURE_SHAPE'], + ['no capture', { version: 1, captures: [] }, 'CAPTURE_COUNT'], + ['two captures', { version: 1, captures: [[valid], [valid]] }, 'CAPTURE_COUNT'], + ['a stack that is not an array', { version: 1, captures: [{}] }, 'CAPTURE_SHAPE'], + ['an empty stack', { version: 1, captures: [[]] }, 'CAPTURE_SHAPE'], + ['65 frames', { version: 1, captures: [Array(65).fill(valid)] }, 'CAPTURE_SHAPE'], + ['a null frame', { version: 1, captures: [[null]] }, 'CAPTURE_SHAPE'], + ['a numeric file', { version: 1, captures: [[{ ...valid, file: 5 }]] }, 'CAPTURE_SHAPE'], + [ + 'a non-boolean async', + { version: 1, captures: [[{ ...valid, async: 'no' }]] }, + 'CAPTURE_SHAPE', + ], + ['a fractional line', { version: 1, captures: [[{ ...valid, line: 1.5 }]] }, 'CAPTURE_SHAPE'], + ['line zero', { version: 1, captures: [[{ ...valid, line: 0 }]] }, 'CAPTURE_SHAPE'], + [ + 'a fractional column', + { version: 1, captures: [[{ ...valid, column: 1.5 }]] }, + 'CAPTURE_SHAPE', + ], + ['column zero', { version: 1, captures: [[{ ...valid, column: 0 }]] }, 'CAPTURE_SHAPE'], + [ + 'one bad frame among good ones', + { version: 1, captures: [[valid, { ...valid, line: 0 }]] }, + 'CAPTURE_SHAPE', + ], + ]; + const admitted = [ + ['64 frames', { version: 1, captures: [Array(64).fill(valid)] }], + [ + 'a native frame without a position', + { version: 1, captures: [[{ file: null, line: null, column: null, async: false }, valid]] }, + ], + ]; + // A stand-in run that imports the canary into its project and writes the given record. + const writing = (record) => async (_command, args, options) => { + const workflows = join(args.at(-1), '.relavium', 'workflows'); + mkdirSync(workflows, { recursive: true }); + writeFileSync( + join(workflows, `${options.env.RELAVIUM_FRAME_CAPTURE_CANARY}.relavium.yaml`), + '', + ); + if (record !== undefined) { + writeFileSync( + options.env.RELAVIUM_FRAME_CAPTURE_LOG, + typeof record === 'string' ? record : JSON.stringify(record), + ); + } + return { stdout: '', stderr: '' }; + }; + for (const [label, record, code] of cases) { + const work = fresh('record'); + mkdirSync(work, { mode: 0o700 }); + await assert.rejects( + captureFrames({ candidate: scratch, work, run: writing(record) }), + refuses(code), + label, + ); + } + for (const [label, record] of admitted) { + const work = fresh('record'); + mkdirSync(work, { mode: 0o700 }); + assert.deepEqual( + await captureFrames({ candidate: scratch, work, run: writing(record) }), + record.captures[0], + label, + ); + } +}); + +test('the capture refuses a run that fails, never parses the canary or parses it twice', async () => { + for (const [label, entry, code] of [ + ['a failed run', `${IMPORTS}process.exit(3);\n`, 'CAPTURE_FAILED'], + [ + 'no canary parse', + `${IMPORTS}import('yaml').then((y) => y.parse('a: 1'));\n`, + 'CAPTURE_COUNT', + ], + [ + 'two canary parses', + `${IMPORTS}import { readFileSync } from 'node:fs'; import { parse } from 'yaml';\n` + + "const t = readFileSync(process.argv[3], 'utf8'); parse(t); parse(t);\n", + 'CAPTURE_COUNT', + ], + ]) { + const fixture = await candidate(); + writeFileSync(join(fixture.cli, 'dist', 'index.js'), entry); + await assert.rejects( + captureFrames({ candidate: fixture.root, work: fixture.work }), + refuses(code), + label, + ); + } +}); + +/** Run `entry` as the finalized CLI under the preload, with only PATH and `env` set. */ +function preload(fixture, entry, env, args = []) { + writeFileSync(join(fixture.cli, 'dist', 'index.js'), entry); + return spawnSync( + process.execPath, + ['--import', pathToFileURL(PRELOAD).href, join(fixture.cli, 'dist', 'index.js'), ...args], + { encoding: 'utf8', timeout: 30_000, env: { PATH: process.env.PATH, ...env } }, + ); +} + +test('the preload refuses the keyring and native modules and leaves V8’s stack formatting alone', async () => { + const fixture = await candidate(); + const canary = 'frames-canary-preload'; + const document = join(fixture.work, `${canary}.relavium.yaml`); + writeFileSync(document, `id: ${canary}\nv: yes\n`); + const log = join(fixture.work, 'frames.json'); + const configured = { + RELAVIUM_FRAME_CAPTURE_LOG: log, + RELAVIUM_FRAME_CAPTURE_PACKAGE: join(fixture.cli, 'package.json'), + RELAVIUM_FRAME_CAPTURE_CANARY: canary, + }; + const read = () => JSON.parse(readFileSync(log, 'utf8')); + const clear = () => rmSync(log, { force: true }); + let run = preload( + fixture, + "import { readFileSync } from 'node:fs';\n" + + "import { Entry, findCredentials } from '@napi-rs/keyring';\n" + + "import { parse } from 'yaml';\n" + + 'const message = (action) => { try { action(); return "ran"; } catch (error) { return error.message; } };\n' + + "const keyring = [message(() => new Entry('relavium', 'probe')), message(() => findCredentials('relavium'))];\n" + + "const native = message(() => process.dlopen({ exports: {} }, '/nonexistent/addon.node'));\n" + + "parse(readFileSync(process.argv[2], 'utf8'));\n" + + // A formatter left installed would make `stack` an array of call sites, not a string. + "process.stdout.write(JSON.stringify({ keyring, native, limit: Error.stackTraceLimit, stack: typeof new Error('x').stack }));\n", + configured, + [document], + ); + assert.equal(run.status, 0, run.stderr); + assert.deepEqual(JSON.parse(run.stdout), { + keyring: [ + 'The symbolication run refuses the keyring', + 'The symbolication run refuses the keyring', + ], + native: 'The symbolication run refuses native modules', + limit: 10, + stack: 'string', + }); + assert.equal(read().captures.length, 1); + if (posixModes) assert.equal(statSync(log).mode & 0o777, 0o600); + // An explicit process.exit still writes the record. + clear(); + run = preload( + fixture, + "import { readFileSync } from 'node:fs'; import { parse } from 'yaml';\n" + + "parse(readFileSync(process.argv[2], 'utf8'));\nprocess.exit(0);\n", + configured, + [document], + ); + assert.equal(run.status, 0, run.stderr); + assert.equal(read().captures.length, 1); + // The first frame is the caller of parse, never the preload; parse options pass through. + clear(); + run = preload( + fixture, + "import { readFileSync } from 'node:fs'; import { parse } from 'yaml';\n" + + "const value = parse(readFileSync(process.argv[2], 'utf8'), { version: '1.1' });\n" + + 'process.stdout.write(JSON.stringify(value.v));\n', + configured, + [document], + ); + assert.equal(run.status, 0, run.stderr); + assert.equal(run.stdout, 'true'); + assert.equal( + read().captures[0][0].file, + pathToFileURL(join(fixture.cli, 'dist', 'index.js')).href, + ); + // An existing record is never overwritten. + writeFileSync(log, 'keep'); + preload(fixture, "import 'yaml';\n", configured); + assert.equal(readFileSync(log, 'utf8'), 'keep'); + // Asynchronous callers are marked async and a native frame has no file. + clear(); + run = preload( + fixture, + "import { readFileSync } from 'node:fs'; import { parse } from 'yaml';\n" + + 'async function inner(text) { await null; [text].map((t) => parse(t)); }\n' + + 'async function outer(text) { await inner(text); }\n' + + "await outer(readFileSync(process.argv[2], 'utf8'));\n", + configured, + [document], + ); + assert.equal(run.status, 0, run.stderr); + const frames = read().captures[0]; + assert.ok( + frames.some((frame) => frame.async === true), + 'an async frame', + ); + assert.ok( + frames.some((frame) => Object.hasOwn(frame, 'file') && frame.file === null), + 'a native frame', + ); + // An eval frame has no file name: it is recorded as null, never dropped. + clear(); + run = preload( + fixture, + "import { readFileSync } from 'node:fs'; import { parse } from 'yaml';\n" + + "const text = readFileSync(process.argv[2], 'utf8');\neval('parse(text)');\n", + configured, + [document], + ); + assert.equal(run.status, 0, run.stderr); + assert.ok(read().captures[0].every((frame) => Object.hasOwn(frame, 'file'))); + assert.ok(read().captures[0].some((frame) => frame.file === null)); + // A deep stack stops at the 64 frames the producer admits. + clear(); + run = preload( + fixture, + "import { readFileSync } from 'node:fs'; import { parse } from 'yaml';\n" + + "const text = readFileSync(process.argv[2], 'utf8');\n" + + 'function deep(n) { if (n === 0) return parse(text); return deep(n - 1); }\ndeep(100);\n', + configured, + [document], + ); + assert.equal(run.status, 0, run.stderr); + assert.equal(read().captures[0].length, 64); + // A platform keyring binding is refused by name. + clear(); + run = preload( + fixture, + "let result;\ntry { await import('@napi-rs/keyring-darwin-arm64'); result = 'loaded'; } catch (error) { result = error.message; }\nprocess.stdout.write(result);\n", + configured, + ); + assert.equal(run.stdout, 'Native keyring import refused'); + // Without a canary, a log or a package the preload refuses before the entry runs. + const marker = join(fixture.work, 'ran'); + for (const missing of Object.keys(configured)) { + const env = { ...configured }; + delete env[missing]; + clear(); + run = preload( + fixture, + `import { writeFileSync } from 'node:fs';\nwriteFileSync(${JSON.stringify(marker)}, 'x');\n`, + env, + ); + assert.notEqual(run.status, 0, missing); + assert.equal(existsSync(marker), false, missing); + assert.equal(existsSync(log), false, missing); + } +}); + +test('the preload refuses any module that resolves outside the candidate', async () => { + const fixture = await candidate(); + // A module another local user could plant in a `node_modules` above the candidate, as ws's optional + // `bufferutil` would find one in a shared temporary directory. + const name = `relavium-planted-${process.pid}`; + const planted = join(dirname(fixture.root), 'node_modules', name); + mkdirSync(planted, { recursive: true }); + const marker = join(fixture.work, 'planted-ran'); + writeFileSync(join(planted, 'package.json'), JSON.stringify({ name, main: 'index.js' })); + writeFileSync( + join(planted, 'index.js'), + `require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'x');\n`, + ); + const entry = + "import { createRequire } from 'node:module';\n" + + `let required; try { createRequire(import.meta.url)(${JSON.stringify(name)}); required = 'loaded'; } catch (error) { required = error.message; }\n` + + `let imported; try { await import(${JSON.stringify(name)}); imported = 'loaded'; } catch (error) { imported = error.message; }\n` + + 'process.stdout.write(JSON.stringify({ required, imported }));\n'; + // Without the preload, Node's resolution walks up and runs it. + writeFileSync(join(fixture.cli, 'dist', 'index.js'), entry); + let run = spawnSync(process.execPath, [join(fixture.cli, 'dist', 'index.js')], { + encoding: 'utf8', + timeout: 30_000, + env: { PATH: process.env.PATH }, + }); + assert.equal(run.status, 0, run.stderr); + assert.deepEqual(JSON.parse(run.stdout), { required: 'loaded', imported: 'loaded' }); + assert.equal(existsSync(marker), true); + rmSync(marker); + // Under the preload both a require and an import refuse, and it never runs. + const canary = 'frames-canary-confined'; + run = preload(fixture, entry, { + RELAVIUM_FRAME_CAPTURE_LOG: join(fixture.work, 'frames.json'), + RELAVIUM_FRAME_CAPTURE_PACKAGE: join(fixture.cli, 'package.json'), + RELAVIUM_FRAME_CAPTURE_CANARY: canary, + }); + assert.equal(run.status, 0, run.stderr); + const refusal = 'The symbolication run refuses a module outside the candidate'; + assert.deepEqual(JSON.parse(run.stdout), { required: refusal, imported: refusal }); + assert.equal(existsSync(marker), false); +}); + +test('the capture needs module.registerHooks, and the preload refuses without it', async () => { + assert.equal(captureSupported(), true); + assert.equal(captureSupported({}), false); + assert.equal(captureSupported({ registerHooks: () => {} }), true); + // On a Node without the API the preload's own guard runs: it refuses before the entry runs. + const fixture = await candidate(); + const shim = join(fixture.work, 'no-register-hooks.cjs'); + writeFileSync(shim, "delete require('node:module').registerHooks;\n"); + const marker = join(fixture.work, 'ran'); + writeFileSync( + join(fixture.cli, 'dist', 'index.js'), + `import { writeFileSync } from 'node:fs';\nwriteFileSync(${JSON.stringify(marker)}, 'x');\n`, + ); + const run = spawnSync( + process.execPath, + [ + '--require', + shim, + '--import', + pathToFileURL(PRELOAD).href, + join(fixture.cli, 'dist', 'index.js'), + ], + { + encoding: 'utf8', + timeout: 30_000, + env: { + PATH: process.env.PATH, + RELAVIUM_FRAME_CAPTURE_LOG: join(fixture.work, 'frames.json'), + RELAVIUM_FRAME_CAPTURE_PACKAGE: join(fixture.cli, 'package.json'), + RELAVIUM_FRAME_CAPTURE_CANARY: 'frames-canary-old-node', + }, + }, + ); + assert.notEqual(run.status, 0); + assert.match(run.stderr, /The symbolication preload needs module\.registerHooks/u); + assert.equal(existsSync(marker), false); +}); + +test('the keyring stand-in has every export the real package declares, and each refuses', () => { + // Read from the package's own declarations and entry, so the test loads no native binding. + const root = dirname(cliRequire.resolve('@napi-rs/keyring/package.json')); + const names = new Set([ + ...[ + ...readFileSync(join(root, 'index.d.ts'), 'utf8').matchAll( + /^export declare (?:class|function) (\w+)/gmu, + ), + ].map((match) => match[1]), + ...[ + ...readFileSync(join(root, 'index.js'), 'utf8').matchAll(/^module\.exports\.(\w+) =/gmu), + ].map((match) => match[1]), + ]); + assert.ok(names.size >= 4, 'the package declares its exports'); + for (const name of names) assert.equal(Object.hasOwn(keyringStub, name), true, name); + for (const [name, value] of Object.entries(keyringStub)) { + assert.throws( + () => (/^[A-Z]/u.test(name) ? new value() : value()), + /^Error: The symbolication run refuses the keyring$/u, + name, + ); + } +}); diff --git a/tools/cli-package/ustar-fixture.mjs b/tools/cli-package/ustar-fixture.mjs new file mode 100644 index 00000000..b72277e5 --- /dev/null +++ b/tools/cli-package/ustar-fixture.mjs @@ -0,0 +1,20 @@ +/** + * Test-only USTAR writer: the exact archive shape the inspector accepts (pnpm's layout), so a test + * can build a package tarball from an in-memory inventory. `package/dist/index.js` is 0755, every + * other entry 0644. The dialect itself is `ustar.mjs`'s. + */ +import { ustarEntry, writeUstar } from './ustar.mjs'; + +const packageMode = (path) => (path === 'package/dist/index.js' ? 0o755 : 0o644); + +export { writeChecksum as checksum } from './ustar.mjs'; + +/** @param {string} path @param {Uint8Array} bytes */ +export function entry(path, bytes) { + return ustarEntry(path, bytes, packageMode(path)); +} + +/** @param {Map} files */ +export function ustar(files) { + return writeUstar(files, packageMode); +} diff --git a/tools/cli-package/ustar.mjs b/tools/cli-package/ustar.mjs new file mode 100644 index 00000000..56bfdac9 --- /dev/null +++ b/tools/cli-package/ustar.mjs @@ -0,0 +1,164 @@ +/** + * The one strict USTAR dialect the packaging tools read and write (Step 9b.1, the 2026-10-19 C1 ruling): + * regular files only, no linkname, owner strings, prefix, extension records or reserved bytes, zero + * owner ids and device numbers, an exact two-block end and zero padding. The caller decides which + * paths and modes it admits and how a refusal is reported. There is no extraction here and no fallback + * to a more permissive platform tar. + */ +import { Buffer } from 'node:buffer'; +import { TextDecoder } from 'node:util'; + +const BLOCK = 512; +const MAGIC = Buffer.from('ustar\0' + '00'); + +/** @param {Buffer} header */ +function checksumOf(header) { + let sum = 0; + for (let index = 0; index < BLOCK; index += 1) { + sum += index >= 148 && index < 156 ? 32 : (header[index] ?? 0); + } + return sum; +} + +/** @param {Buffer} header */ +export function writeChecksum(header) { + header.fill(32, 148, 156); + header.write(checksumOf(header).toString(8).padStart(6, '0'), 148); + header[154] = 0; + header[155] = 32; +} + +/** + * One regular-file entry: header, bytes and zero padding. + * @param {string} path @param {Uint8Array} bytes @param {number} mode + */ +export function ustarEntry(path, bytes, mode) { + const header = Buffer.alloc(BLOCK); + header.write(path, 0, 100); + for (const offset of [100, 108, 116, 329, 337]) header.write('000000 \0', offset); + header.write(mode.toString(8).padStart(6, '0'), 100); + header.write(bytes.length.toString(8).padStart(11, '0') + ' ', 124); + header.write('00000000000 ', 136); + header[156] = 48; + MAGIC.copy(header, 257); + writeChecksum(header); + return Buffer.concat([header, bytes, Buffer.alloc((BLOCK - (bytes.length % BLOCK)) % BLOCK)]); +} + +/** + * A whole archive in the given entry order. + * @param {Iterable<[string, Uint8Array]>} entries + * @param {(path: string) => number} modeFor + */ +export function writeUstar(entries, modeFor) { + return Buffer.concat([ + ...[...entries].map(([path, bytes]) => ustarEntry(path, bytes, modeFor(path))), + Buffer.alloc(2 * BLOCK), + ]); +} + +/** + * @typedef {{ + * admitPath: (path: string) => void, + * modeFor: (path: string) => number, + * maxFiles: number, + * maxFileBytes: number, + * refuse: (code: string, cause?: unknown) => never, + * admitEntry?: (path: string) => void, + * onFile: (file: {path: string, mode: number, size: number, contents: Buffer}) => void, + * }} UstarReader + */ + +/** + * Read every entry of an uncompressed archive in order. `admitEntry` runs after the header, duplicate + * and case-alias checks and before the entry's bytes are bounded; `onFile` runs once its bytes and + * padding are checked. Returns the entry count. + * @param {Buffer} tar + * @param {UstarReader} reader + */ +export function readUstar(tar, reader) { + const { refuse } = reader; + /** @param {Buffer} field */ + const octal = (field) => { + const text = field.toString('latin1'); + if (!/^[0-7]+[ \0]*$/u.test(text)) refuse('TAR_NUMBER'); + const value = Number.parseInt(text, 8); + if (!Number.isSafeInteger(value)) refuse('TAR_NUMBER'); + return value; + }; + /** @param {Buffer} field */ + const nameField = (field) => { + const end = field.indexOf(0); + if (end < 1 || field.subarray(end).some((byte) => byte !== 0)) refuse('TAR_NAME'); + try { + return new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode( + field.subarray(0, end), + ); + } catch (cause) { + return refuse('UTF8', cause); + } + }; + /** @param {Buffer} header */ + const readHeader = (header) => { + if (octal(header.subarray(148, 156)) !== checksumOf(header)) refuse('TAR_CHECKSUM'); + if (!header.subarray(257, 265).equals(MAGIC)) refuse('TAR_FORMAT'); + if (header[156] !== 48) refuse('TAR_TYPE'); + for (const [start, end] of [ + [157, 257], + [265, 329], + [345, 512], + ]) { + if (header.subarray(start, end).some((byte) => byte !== 0)) refuse('TAR_METADATA'); + } + for (const [start, end] of [ + [108, 116], + [116, 124], + [329, 337], + [337, 345], + ]) { + if (octal(header.subarray(start, end)) !== 0) refuse('TAR_METADATA'); + } + octal(header.subarray(136, 148)); // Validate mtime syntax without requiring a clock value. + const path = nameField(header.subarray(0, 100)); + reader.admitPath(path); + const mode = octal(header.subarray(100, 108)); + if (mode !== reader.modeFor(path)) refuse('TAR_MODE'); + const size = octal(header.subarray(124, 136)); + if (size > reader.maxFileBytes) refuse('FILE_SIZE'); + return { path, mode, size }; + }; + + const seen = new Set(); + const portableNames = new Set(); + let count = 0; + let offset = 0; + let ended = false; + while (offset < tar.length) { + if (tar.length - offset < BLOCK) refuse('TAR_TRUNCATED'); + const header = tar.subarray(offset, offset + BLOCK); + if (header.every((byte) => byte === 0)) { + if (tar.length - offset !== 2 * BLOCK || tar.subarray(offset).some((byte) => byte !== 0)) { + refuse('TAR_END'); + } + ended = true; + break; + } + if (count >= reader.maxFiles) refuse('FILE_COUNT'); + const { path, mode, size } = readHeader(header); + if (seen.has(path)) refuse('DUPLICATE_FILE'); + seen.add(path); + if (portableNames.has(path.toLowerCase())) refuse('PATH_ALIAS'); + portableNames.add(path.toLowerCase()); + reader.admitEntry?.(path); + const start = offset + BLOCK; + const end = start + size; + const next = start + Math.ceil(size / BLOCK) * BLOCK; + if (next > tar.length) refuse('TAR_TRUNCATED'); + if (tar.subarray(end, next).some((byte) => byte !== 0)) refuse('TAR_PADDING'); + reader.onFile({ path, mode, size, contents: tar.subarray(start, end) }); + count += 1; + offset = next; + } + if (!ended) refuse('TAR_END'); + return count; +} diff --git a/tools/cli-package/ustar.test.mjs b/tools/cli-package/ustar.test.mjs new file mode 100644 index 00000000..df1c586d --- /dev/null +++ b/tools/cli-package/ustar.test.mjs @@ -0,0 +1,163 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { Buffer } from 'node:buffer'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { after, test } from 'node:test'; +import { readUstar, ustarEntry, writeChecksum, writeUstar } from './ustar.mjs'; + +const scratch = mkdtempSync(join(tmpdir(), 'relavium-ustar-')); +after(() => rmSync(scratch, { recursive: true, force: true })); + +class Refused extends Error { + constructor(code) { + super(code); + this.code = code; + } +} +const refuses = (code) => (error) => error instanceof Refused && error.code === code; + +/** A private, owner-only dialect like the custody container's. */ +function read(tar, overrides = {}) { + const files = []; + const order = []; + readUstar(tar, { + admitPath: (path) => { + if (!/^(?:[A-Za-z-]+\.json|maps\/[a-z]+\.js\.map)$/u.test(path)) throw new Refused('PATH'); + }, + modeFor: () => 0o600, + maxFiles: 4, + maxFileBytes: 4096, + refuse: (code) => { + throw new Refused(code); + }, + admitEntry: (path) => order.push(`admit ${path}`), + onFile: ({ path, mode, contents }) => { + order.push(`file ${path}`); + files.push([path, mode, Buffer.from(contents)]); + }, + ...overrides, + }); + return { files, order }; +} + +/** Unaligned, empty, exactly one block, and one byte over a block: every padding case. */ +const ENTRIES = [ + ['manifest.json', Buffer.from('{"synthetic":true}\n')], + ['empty.json', Buffer.alloc(0)], + ['maps/index.js.map', Buffer.alloc(512, 0x62)], + ['metafile-esm.json', Buffer.alloc(513, 0x61)], +]; + +test('a written archive reads back entry by entry, in order, with its mode', () => { + const { files, order } = read(writeUstar(ENTRIES, () => 0o600)); + assert.deepEqual( + files, + ENTRIES.map(([path, bytes]) => [path, 0o600, bytes]), + ); + assert.deepEqual( + order, + ENTRIES.flatMap(([path]) => [`admit ${path}`, `file ${path}`]), + ); +}); + +test('the platform tar extracts the same bytes, nested directories included', () => { + const archive = join(scratch, 'container.tar'); + const out = join(scratch, 'extracted'); + mkdirSync(out); + writeFileSync( + archive, + writeUstar(ENTRIES, () => 0o600), + ); + const run = spawnSync('tar', ['-xf', archive, '-C', out], { encoding: 'utf8' }); + assert.equal(run.status, 0, 'the platform tar must extract the archive'); + for (const [path, bytes] of ENTRIES) { + assert.deepEqual(readFileSync(join(out, ...path.split('/'))), bytes); + } +}); + +/** One entry archive whose header the mutation edits, with a repaired checksum unless told not to. */ +function mutated(mutate, { repair = true } = {}) { + const block = ustarEntry('manifest.json', Buffer.from('{}'), 0o600); + mutate(block); + if (repair) writeChecksum(block.subarray(0, 512)); + return Buffer.concat([block, Buffer.alloc(1024)]); +} + +test('each header field outside the dialect refuses with its own code', () => { + for (const [code, archive] of [ + ['TAR_CHECKSUM', mutated((block) => (block[0] = 0x4d), { repair: false })], + ['TAR_FORMAT', mutated((block) => block.write('gnutar', 257))], + ['TAR_TYPE', mutated((block) => (block[156] = 0x35))], + ['TAR_METADATA', mutated((block) => (block[157] = 0x61))], + ['TAR_METADATA', mutated((block) => block.write('000001 \0', 108))], + ['TAR_METADATA', mutated((block) => block.write('000001 \0', 116))], + ['TAR_METADATA', mutated((block) => block.write('000001 \0', 329))], + ['TAR_METADATA', mutated((block) => block.write('000001 \0', 337))], + ['TAR_NUMBER', mutated((block) => block.write('0000000000x ', 136))], + ['TAR_FORMAT', mutated((block) => block.write('xx', 263))], + ['TAR_NUMBER', mutated((block) => block.write('00000000009 ', 124))], + ['TAR_NAME', mutated((block) => (block[20] = 0x61))], + ['TAR_NAME', mutated((block) => block.fill(0, 0, 100))], + ['UTF8', mutated((block) => block.fill(0xff, 0, 4))], + ['TAR_MODE', mutated((block) => block.write('000644', 100))], + ['FILE_SIZE', mutated((block) => block.write('00000010001 ', 124))], + ]) { + assert.throws(() => read(archive), refuses(code), code); + } +}); + +test('the archive structure refuses duplicates, aliases, truncation, padding, count and end', () => { + const one = ustarEntry('manifest.json', Buffer.from('{}'), 0o600); + const end = Buffer.alloc(1024); + const padded = Buffer.from(one); + padded[512 + 2] = 1; + for (const [code, archive] of [ + ['DUPLICATE_FILE', Buffer.concat([one, one, end])], + [ + 'PATH_ALIAS', + Buffer.concat([one, ustarEntry('MANIFEST.json', Buffer.from('{}'), 0o600), end]), + ], + ['TAR_TRUNCATED', one.subarray(0, 600)], + ['TAR_TRUNCATED', Buffer.concat([one.subarray(0, 512), Buffer.alloc(100)])], + ['TAR_PADDING', Buffer.concat([padded, end])], + ['TAR_END', one], + ['TAR_END', Buffer.concat([one, end, Buffer.alloc(512)])], + ['TAR_END', Buffer.concat([one, Buffer.alloc(1023), Buffer.from([1])])], + [ + 'FILE_COUNT', + writeUstar( + ['a', 'b', 'c', 'd', 'e'].map((name) => [`${name}.json`, Buffer.from('{}')]), + () => 0o600, + ), + ], + ]) { + assert.throws(() => read(archive), refuses(code), code); + } + // A one-character name is a name: the caller's path rule judges it. + assert.throws( + () => read(Buffer.concat([ustarEntry('a', Buffer.from('{}'), 0o600), end])), + refuses('PATH'), + ); + // The caller's path rule runs inside the header check, before its mode and size are judged. + assert.throws( + () => read(Buffer.concat([ustarEntry('outside.txt', Buffer.alloc(8192), 0o644), end])), + refuses('PATH'), + ); +}); + +test("the caller's entry hook runs before the entry's bytes are bounded", () => { + const order = []; + assert.throws( + () => + read(writeUstar(ENTRIES, () => 0o600).subarray(0, 600), { + admitEntry: (path) => { + order.push(path); + throw new Refused('EXTRA'); + }, + }), + refuses('EXTRA'), + ); + assert.deepEqual(order, ['manifest.json']); +}); diff --git a/tools/fs-policy-controls/README.md b/tools/fs-policy-controls/README.md index b9a9f965..e46f6cc6 100644 --- a/tools/fs-policy-controls/README.md +++ b/tools/fs-policy-controls/README.md @@ -176,11 +176,35 @@ retain their original source attribution. The generated-result correction adds one object-preflight control: removing both encoded/exact decoded guards must break the capacity +1 and +3 pre-reservation witnesses in the complete 18-case generated-result suite. Request-input and direct-store ceilings remain separate. -The current declaration in `mutations.mjs` contains 110 controls /330 phases, +The declaration before NI contained 110 controls /330 phases, projecting 1,987 retained files (110 × 18 + 7) under the unchanged 2,000-file /64-MiB ceilings. This is capacity arithmetic, not a measured export or completed capture; historical captures retain their own source heads and actual counts. +That arithmetic counted files only. Every phase also retains its exact source manifest, +one per phase and about 200 KB each because it lists the whole source graph, and the +export's raw manifests share the 64-MiB bound. Once Wave 3's PR95 increment grew that +graph, a manifest measured 203,421 bytes and the 330 manifests of one export +67,128,930 bytes, 20,066 over the bound, so CI's export refused (2026-10-22). Like the +secrets family, the 110-control declaration then ran as two round-robin shards of 55 controls +(`--shard 1/2` and `--shard 2/2`), each exporting its own evidence: 165 manifests +(33,564,465 bytes) and 997 retained files (55 × 18 + 7). Both shards ran locally with +`RELAVIUM_ACCEPTANCE_EVIDENCE` set at `2a1f7359`'s source; every control passed, and +the exports measured 25.65 MiB and 25.59 MiB. That is one local measurement, not a CI +capture. + +NI (2026-10-25) appends five precision controls: `fs-root-identity-precision`, +`catalog-identity-precision`, `cas-root-pin-precision`, `cas-owned-identity-precision` +and `media-temp-same-precision`. Each removal returns one site's whole native identity +comparison to `Number`. Its complete witness file (four CLI cases or seven media-store +cases) substitutes every inode the code reads, keyed by its real device and inode, with +2^60 + k, which stays distinct as a bigint but collapses to one Number, so a real replacement must refuse with the named +assertion. The declaration is now 115 controls /345 phases. Its round-robin shards hold +58 and 57 controls, projecting 174 and 171 phase manifests and 1,051 and 1,033 retained +files under the unchanged ceilings. One new control's export at `209e7193` measured a phase +manifest of 203,728 bytes, so the larger shard's manifests come to 35,448,672 bytes (52.8 % +of the bound); that is one local measurement, not a CI capture. + The declaration is not acceptance evidence by itself. The complete matrix still requires a stable committed head, all fixed/removal/restored phases, named semantic witnesses, child source/dependency receipts, byte-exact restoration, diff --git a/tools/fs-policy-controls/mutations.mjs b/tools/fs-policy-controls/mutations.mjs index 7210b8d4..55f9be89 100644 --- a/tools/fs-policy-controls/mutations.mjs +++ b/tools/fs-policy-controls/mutations.mjs @@ -186,8 +186,8 @@ mutations.push( control( 'CAS_COOPERATIVE_READ', paths, - ' assertReadableFile(path: string, opened: Stats): void {\n const current = this.fileSnapshot(path);', - ' assertReadableFile(path: string, opened: Stats): void {\n this.assertOwnedFile(path, opened);\n const current = this.fileSnapshot(path);', + ' assertReadableFile(path: string, opened: ExactStats): void {\n const current = this.fileSnapshot(path);', + ' assertReadableFile(path: string, opened: ExactStats): void {\n this.assertOwnedFile(path, opened);\n const current = this.fileSnapshot(path);', 'FS_CAUSAL_CAS_COOPERATIVE_READ', [cooperativeCases], 'preserves native', @@ -1434,3 +1434,61 @@ mutations.push({ }, ], }); + +// NI: the product's file and directory identities compare exact bigints. Each removal returns one site's whole +// comparison to Number and must break a real replacement whose substituted identities (2^60 + k) +// differ only beyond a Number's precision, as a 64-bit Windows file ID can. +const nativeIdentityCli = 'apps/cli/src/engine/tool-host/native-identity.test.ts'; +const nativeIdentityMedia = 'packages/db/src/media-store-native-identity.test.ts'; +mutations.push( + mediaControl( + 'FS_ROOT_IDENTITY_PRECISION', + policy, + ' current.dev !== pin.dev ||\n current.ino !== pin.ino ||\n current.lexicalDev !== pin.lexicalDev ||\n current.lexicalIno !== pin.lexicalIno ||', + ' Number(current.dev) !== Number(pin.dev) ||\n Number(current.ino) !== Number(pin.ino) ||\n Number(current.lexicalDev) !== Number(pin.lexicalDev) ||\n Number(current.lexicalIno) !== Number(pin.lexicalIno) ||', + 'FS_ROOT_IDENTITY_PRECISION', + nativeIdentityCli, + 'refuses a replaced selected directory whose identity collides as a Number', + 4, + ), + mediaControl( + 'CATALOG_IDENTITY_PRECISION', + 'apps/cli/src/workflows/catalog.ts', + 'opened.dev !== stats.dev || opened.ino !== stats.ino', + 'Number(opened.dev) !== Number(stats.dev) || Number(opened.ino) !== Number(stats.ino)', + 'CATALOG_IDENTITY_PRECISION', + nativeIdentityCli, + 'refuses a catalog file swapped between its admission and its open', + 4, + ), + mediaControl( + 'CAS_ROOT_PIN_PRECISION', + paths, + ' pin.dev !== info.dev ||\n pin.ino !== info.ino ||\n pin.entryDev !== entry.dev ||\n pin.entryIno !== entry.ino ||', + ' Number(pin.dev) !== Number(info.dev) ||\n Number(pin.ino) !== Number(info.ino) ||\n Number(pin.entryDev) !== Number(entry.dev) ||\n Number(pin.entryIno) !== Number(entry.ino) ||', + 'CAS_ROOT_PIN_PRECISION', + nativeIdentityMedia, + 'refuses a replaced root directory whose identity collides as a Number', + 7, + ), + mediaControl( + 'CAS_OWNED_IDENTITY_PRECISION', + paths, + ' current.dev !== opened.dev ||\n current.ino !== opened.ino', + ' Number(current.dev) !== Number(opened.dev) ||\n Number(current.ino) !== Number(opened.ino)', + 'CAS_OWNED_IDENTITY_PRECISION', + nativeIdentityMedia, + 'refuses to publish a producer temporary replaced after its exclusive open', + 7, + ), + mediaControl( + 'MEDIA_TEMP_SAME_PRECISION', + mediaTemp, + 'function same(a: ExactStats, b: ExactStats): boolean {\n return a.dev === b.dev && a.ino === b.ino;\n}', + 'function same(a: ExactStats, b: ExactStats): boolean {\n return Number(a.dev) === Number(b.dev) && Number(a.ino) === Number(b.ino);\n}', + 'MEDIA_TEMP_SAME_PRECISION', + nativeIdentityMedia, + 'refuses to claim into a replaced claim directory', + 7, + ), +); diff --git a/tools/overflow-capture/capture.mjs b/tools/overflow-capture/capture.mjs index f5ed5f58..43b9c246 100644 --- a/tools/overflow-capture/capture.mjs +++ b/tools/overflow-capture/capture.mjs @@ -58,7 +58,8 @@ async function main() { process.on('SIGINT', abort); process.on('SIGTERM', abort); try { - identity = fstatSync(fd); + // Exact identities: a 64-bit file ID (NTFS, overlayfs xino) exceeds a Number's precision. + identity = fstatSync(fd, { bigint: true }); const artifact = await captureResponse(options, key, { fetch, newAbortController: () => new globalThis.AbortController(), @@ -70,7 +71,7 @@ async function main() { signal: caller.signal, }); const stillOwnsDestination = () => { - const current = lstatSync(options.out, { throwIfNoEntry: false }); + const current = lstatSync(options.out, { bigint: true, throwIfNoEntry: false }); return current?.dev === identity.dev && current.ino === identity.ino; }; if (!stillOwnsDestination()) throw new CaptureError('destination_changed'); diff --git a/tools/overflow-capture/check.mjs b/tools/overflow-capture/check.mjs index 56d8357c..239f34ba 100644 --- a/tools/overflow-capture/check.mjs +++ b/tools/overflow-capture/check.mjs @@ -44,9 +44,26 @@ import { syncBuiltinESMExports } from 'node:module'; // Delay-only regression: return the real lstat result after the parent swaps the pathname. The fixed // runner has no failure-path lstat/unlink, so it closes before this scheduling point can be reached. const rawLstat = fs.lstatSync; +const rawFstat = fs.fstatSync; +// Exact identity probe: every native identity becomes 2^60 + k, distinct as bigints but one Number. +const fakes = new Map(); +const collision = process.env.CAPTURE_PROBE_MODE === 'replacement-collision'; +function substitute(info, exact) { + if (!collision || info === undefined || exact === undefined) return info; + const key = exact.dev + ':' + exact.ino; + const value = fakes.get(key) ?? (1n << 60n) + BigInt(fakes.size); + fakes.set(key, value); + Object.defineProperty(info, 'ino', { value: typeof info.ino === 'bigint' ? value : Number(value) }); + return info; +} +if (collision) + process.on('exit', () => writeFileSync(process.env.CAPTURE_PROBE_MARKER + '.identities', JSON.stringify([...fakes.values()].map(String)))); +fs.fstatSync = function(fd, ...args) { + return substitute(rawFstat(fd, ...args), rawFstat(fd, { bigint: true })); +}; let scheduled = false; fs.lstatSync = function(path, ...args) { - const result = rawLstat(path, ...args); + const result = substitute(rawLstat(path, ...args), rawLstat(path, { bigint: true, throwIfNoEntry: false })); if (process.env.CAPTURE_PROBE_MODE === 'cleanup-race' && !scheduled && path === process.env.CAPTURE_PROBE_OUT) { scheduled = true; writeFileSync(process.env.CAPTURE_PROBE_RACE_READY, '1'); @@ -69,10 +86,11 @@ globalThis.fetch = async (url, init) => { queueMicrotask(() => process.emit('SIGTERM')); return new Promise(() => {}); case 'replacement-success': + case 'replacement-collision': case 'replacement': unlinkSync(process.env.CAPTURE_PROBE_OUT); writeFileSync(process.env.CAPTURE_PROBE_OUT, 'replacement user file'); - if (process.env.CAPTURE_PROBE_MODE === 'replacement-success') return new Response('{}', {headers:{'content-type':'application/json'}}); + if (process.env.CAPTURE_PROBE_MODE !== 'replacement') return new Response('{}', {headers:{'content-type':'application/json'}}); throw new Error('refused after replacement'); case 'cleanup-race': case 'transport': throw new Error(${JSON.stringify(key)}); @@ -175,6 +193,17 @@ try { assert.equal(replacedSuccess.code, 1, 'success reported a response at an unowned destination'); assert.ok(!replacedSuccess.stdout.includes('capture saved')); assert.equal(readFileSync(out, 'utf8'), 'replacement user file'); + rmSync(out); + // The replacement differs from the reserved file only beyond a Number's precision (NI). + const collided = await run(args, 'replacement-collision'); + const identities = JSON.parse(readFileSync(`${marker}.identities`, 'utf8')).map(BigInt); + assert.ok( + identities.length >= 2 && identities.every((value) => Number(value) === Number(identities[0])), + 'the identity fixture must collide as Numbers only', + ); + assert.equal(collided.code, 1, 'success reported a response at a Number-colliding replacement'); + assert.ok(!collided.stdout.includes('capture saved')); + assert.equal(readFileSync(out, 'utf8'), 'replacement user file'); console.log( '✓ overflow capture command passed offline stdin, file, secrecy and cancellation checks.', ); diff --git a/tools/secrets-controls/README.md b/tools/secrets-controls/README.md index bce9595c..14f8440a 100644 --- a/tools/secrets-controls/README.md +++ b/tools/secrets-controls/README.md @@ -4,7 +4,7 @@ Step 4 first-run input matrix, Step 5 snapshot/export, Step 7 provider-header and PR #95 B2 redaction controls on physical source copies. `--mechanism ` supports diagnosis only; selected runs retain `complete: false`. The current manifest -declares **57 independent controls and 336 required named assertion witnesses**. +declares **130 independent controls and 513 required named assertion witnesses**. These counts describe the workload; a declaration is not evidence that its removal passed. Actual fixed/removal/restored results belong to the coordinator's summary. @@ -41,14 +41,15 @@ secret suffix after an overlapping query value. The normalized-secret-probe cont removes the canonical `stripTerminalControls` view from the shared non-query probe; 29 named normalized-key overlap witnesses require that view in LLM messages/codes, both CLI sanitizer functions, all four renderer modes and a verbose stack. The -non-query-before-query control reverses the final composition and requires 58 named -Bearer/Basic overlap witnesses across those same surfaces. These controls require -each finite non-query rule to inspect its own original diagnostic views before any -other replacement erases its evidence, followed by the finite query replacement. +non-query-before-query control applies the literal query names before the merged +pass and requires 58 named Bearer/Basic overlap witnesses across those same surfaces. +These controls require each finite non-query rule to inspect its own original +diagnostic views before any other replacement erases its evidence; the literal query +values then join the same merged masking pass instead of following it. The views include the canonical terminal-normalized text and a whitespace-joined residual view. The split literal query name remains an independent CLI obligation. Across the thirteen Step 3 controls, 191 named assertion witnesses are required. The shared -overlap controls collect 277 tests across the LLM, CLI sanitizer and renderer files. +overlap controls collect 278 tests across the LLM, CLI sanitizer and renderer files. The B2 controls below pin the conservative LF/TAB continuation rule when a first fragment already meets a key-length floor. Broader whitespace residual checks retain their finite scope; neither mechanism claims universal secret detection. @@ -128,7 +129,7 @@ permanent negative cases; no notice is produced before a successful export. Four snapshot controls remove the finite scrubber detection site and the actual `create`, `import`, and `run` warning calls. Each has a named value-free assertion in -its complete 10-case detector, 11-case create, 11-case import or 84-case run file. +its complete 11-case detector, 11-case create, 11-case import or 84-case run file. The adjacent tests check preserved authored files and admitted run snapshots, JSON stdout, and scan-budget/incomplete notices. These controls certify warning delivery for the planted supported shapes at these entry points, not universal secret @@ -177,9 +178,9 @@ modern `sk-` body characters, joined-span application, the original-text left-bo check, split scheme labels, split API-header labels and raw `?`/`#` password recognition independently. Four LLM controls remove the before-masking LF/TAB probe, its original left-boundary check, split API-header recognition and raw-password recognition. -Every core phase executes the complete 142-case bounding file; every new LLM phase -executes the complete 89-case scrubber file. Independent ordinary tooling tests pin -these inventories, the 277-case overlap cohort, 179-case CLI diagnostic cohort and +Every core projection phase executes the complete 234-case bounding file (the journal-identity +control's phases the complete 116-case registry file); every new LLM phase executes the complete 224-case scrubber file. Independent ordinary tooling tests pin +these inventories, the 278-case overlap cohort, 180-case CLI diagnostic cohort and 18-case command-export cohort. Actual collection still has to match in every phase. The boundary controls preserve an embedded public-word key-shaped run containing an @@ -189,14 +190,15 @@ would survive its removal. Other witnesses require complete recognized spans and retain public tails, host contacts and the full-value identity policy. The finite LF/TAB continuation and malformed-userinfo policy is documented in the canonical [effect-journal contract](../../docs/reference/shared-core/effect-journal.md#11-secrets-what-a-row-may-hold). -An eight-MiB permanent fixture checks completion and whole-span behavior without a -machine-speed threshold; these controls do not certify arbitrary obfuscation. +Eight-MiB permanent fixtures in the core and scrubber files check completion and whole-span +behavior under a 15-second case deadline rather than a machine-speed threshold; these controls do +not certify arbitrary obfuscation. Two R1 original-boundary recovery controls add eight named assertions, retaining all 51 prior identities and required witnesses. The core removal admits embedded-word starts while retaining the mapper's late refusal and bounded probe; the rejected candidate consumes the later valid start. The LLM removal replaces original-boundary checks with joined-boundary checks only -for the non-header rules in its unified scanner, preserving the finite-header path. All four +for the key (`sk-`/`AIza`) rules in its unified scanner (the URL and header rules supply their own joined patterns), preserving the finite-header path. All four OpenAI/Google × LF/TAB poisoned-prefix cases must then fail with `SECRETS_B2_ORIGINAL_BOUNDARY`; public embedded-word preservation and the later complete split credential are @@ -213,7 +215,9 @@ all 53 prior control identities, paths, targets, markers and witness arrays. Fou `SECRETS_B2_HEADER_ORIGINAL_BOUNDARY`. The core removal admits an embedded outer label but retains the mapper's late boundary refusal, reproducing consumption of the valid inner label. The LLM removal substitutes joined-boundary checks only for -its finite-header rule; other rules still use original boundaries. Neither removal +its finite-header rule; other rules still use original boundaries. (The URL rule is selected +too, because it supplies its own joined pattern, but its boundary blocks no start, so the +substitution cannot change its result.) Neither removal disables bounded JWT probing or replaces canonical matcher authority. The earlier split-header-label control now removes the header helper's actual mask delivery. Core and LLM stress cases check bounded preparation with length/Boolean assertions, @@ -240,7 +244,7 @@ The independent `diagnostic-original-control-boundary` removal strips terminal controls before the initial shared scrubber can observe their original credential boundary. Eight multiline/inline sanitizer cases cover removed C0, DEL, C1 and bidi units; the same four real renderer cases must fail their distinct -`SECRETS_B2_DISPLAY_BOUNDARY` assertion. Both phases execute the complete 179-case +`SECRETS_B2_DISPLAY_BOUNDARY` assertion. Both phases execute the complete 180-case sanitizer/renderer cohort. The earlier CLI residual-query control remains independent and retains its required witnesses. Its removal now skips the actual remaining-span loop: the new outer raw scrubber can already mask ordinary matches, @@ -256,18 +260,302 @@ The LLM original-boundary removal now removes both the numeric coarse previous-word prefilter and the rule's own boundary predicate; retaining either would leave the tested policy active. Header/key recovery removals still change their original-rule boundary only, and all removals retain the same canonical -matchers and bounded raw/joined standalone visitors. Current collection is 25 -complete files / 1,320 cases, with each control's exact workload and named target +matchers and bounded raw/joined standalone visitors. Current collection is 26 +complete files / 1,651 cases, with each control's exact workload and named target checked separately. These are updated declarations and author checks, not new physical causal captures or a clean review round. R3 is preserved as NONCLEAN; later corrective acceptance belongs to the owning phase register. Prior selected `7e9096a8` captures retain their older source/workload attribution. -The current 57-control / 171-phase full declaration projects 1,033 retained files -(57 × 18 + 7) under the unchanged 2,000-file / 64-MiB export bounds. This is a capacity -calculation, not a measured full capture. Historical 41-control captures retain their -original source and cohort attribution; selected B2 or original-boundary recovery captures are not full-family -acceptance. +The current 130-control / 390-phase full declaration runs as two round-robin shards of 65 controls +(`--shard 1/2` and `--shard 2/2`), each exporting its own evidence: each projects 1,177 retained +files (65 × 18 + 7) of the unchanged 2,000. Locally both exports held exactly that many and measured +50.24 MiB and 50.14 MiB, final export record included, against the unchanged +64-MiB bound. That is one local measurement at the round-10 source, not a CI capture; one export of +the whole family would exceed the byte bound. Historical 41-control captures retain their original +source and cohort attribution; selected B2 or original-boundary recovery captures are not +full-family acceptance. + +Wave 3's PR95 increment adds nine controls and 80 named assertions, retaining all 57 +prior identities and witnesses. The LLM header control restores the old header value +class, so a value that begins with a `Bearer`/`Basic` label leaks its token again. Two +LLM and two core controls each drop one half of the raw `?`/`#` contact predicate (the +host-shaped prefix, or the `=` assignment that must follow it). The connection-scheme +control restores the original six-scheme list (`sftp` is no witness: the old list still +masks it by its `ftp` suffix). Three controls cut the URL rule's split-scanner support: +its joined form, its no-boundary start (a glued scheme) and its `://` window (`ws` alone +fits a two-character window, so it is no witness). Each of the nine ran in diagnosis mode +with its fixed and restored phases green and every declared witness failing on its marker; +that is author evidence, not a full-family capture. The eight-MiB scrubber cases now carry a 15-second +case deadline instead of a wall-clock assertion, inside the phases' `CAUSAL_TIMEOUT_BUDGET`. + +The increment's first review round added six more controls and 14 named assertions. Two +header controls narrow the label prefix back to one label, or its whitespace back to one +character, so a doubled `Bearer Bearer` label or a multi-character gap leaks the token. Two +controls per masker remove the host-shaped prefix's length bound (an admitted-size dotted +prefix then exhausts V8's RegExp stack, and its witness catches the throw) or its `^`/`$` +anchors (look-alike prefixes such as `john.doe:s3cret` turn public). The snapshot detector's +complete workload is now 11 cases. + +The second review round added seven controls and 11 named assertions: an unbounded header label +run, and the label-as-token and masked-marker guards of the header and standalone rules. The +host-length controls also witness the exact 260/261 boundary. + +The third round replaced sequential rule application with one merged masking pass: every rule +finds its matches in the original text, the masked ranges are merged, and a token rule masks only +its token, so masking an exact `[REDACTED]` is the identity and the three masked-marker guards are +gone. The header and label block now has 23 controls, 13 more than the ten it replaced. Seven +re-anchor the earlier label-run, label-whitespace, label-bound and not-a-label controls; the new +ones restore the sequential pass, disable range merging or mask a whole header match; unbound the +concatenated-label lookaheads (an admitted-size glued run then exhausts the RegExp stack); drop +the header's not-a-nested-header guard, nested-header stop or zero-width boundary; weaken its +escaped separator, quote-run prefix, first-character or escaped-stop rules; drop the standalone +rules' closing-quote or glued-label guard or their embedded-label stop; or reintroduce round 2's +masked-marker guard, which exposes a tail glued to the marker. + +The fourth round merged the two standalone label rules into one, whose first label is either +label, so neither copy can drift from the other: the former Bearer and Basic not-a-label controls +became `diagnostic-label-token-not-label`, and every label control now covers both labels. A run +of up to eight concatenated labels (`BearerBearer`) is one label, nine blank the diagnostic, the +literal query names join the merged pass, an ambiguous authority is masked through its end, and +the scrubber repeats its pass to a fixed point. Nineteen controls were added: the label run, its +embedded stop, its not-a-label refusal (`{1,8}` weakened to `{1}`), its stack bound and its +leading quote run; the header's concatenated run and its `}`/`'` stops; the nine-label bound; +the query merge; the merge extent; the fixed point; the ambiguous-authority extent and its host +length bound (with core twins for the last two); and core's URL-scheme lookahead after a scheme +label, its split-userinfo edges and its fixed point. With the merged pair, 109 controls remain: +`diagnostic-label-concat-bound` was retired as an equivalent mutant: the label rule's not-a-label bound is reached only by nine or more +concatenated labels, which the nine-label bound blanks first, or in the whitespace-stripped +residual, where a label rule (which needs whitespace after the label) never matches. The header's +bound stays controlled through the residual of a separate-label run. Edges that only change how +much is masked, never whether a token leaks (a word-glued label is no label, whitespace before a +header colon, a host after an ambiguous authority staying visible), are pinned by tests without a +control, which keeps the family inside the 2,000-file export bound. Three guards are pinned only +by the admitted-size case deadline: the label token's first-character lookahead and the URL rule's +`://` prefilter are output-equivalent, and the header stop's bounded escape run only decides how +many escapes past fifteen fold into the mask; removing any of them brings back a quadratic scan, +and the coordinator counts a timeout as infrastructure, never as a removal's failure, so they have +no control. + +The fifth round found that core's round-4 authority extent erased the context its later rules +read: core's rules run in sequence, so masking through an ambiguous authority's end swallowed the +key name, label, scheme or PEM marker of the next credential, and in the LF/TAB-joined view it ran +across a real line break. Core now masks an ambiguous userinfo together with exactly the query +values its opaque-query and secret key/value rules would mask whole, ending where those rules end +them, and masks the userinfo only in the joined view; the URL-scheme lookahead after a scheme label +is gone, because the split view now masks URL userinfo before scheme labels, as the main pass +always did. Three core controls were replaced by three for those mechanisms +(`core-userinfo-query-extent`, `core-userinfo-joined-extent`, +`core-split-userinfo-before-scheme`); `core-userinfo-split-scheme` controls a separator inside a +URL's `scheme://`, which now splits its userinfo as one at the userinfo's edges does (core had left +`https:/\n/user:pass@host` unmasked since before the increment); and `diagnostic-url-split-retry` +controls the split scan's retry of a kept candidate glued across a line break (`files\nftp://` +read as `sftp://`). `diagnostic-label-concat-refusal` is retired to make room, and its test stays. +The family is now 110 controls, the most the unchanged 2,000-file export bound admits (110 × 18 + 7 += 1,987). From this round on, a control pins each new leak-bearing mechanism, and the pieces inside +a mechanism's pattern (one character of a delimiter class, a repetition bound, a flag, a fixed +point's fail-closed branch) are pinned by discriminating tests instead: the fifth round's tests for +the label rule's trailing `;`/`,`, its embedded stop's closing quote run and bound, the nine-label +bound's case and LF/TAB letters, the exactly masked password, the escaped joined header name and +both fixed-point helpers were each checked against the mutation they kill. + +The sixth round found that round 5's value mirror rescanned the text from every query assignment +(quadratic at admitted size), still masked through a later credential's key name, label or PEM +marker, and was dropped by the split view, which treated a URL after any line break as split. +Core now works like the scrubber: every rule reports its ranges on one text and overlapping ranges +merge before anything is replaced, so no mask removes another rule's context. A key or parameter +in a URL userinfo's own text — all of an unambiguous userinfo, and a `user:password` before a raw +`?`/`#` — belongs to it; the key/value and opaque-query rules read the rest of an ambiguous +userinfo. The key/value rule resumes its scan at each value, so a key inside a value is read too, +and reads values through earlier masks, line breaks included; the scheme-label rule resumes after +each label; an LF/TAB-split header label, and a scheme label even where a key's value runs on over +it, may start right after a credential (one helper, `afterMasks`, fills the credentials before them +as applied masks would); and PEM blocks are found with one shared END search instead of a bounded +rescan after every BEGIN marker. Six controls pin the new mechanisms (`core-merged-ranges`, +`core-userinfo-query-zone`, `core-key-value-nested`, `core-key-value-through-masks`, +`core-scheme-nested-label`, `core-mask-boundaries`). The three round-5 core controls are gone with their mechanisms; +`core-joined-credential-spans`, `core-joined-original-boundary`, `core-userinfo-split-scheme`, +`core-userinfo-split-edges`, `diagnostic-label-run-bound` and `query-names` are re-anchored; and +three single-character pieces of the header rule (`diagnostic-header-stop-brace`, +`diagnostic-header-stop-quote`, `diagnostic-header-backslash-first`) are retired to make room, each +after its test was checked to still kill its mutation. Pinned by tests without a control: the +userinfo zone's colon and unambiguous branches, the opaque rule's parameter view, the fill's +boundary brackets, the PEM body bound (20,000 and 20,001), the scrubber's raw-text nine-label check +and `#` query names, and both maskers' fail-closed values, observed through a module mock of the +fixed-point helper that keeps its implementation. The split view still maps a URL's scheme past a +separator right before it; that only spares reporting the main view's range a second time, so the +output cannot show it and it has neither a test nor a control. The shared END search is pinned by +the admitted-size case deadline alone, like round 4's deadline-only guards. + +The seventh round found that a key in a URL userinfo's own text could still hide a real +credential. An unambiguous userinfo owned every key in it, so a key in its password or parameter +position (`jdbc:sqlserver://db:1433;user=sa;password=P@ss…`, `https://svc;a=1;password=P@ss…`) and, +in the LF/TAB-joined view, a key on the next line (`DATABASE_URL=https://h\nDB_PASSWORD=P@ss…`) left +its value's text after the last `@` visible. Only a userinfo's username — the text before its first +`:` when no raw `?`/`#` comes before that colon — now owns its keys, and never past an LF or TAB. The +quoted-value pattern's repeated group exhausted V8's RegExp stack at an exactly admitted 8-MiB value, +after a write had landed; quoted values are now found by a character scan. A credential glued to the +end of a fixed-length one (`AKIA…ASIA…`) is a credential of its own, so a run of them is masked in one +pass, and a split span glued to the one before it is bounded by it as by an applied mask. Summaries +recognize split shapes with the projection's fixed point. A projection that does not settle becomes +one constant, which would give two different calls one journal digest, so a tiered call whose +journal projection does not settle is refused before its prepare, its argument names masked. The +family also could not export its evidence: one export of its 110 controls measured about 77 MiB +against the unchanged 64-MiB bound, and no author gate exercised the export, because only the +dedicated job sets `RELAVIUM_ACCEPTANCE_EVIDENCE`. It now runs as two shards +([causal-controls](../causal-controls/README.md)), and both ran locally with the export enabled. +Ten controls were added: `core-userinfo-username-zone`, `core-split-zone-cap`, +`core-quoted-value-scan`, `core-standalone-chain`, `core-glued-span-boundary`, +`core-summary-fixed-point`, `core-journal-settled` and `core-unsettled-refusal`, and +`diagnostic-label-run-normalized` and `diagnostic-label-run-raw`, which remove one half of the +nine-label bound each, since removing the whole line (`diagnostic-label-run-bound`, kept) left the +normalized half untested. `core-userinfo-query-zone` is re-anchored and no control is retired. +Pinned by tests without a control: the username zone's colon comparisons, the merged run's +replacement, the fill's merge and order and the place of opaque values among the masks, the key +rule's keyword alternatives, suffix and boundary and the value's delimiters, the quoted scan's escape +and line-break rules, the refused argument names' masking and the helper's settled report. The header +label join's `:` prefilter only skips work whose result is the text unchanged, so the output cannot +show it and it has neither a test nor a control. + +The eighth round found that round 7's glued-credential chain could start a narrower family inside a +wider credential or swallow a key name or label after it, and that four older gaps remained. The +chain and its glued-span exemption are gone: a glued credential is read at the earlier mask's +boundary on the next pass, and a run that never settles is redacted and refused. A structured +result's summary is read from its JSON as one text once credential members and flag arguments are +masked whole; quoted values are read one JSON string down, also behind escaped tabs and line breaks, +where an apostrophe opens a value too; a quoted value that does not close on its line, or one JSON +string down before its string ends, is decided once the pass's other masks are known, masked whole +when they join its lines and it closes and otherwise read as an unquoted run from after its quote, +no further than that end, while a summary masks one stopped by a line break through a later quote +that could close it (else to the end of its line) and one that reaches the end of its string or of +the text to that end; two member names that project alike make the journal projection unsettled; a +header label at a line start inside a split continuation is read through a second label view; and +the scrubber's split probe also reads the raw text when terminal normalization changed it. The +round's password-position finding is documented rather than changed: such a key's value runs to the +key rule's delimiter, through a URL's host and path, and calls that differ only there share a +projection, as recognized credentials do. Before the commit the author ran the reviewers' method on +the candidate, and six internal pre-reviews reviewed it. The first found that the draft's +alternatives each opened a gap, and they were dropped: an authority-end rule that trusted a +host-shaped tail revealed a password tail BASE had masked, quoted values that crossed lines took a +quote that closes a string for one that opens a value, and a line-start credential inside a +continuation made the split scan quadratic. The second found that the summary's pre-pass could erase +the key of a nested multi-line value and that JSON escaping hid such values in structured results, +that a value one JSON string down ran across a raw line break behind a backslash, and that two +member names projecting alike dropped a member from the journal identity. The third found that the +round's own additions failed: masking a value that never closes to the end of its line, decided from +the end of the text back, merged the journal identities of calls whose text after a quote that +closes a string around the key differed, so such a value is read as an unquoted run instead (the +author's distinctness oracle then found the same merge one JSON string down, where a value ran to +the end of its string, which now ends its run too); and the scrubber's reading of JSON-escaped tabs +after a header's colon and its view of terminal sequences' final bytes each let a token leak or +blanked coloured diagnostics, so both are withdrawn and their cases documented as limits. The fourth +found that the run, deciding a value in the pass whose other rule also masked its line break, and +the opaque mask's kept backslash showed values that every earlier version masked whole in its next +pass, so such a value is now decided once the pass's other masks are known and the opaque mask takes +the backslash again (the value one JSON string down that needed it is a documented limit); it also +found that a terminal hyperlink to a bare origin before a later `@` blanks a diagnostic through the +raw split probe, now a documented trade-off. The fifth found that projecting a structured result's +strings one by one masked the keys and PEM markers that reading its JSON needs across members, so a +structured summary reads its JSON as one text again and the per-string cache and summary-only runs +over line breaks are withdrawn. The sixth found that the closing view and the summary's closer read +a quote's escape after a nested key's value had taken the backslash before it, which showed a +value's tail that base had masked; that a structured summary's walk took a private-key marker that a +block across members needs; and that a structured summary read no JSON-escaped line break or tab, a +gap base shares. A quoted value now closes at the later of two readings, with the masks before and +inside it filled and with each mask keeping the backslashes it ends with, which also masks the tail +that an opaque value's mask unescaped, behind a URL userinfo (which base masked) or not (which base +showed too); the walk keeps a marker whose partner lies outside its member; and every summary also +reads a view in which each escaped line feed, carriage return or tab is that separator. A structured +summary's walk also became linear in its members at a small constant factor (one media-redaction +walk serves both texts, that walk applies no text rule to a string because the text rules read the +masked JSON once, and the key and label rules skip text that cannot hold them), and a many-part +result takes a case deadline. Core's four eight-MiB cases take the scrubber's case deadline in place +of their wall-clock bounds. `core-standalone-chain` and `core-glued-span-boundary` are removed with +their mechanisms; `core-quoted-value-scan`, `core-joined-original-boundary`, +`core-summary-fixed-point`, `core-key-value-through-masks`, `core-key-value-nested` and +`core-journal-settled` are re-anchored; twelve controls are added (`core-summary-walk`, +`core-escaped-quote`, `core-unclosed-run`, `core-unclosed-closing-view`, `core-continuation-label`, +`core-summary-through-closer`, `core-summary-whole-json`, `core-journal-key-collision`, +`core-escaped-value-start`, `core-summary-escape-view`, `core-summary-pem-markers`, +`diagnostic-split-probe-raw`); and `core-unsettled-refusal` gains an unmocked witness and one for +its whole-projection fallback. Pinned by tests without a control, each killed by its mutation: the +separator before an escaped quote in the key rule; the escaped scan's decoding, its raw line breaks, +also behind a backslash, and the raw quote that ends its string; a plain value's CR and backslash +line ends, a lone backslash at the end of the text and the end of the text that leaves a value open; +the run's start after its quote, its limit at its line's end, its six-character floor and its +delimiters, which unquoted values share, the closing view's reread, its guard on a quote that +another range takes and its second reading, the unquoted values' shared delimiter scan, the key +rule's second reading of a quoted value, the backslashes each mask keeps in it and the later of the +two readings; the raw tab, escaped CR, escaped apostrophe and raw apostrophe behind escaped +separators; the summary's closers (the masked values' coverage, the nearest outside closer, the +furthest masked one, the fill, the text end, the order by end, an escaped value's quote kind, the +closers' backslash parity, even, 4k+1 and 4k+3, and the second reading's closers, masks and escapes +with the later of the two), its fail-closed value, the summary projection in both passes, and the +undefined and unserializable results; the conditions of the key and scheme rules' early exits and of +the second scheme-label read, whose removal is output-neutral; the collision report inside arrays +and under keys; the second label view's first-separator fill, the fill's opening bracket and the `'` +delimiter; the structured summary's walk-only values; the escaped view's line feed, carriage return +and tab, its doubled separator, its run of backslashes and each of the three rules that read it; the +kept markers' first END and last BEGIN, the conditions under which a marker's partner lies inside +its member, and the whole mask through arrays, members and both spellings of a flag argument, with +the journal's and the display's own; the scrubber's host cut at a query or a fragment and the TAB +half of its split retry; and the registry's masked, deduplicated refused names and its +whole-projection fallback. Two and three PEM blocks in one text pin the shared END search's +re-search, whose removal loops until the test worker runs out of memory, so no control can certify +it. Four mutations are output-neutral and have neither a test nor a control: the kept markers' +`-----` prefilter and the two shortcuts that reuse a view when no mask ends with a backslash, which +only skip work, and preferring a second reading that closes at all, since a reading that keeps the +masks' backslashes closes only at a quote where the other reading has closed by then. + +The ninth round found two high findings in round 8's own code and eight medium regressions against +base in five shapes. Round 8 compared whole views with `===` inside its loops and walked a run of +backslashes once per mask that ends in it, both quadratic at admitted size: each view of a quoted +value is now built once per pass, its presence a value decided once, and a run's start is remembered +at every end it serves. A value whose line break a scheme label's token takes did not close where +another key's value took its closing quote or ended at its opening one, and a reading closed early +where a filled scheme token would have let it reach its real quote: such a value is now reread with +and without the key rule's own masks, and the key rule's first decision also reads a view with the +masks' backslashes kept and the scheme tokens filled, which only moves a closing that a plain +reading reaches. A summary took one reading's early closing over another reading that stops at a +line break, and now masks such a value through its later closer. A structured summary's walk +replaced credential members and flag arguments before the text rules read the JSON: the summary now +reads the unwalked JSON once, each masked value spliced back where the walk left a private-use +placeholder, with the walk's masks as ranges of that pass, and round 8's kept PEM markers are +withdrawn. A summary's pass before normalization cut a private-key marker that a line break, a tab +or more than one space splits, and a summary now reads such markers. `core-summary-pem-markers`, +`core-summary-whole-json` and `core-unclosed-closing-view` are re-anchored, and +`core-key-value-through-masks` now targets round 7's reading through earlier masks, since the +unclosed value's reread also reads round 6's quoted witnesses; no control is added, and a control +for the later closing is deferred. Pinned by tests without a control, each killed by its +mutation: the quoted value's plain and scheme views, the gate that keeps a scheme view to a closing +a plain reading reaches, and the latest closing; the summary's stopped reading and its later closer; +the unclosed value's views with and without the key rule's masks, their kept variants, the guard on +a view that fills the value's quote and the latest reread; the unwalked reading's placeholders, +whole and inside a string, a value with no JSON form through its `??` fallback, its mask and its +collision guard; the scheme-labelled and argument whole masks; the wrapped markers in the summary +alone, their whitespace between `PRIVATE` and `KEY` and the prefilter; and the opaque parameter in a +URL username. (Corrected after round 10: the no-JSON-form value's try/catch and the whitespace inside +the wrapped markers' type words have no killing test; both gaps are deferred.) An admitted-size case deadline +pins the four quadratic shapes in the journal, display and summary paths; a timeout counts as +infrastructure, so no control can certify it. Five mutations are output-neutral and have neither a +test nor a control: the closing mask of a summary value that also stops at a line break, which its +later closer covers; a kept run's clamp to its range, whose empty ranges a fill skips anyway; and +three shortcuts that keep or reuse a view when no mask ends with a backslash. + +The tenth round, the last under the maintainer's round cap, found no high finding. The maintainer +counted one realistic gap that base shares as high: in a structured result's JSON a query value at +the end of a line runs on through the next line's key and takes the backslash of its value's escaped +quote, so a `cat .env` result's password showed in its summary. A summary now opens such a value +where the earlier mask keeps its trailing backslashes; the journal keeps its reading. +`core-quoted-value-scan` and `core-escaped-quote` are re-anchored on the changed lines; no control is +added. Pinned by a test without a control, each killed by its mutation: the summary-only gate, the +kept opening and both of its reading sets. Two mutations are output-neutral and have neither a test +nor a control: the condition that the filled view holds no backslash at the opening and the +condition that the kept view holds one, since a mask the kept view keeps ends before a delimiter, +where a view that keeps the backslashes closes no later than the filled one. The round's other +findings are recorded in deferred-tasks.md. After the closure, CI's coverage job took round 9's +linear-views case past the case deadline, since instrumented coverage slows every projection, so that +case became one case per shape (four cases, the same inputs and assertions). The thin entry point uses the same coordinator, outcome and bounded process join as the filesystem family in [causal-controls](../causal-controls/README.md). diff --git a/tools/secrets-controls/mutations.mjs b/tools/secrets-controls/mutations.mjs index 96d47e90..c7008a2f 100644 --- a/tools/secrets-controls/mutations.mjs +++ b/tools/secrets-controls/mutations.mjs @@ -66,6 +66,134 @@ const rawPasswordAssignments = ['?', '#'].map( (separator) => `preserves public ${separator} tails after userinfo and masks raw password assignments`, ); +// PR95: the extended URL scheme list, the header scheme label and the raw ?/# contact predicate. +const urlSchemes = String.raw`https?|wss?|ftps?|postgres(?:ql)?|mysql|mariadb|rediss?|mongodb(?:\+srv)?|amqps?|ssh|sftp|ldaps?|smtps?`; +const colonlessSchemes = ['http', 'https', 'ws', 'wss', 'ftp', 'ftps']; +const connectionSchemes = [ + 'postgres', + 'postgresql', + 'mysql', + 'mariadb', + 'redis', + 'rediss', + 'mongodb', + 'mongodb+srv', + 'amqp', + 'amqps', + 'ssh', + 'sftp', + 'ldap', + 'ldaps', + 'smtp', + 'smtps', +]; +const headerSchemeLabelMarker = 'SECRETS_B2_HEADER_SCHEME_LABEL'; +const headerSchemeLabelCases = [ + ...['api-key', 'x-api-key', 'x-goog-api-key', 'X-Api-Key'].map( + (name) => `masks the token after a scheme label in the ${name} header`, + ), + 'masks every LF/TAB split position inside a scheme label after a header', + 'masks the next token, failing closed, when a scheme label ends the line', + 'masks the token after stacked or quoted scheme labels in a header value', + 'masks the token after more than one whitespace character following a scheme label', + 'masks quoted and escaped tokens beside header scheme labels', +]; +const schemeLabel = String.raw`B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c`; +const headerName = String.raw`(?:(?:x-)?api-key|x-goog-api-key)\\*["']?\s*:`; +// The header rule, piece by piece, so a control can remove or weaken exactly one guard. +const header = { + boundary: '(?[^\s"',;}\\][^\s"',;}]*?)`, + stop: String.raw`(?=(?<=[^A-Za-z0-9_-])${headerName}|\\{0,15}[\s"',;}]|\\{0,15}$)`, +}; +const headerLine = (part = {}) => { + const value = { ...header, ...part }; + return ( + ` /${value.boundary}(?:(?:x-)?api-key|x-goog-api-key)${value.separator}${value.labelRun}` + + `${value.notLabel}${value.prefix}${value.notNested}${value.token}${value.stop}/dgi,` + ); +}; +// The one standalone label rule, piece by piece; its first label is either label. +const label = { + first: String.raw`\b(?:${schemeLabel})`, + concat: String.raw`(?:[\\"']*(?:${schemeLabel})){0,7}`, + close: String.raw`[\\"']*\s+`, + notLabel: String.raw`(?![^\w\s]*(?:(?:${schemeLabel})[\\"']*){1,8}(?:\s|$))`, + lead: String.raw`[\\"']*`, + token: String.raw`(?(?=[^\s\\"'])\S*?[^\s\\"',;}])`, + trail: String.raw`[\\"',;}]*`, + stop: String.raw`(?=(?<=[^\w])(?:${schemeLabel})(?:[\\"']*(?:${schemeLabel})){0,7}[\\"']*(?:\s|$)|\s|$)`, +}; +const labelLine = (part = {}) => { + const value = { ...label, ...part }; + return ( + ` /${value.first}${value.concat}${value.close}${value.notLabel}${value.lead}${value.token}` + + `${value.trail}${value.stop}/dgi,` + ); +}; +const mergedMasking = [ + ' return maskRanges(', + ' text,', + ' SECRET_RULES.flatMap((rule) => ruleRanges(text, rule)),', + ' );', +].join('\n'); +const queryRule = (names) => String.raw`/[?&#](?:${names})=(?[^&\s#]+)/dgi`; +const queryNames = + '(?:x[-_])?api[-_]?key|api[-_]?secret|(?:access|secret)[-_]?key|(?:access|refresh|id|auth|session)[-_]?token|client[-_]?secret|key|token|auth|password|secret'; +const overlapMerge = ' if (open !== undefined && range.start < open.end) {'; +const mergeExtent = ' open.end = Math.max(open.end, range.end);'; +const hostFollowsLine = + ' return host.length <= HOST_SHAPED_MAX_LENGTH && HOST_SHAPED_AUTHORITY.test(host);'; +const llmFixedPoint = ' return toFixedPoint(scrubOnce, text, MASKED);'; +const coreFixedPoint = " return toFixedPoint(redactSecretShapedOnce, text, '[redacted]');"; +const coreUserinfoZone = + ' const ownsUntil = colon >= 0 && (query < 0 || colon < query) ? colon : 0;'; +const headerTokenMask = ' mask: tokenGroup,\n joinedPattern:'; +const stackedLabelCase = + 'masks a header token after up to eight stacked labels and fails closed beyond them'; +const standaloneStackCase = 'masks the token after any number of stacked standalone labels'; +const fixedPointCase = 'keeps already-masked scheme tokens as fixed points'; +const hostLengthGuard = ' if (query > HOST_SHAPED_MAX_LENGTH) return false;'; +const hostShapedLine = String.raw` /^(?:[a-z\d-]+(?:\.[a-z\d-]+)+\.?|localhost|\[[\da-f:.]+\])(?::\d{1,5})?$/i;`; +const hostShapedUnanchored = String.raw` /(?:[a-z\d-]+(?:\.[a-z\d-]+)+\.?|localhost|\[[\da-f:.]+\])(?::\d{1,5})?/i;`; +const hostLookalikeCases = [ + 'https://john.doe:s3cret?x=1@h.example/', + 'https://localhostuser:pw?a=b@h.example/', + 'https://user:pw.evil.com?x=1@h.example/', + 'https://h.example:123456?x=1@h.example/', +].map((input) => `masks userinfo whose prefix only resembles a host: ${input}`); +const userinfoContactMarker = 'SECRETS_B2_USERINFO_CONTACT'; +const coreUserinfoContactMarker = 'SECRETS_B2_CORE_USERINFO_CONTACT'; +const contactAssignmentCases = (secondInput) => + ['https://my.user:1234?TOKEN@h.example/', secondInput].map( + (input) => `masks a host-shaped prefix that no assignment follows: ${input}`, + ); +const contactHostCase = + 'masks userinfo whose raw ?/# follows no host-shaped prefix: https://us?er=1:TOKEN@h.example/'; +const contactAssignment = + " return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query)) && userinfo.includes('=', query + 1);"; +const connectionSchemeMarker = 'SECRETS_B2_CONNECTION_SCHEME'; +// sftp also matches the original list by its ftp suffix, so the old list still masks it. +const connectionSchemeCases = connectionSchemes + .filter((scheme) => scheme !== 'sftp') + .map((scheme) => `masks colon-bearing ${scheme} userinfo and keeps a bare user`); +const urlJoinedMarker = 'SECRETS_B2_URL_JOINED'; +const urlJoinedCases = (schemes) => [ + ...schemes.map((scheme) => `masks a split ${scheme} credential around an inner masked span`), + ...['?', '#'].map( + (raw) => `masks a split credential holding a raw ${raw} and an inner masked span`, + ), +]; +const urlGluedMarker = 'SECRETS_B2_URL_GLUED_SCHEME'; +const urlGluedCases = ['\n', '\t'].map( + (separator) => + `masks a glued scheme whose split credential holds a raw @ (${JSON.stringify(separator)})`, +); const originalBoundaryMarker = 'SECRETS_B2_ORIGINAL_BOUNDARY'; const originalBoundaryCases = ['sk-proj-', 'AIza'].flatMap((prefix) => ['\n', '\t'].map( @@ -131,18 +259,6 @@ const coreOriginalBoundary = String.raw` const candidates = /(^|[^\w])([A-Za-z] } } -function redactStandaloneCredentials(text: string): string { - const parts: string[] = []; - let copied = 0; - visitJoinedStandaloneCredentials(text, text, (start, end, replacement) => { - parts.push(text.slice(copied, start), replacement); - copied = end; - }); - if (parts.length === 0) return text; - parts.push(text.slice(copied)); - return parts.join(''); -} - /** * The finite header-label literal is the sole joined-header authority. Check ORIGINAL boundaries * before matching, so an embedded x-api-key/x-goog-api-key cannot consume an inner api-key start. @@ -177,23 +293,25 @@ function visitJoinedHeaderLabels(original: string, joined: string, mask: SecretS } /** - * LF/TAB may split finite credential shapes. Map recognized spans back onto the original text; - * ordinary spaces remain boundaries and public text is never globally normalized for identity. - * Token-character continuations after LF/TAB are intentionally part of a recognized credential. - * The two cursors advance monotonically, with no per-character offset array or recursive matcher. + * LF/TAB may split finite credential shapes. Map the spans recognized in the joined view back onto + * the original text and report the ones a removed separator actually splits; ordinary spaces remain + * boundaries and public text is never globally normalized for identity. Token-character + * continuations after LF/TAB are intentionally part of a recognized credential. The two cursors + * advance monotonically, with no per-character offset array or recursive matcher. */ -function redactJoinedCredentialSpans( +function visitSplitSpans( text: string, visit: (joined: string, mask: SecretSpanVisitor) => void, + report: SecretSpanVisitor, boundary?: RegExp, -): string { - if (!/[\n\t]/.test(text)) return text; + // URL userinfo owns a separator at either edge: one right after the scheme or before the @ splits it. + ownsEdges = false, +): void { + if (!/[\n\t]/.test(text)) return; const joined = text.replace(/[\n\t]+/g, ''); - const parts: string[] = []; const separators = /[\n\t]+/g; let next = separators.exec(text); let removed = 0; - let copied = 0; function advance(target: number, includeAtBoundary = false): number { while ( next !== null && @@ -204,9 +322,12 @@ function redactJoinedCredentialSpans( } return target + removed; } - visit(joined, (start, end, replacement) => { - const originalStart = advance(start, true); - const originalEnd = advance(end); + visit(joined, (start, end, replacement, url) => { + // A separator right before a URL's scheme is outside the URL, so it splits nothing. + const schemeStart = url === undefined ? undefined : advance(url.schemeStart, true); + const originalStart = advance(start, !ownsEdges); + const ownsUntil = url === undefined ? undefined : advance(url.ownsUntil, ownsEdges); + const originalEnd = advance(end, ownsEdges); if (boundary?.test(text[originalStart - 1] ?? '')) return;`; const withoutCoreOriginalBoundary = coreOriginalBoundary .replace( @@ -214,6 +335,8 @@ const withoutCoreOriginalBoundary = coreOriginalBoundary String.raw` const candidates = /([A-Za-z])/g;`, ) .replace(" if (boundary?.test(text[originalStart - 1] ?? '')) return;", ''); +/** The complete core projection test file every core projection control runs. */ +const CORE_PROJECTION_CASES = 234; function coreProjectionControl(id, before, after, targets, marker = coreProjectionMarker) { return control( id, @@ -221,12 +344,16 @@ function coreProjectionControl(id, before, after, targets, marker = coreProjecti before, after, coreProjectionCases, - 142, + CORE_PROJECTION_CASES, targets[0], marker, targets.slice(1).map((target) => ({ target, marker })), ); } +/** The complete registry test file the journal-identity control runs. */ +const REGISTRY_CASES = 116; +/** The complete scrubber test file every LLM diagnostic control runs. */ +const LLM_SCRUB_CASES = 224; function diagnosticControl(id, before, after, targets, marker = diagnosticMarker) { return control( id, @@ -234,7 +361,7 @@ function diagnosticControl(id, before, after, targets, marker = diagnosticMarker before, after, ['packages/llm/src/scrub-secrets.test.ts'], - 103, + LLM_SCRUB_CASES, targets[0], marker, targets.slice(1).map((target) => ({ target, marker })), @@ -342,8 +469,8 @@ export const mutations = [ control( 'query-names', scrubSecretsLeaf, - String.raw`/([?&](?:(?:x[-_])?api[-_]?key|api[-_]?secret|(?:access|secret)[-_]?key|(?:access|refresh|id|auth|session)[-_]?token|client[-_]?secret|key|token|auth|password|secret)=)[^&\s#]+/gi`, - String.raw`/([?&](?:(?:x[-_])?api[-_]?key|key|token|access[-_]?token|auth|password|secret)=)[^&\s#]+/gi`, + queryRule(queryNames), + queryRule('(?:x[-_])?api[-_]?key|key|token|access[-_]?token|auth|password|secret'), ['packages/llm/src/llm-error.test.ts'], 98, 'SECRETS_CAUSAL_QUERY_NAMES masks each added family without another credential shape', @@ -458,8 +585,14 @@ export const mutations = [ control( 'non-query-before-query', scrubSecretsLeaf, - ' return scrubQuerySecrets(scrubNonQuerySecrets(text));', - ' return scrubNonQuerySecrets(scrubQuerySecrets(text));', + mergedMasking, + [ + ' const queried = applyRule(text, QUERY_SECRET_RULE);', + ' return maskRanges(', + ' queried,', + ' NON_QUERY_SECRET_RULES.flatMap((rule) => ruleRanges(queried, rule)),', + ' );', + ].join('\n'), overlapFiles, 278, `${overlapMarker} protects client_secret Bearer before query redaction`, @@ -750,7 +883,7 @@ export const mutations = [ ' if (scrubSecrets(probe) !== probe) matches += 1;', ' void probe;', ['apps/cli/src/authoring/snapshot-notice.test.ts'], - 10, + 11, 'detects existing credential shapes without values or arbitrary keys in the notice', 'SNAPSHOT_CAUSAL_DETECTOR', ), @@ -912,8 +1045,8 @@ export const mutations = [ ), coreProjectionControl( 'core-joined-credential-spans', - ' visit(joined, (start, end, replacement) => {', - " visit('', (start, end, replacement) => {", + ' visit(joined, (start, end, replacement, url) => {', + " visit('', (start, end, replacement, url) => {", [ 'masks every LF/TAB split without losing the surrounding public identity', ...splitCoreHeaders, @@ -961,16 +1094,644 @@ export const mutations = [ ), diagnosticControl( 'diagnostic-split-header-label', - String.raw`joinedPattern: /(?:(?:x-)?api-key|x-goog-api-key)["']?\s*:/gi,`, + String.raw`joinedPattern: /(?:(?:x-)?api-key|x-goog-api-key)\\*["']?\s*:/gi,`, 'joinedPattern: /a^/g,', ['masks each split position in the finite API header names'], ), diagnosticControl( 'diagnostic-raw-password-userinfo', - String.raw`const URL_USERINFO_PATTERN = /(?:https?|wss?|ftps?):\/\/[^/\s]*@/gi;`, - String.raw`const URL_USERINFO_PATTERN = /(?:https?|wss?|ftps?):\/\/[^/\s?#]*@/gi;`, + String.raw` /(?:${urlSchemes}):\/\/[^/\s]*@/gi;`, + String.raw` /(?:${urlSchemes}):\/\/[^/\s?#]*@/gi;`, [...diagnosticPasswords, ...rawPasswordAssignments], ), + // PR95: a header value's leading Bearer/Basic labels are skipped and its token is masked. Without + // the label run, the probes see an unmasked label and token and blank the whole diagnostic. + diagnosticControl( + 'diagnostic-header-scheme-label', + headerLine(), + headerLine({ labelRun: '' }), + headerSchemeLabelCases, + headerSchemeLabelMarker, + ), + control( + 'diagnostic-header-stacked-labels', + scrubSecretsLeaf, + headerLine(), + headerLine({ labelRun: header.labelRun.replace(String.raw`\s+){0,8}`, String.raw`\s+)?`) }), + ['packages/llm/src/scrub-secrets.test.ts'], + LLM_SCRUB_CASES, + 'masks the token after stacked or quoted scheme labels in a header value', + headerSchemeLabelMarker, + [{ target: stackedLabelCase, marker: 'SECRETS_B2_STACKED_LABEL' }], + ), + diagnosticControl( + 'diagnostic-header-label-whitespace', + headerLine(), + headerLine({ labelRun: header.labelRun.replace(String.raw`\s+){0,8}`, String.raw`\s){0,8}`) }), + ['masks the token after more than one whitespace character following a scheme label'], + headerSchemeLabelMarker, + ), + // Every quantified label group is bounded so no admitted-size run can exhaust the RegExp stack. + diagnosticControl( + 'diagnostic-header-label-bound', + headerLine(), + headerLine({ labelRun: header.labelRun.replace(String.raw`\s+){0,8}`, String.raw`\s+)*`) }), + ['keeps admitted-size label runs linear and fails closed'], + 'SECRETS_B2_HEADER_LABEL_BOUND', + ), + diagnosticControl( + 'diagnostic-header-concat-bound', + headerLine(), + headerLine({ notLabel: header.notLabel.replace('{1,8}', '+') }), + ['keeps admitted-size label runs linear and fails closed'], + 'SECRETS_B2_HEADER_LABEL_BOUND', + ), + // Round 4: a run of up to eight concatenated labels is one label, in a header value and alone. + diagnosticControl( + 'diagnostic-header-concat-run', + headerLine(), + headerLine({ + labelRun: header.labelRun.replace(String.raw`(?:[\\"']*(?:${schemeLabel})){0,7}`, ''), + }), + ['masks the token after a concatenated label run in every context'], + 'SECRETS_B4_CONCAT_LABEL', + ), + ...[ + ['diagnostic-label-concat-run', { concat: '' }], + ['diagnostic-label-concat-stop', { stop: label.stop.replace(label.concat, '') }], + ].map(([id, part]) => + diagnosticControl( + id, + labelLine(), + labelLine(part), + ['masks the token after a concatenated label run in every context'], + 'SECRETS_B4_CONCAT_LABEL', + ), + ), + // The concatenated run is bounded; the residual of a separate-label run reaches it. + diagnosticControl( + 'diagnostic-label-start-bound', + labelLine(), + labelLine({ concat: label.concat.replace('{0,7}', '*') }), + ['keeps admitted-size label runs linear and fails closed'], + 'SECRETS_B2_HEADER_LABEL_BOUND', + ), + diagnosticControl( + 'diagnostic-label-lead', + labelLine(), + labelLine({ lead: '' }), + ['masks a quoted standalone token after either label'], + 'SECRETS_B4_LABEL_LEAD', + ), + // Nine concatenated labels exceed every bounded run, so the diagnostic fails closed, read in the + // normalized view and in the raw text alike (round 7: each half has its own control). + ...[ + [ + 'diagnostic-label-run-normalized', + ' if (LABEL_RUN_PAST_BOUND.test(text)) return MASKED;', + 'fails closed on nine labels that only the normalized view joins', + ], + [ + 'diagnostic-label-run-raw', + ' if (LABEL_RUN_PAST_BOUND.test(normalized)) return MASKED;', + 'fails closed on nine labels behind a terminal sequence', + ], + ].map(([id, after, target]) => + diagnosticControl( + id, + ' if (LABEL_RUN_PAST_BOUND.test(normalized) || LABEL_RUN_PAST_BOUND.test(text)) return MASKED;', + after, + [target], + 'SECRETS_B4_LABEL_RUN_BOUND', + ), + ), + diagnosticControl( + 'diagnostic-label-run-bound', + ' if (LABEL_RUN_PAST_BOUND.test(normalized) || LABEL_RUN_PAST_BOUND.test(text)) return MASKED;', + '', + [ + 'fails closed on nine concatenated labels and keeps eight in place', + 'keeps admitted-size label runs total, concatenated or separate', + ], + 'SECRETS_B4_LABEL_RUN_BOUND', + ), + // A header token is never a label or a run of labels that whitespace or the end follows, nor a + // nested header name. + diagnosticControl( + 'diagnostic-header-token-not-label', + headerLine(), + headerLine({ notLabel: '' }), + ['treats a whitespace-separated labels-only header value alike in every context'], + 'SECRETS_B3_LABEL_CONTEXT', + ), + ...[ + ['diagnostic-header-not-nested', { notNested: '' }], + [ + 'diagnostic-header-nested-stop', + { stop: header.stop.replace(`(?<=[^A-Za-z0-9_-])${headerName}|`, '') }, + ], + ['diagnostic-header-lookbehind', { boundary: '(?:^|[^A-Za-z0-9_-])' }], + ].map(([id, part]) => + diagnosticControl( + id, + headerLine(), + headerLine(part), + ['keeps every header value and label token apart from a nested header name'], + 'SECRETS_B3_NESTED_HEADER', + ), + ), + // Quotes and escapes around a header token, also in JSON-encoded JSON. + ...[ + [ + 'diagnostic-header-escaped-separator', + { separator: header.separator.replace(String.raw`\\*["']?\s*:`, String.raw`["']?\s*:`) }, + ], + ['diagnostic-header-quote-run', { prefix: String.raw`(?:\\*["'])?` }], + ].map(([id, part]) => + diagnosticControl( + id, + headerLine(), + headerLine(part), + ['masks a header token after any run of quotes or escapes, also in JSON-encoded JSON'], + 'SECRETS_B3_HEADER_ESCAPES', + ), + ), + diagnosticControl( + 'diagnostic-header-escaped-stop', + headerLine(), + headerLine({ + stop: header.stop.replace( + String.raw`|\\{0,15}[\s"',;}]|\\{0,15}$)`, + String.raw`|[\s"',;}]|$)`, + ), + }), + [fixedPointCase], + 'SECRETS_B2_MASKED_FIXED_POINT', + ), + // The standalone label rule: no label as a token, labels closed by a quote, glued labels. + diagnosticControl( + 'diagnostic-label-token-not-label', + labelLine(), + labelLine({ notLabel: '' }), + [standaloneStackCase], + 'SECRETS_B2_STACKED_LABEL', + ), + diagnosticControl( + 'diagnostic-label-quoted-label', + labelLine(), + labelLine({ close: String.raw`\s+` }), + ['masks the token after a label closed by its own quote'], + 'SECRETS_B3_QUOTED_LABEL', + ), + ...[ + [ + 'diagnostic-label-glued-label', + { notLabel: label.notLabel.replace(String.raw`[^\w\s]*`, String.raw`[\\"']*`) }, + ], + [ + 'diagnostic-label-embedded-stop', + { + stop: label.stop.replace( + String.raw`(?<=[^\w])(?:${schemeLabel})${label.concat}[\\"']*(?:\s|$)|`, + '', + ), + }, + ], + ].map(([id, part]) => + diagnosticControl( + id, + labelLine(), + labelLine(part), + ['masks the token after a label glued behind punctuation or inside another token'], + 'SECRETS_B3_GLUED_LABEL', + ), + ), + // Reintroducing round 2's masked-marker guard exposes a tail glued to the marker again. + diagnosticControl( + 'diagnostic-label-masked-tail', + labelLine(), + labelLine({ + lead: String.raw`(?![\\"']*\[REDACTED\](?:[\s"',;}]|$))` + label.lead, + }), + ["masks a tail glued to a masked token, including the scrubber's own masks"], + 'SECRETS_B3_MASKED_TAIL', + ), + // Masks come from the original text and are merged; each token rule masks only its token. + diagnosticControl( + 'diagnostic-merged-masking', + mergedMasking, + ' return SECRET_RULES.reduce((masked, rule) => applyRule(masked, rule), text);', + ['masks the token after a label glued behind punctuation or inside another token'], + 'SECRETS_B3_GLUED_LABEL', + ), + // Round 4: applying the query names after the merged pass loses a value a header token cut short. + diagnosticControl( + 'diagnostic-query-merge', + mergedMasking, + [ + ' return applyRule(', + ' maskRanges(', + ' text,', + ' NON_QUERY_SECRET_RULES.flatMap((rule) => ruleRanges(text, rule)),', + ' ),', + ' QUERY_SECRET_RULE,', + ' );', + ].join('\n'), + ['masks a literal query value in the same pass as every other rule'], + 'SECRETS_B4_QUERY_MERGE', + ), + diagnosticControl( + 'diagnostic-mask-merge-extent', + mergeExtent, + ' open.end = range.end;', + ['masks the whole of a range that contains a later one'], + 'SECRETS_B4_MERGE_EXTENT', + ), + diagnosticControl( + 'diagnostic-fixed-point', + llmFixedPoint, + ' return scrubOnce(text);', + ['reaches a fixed point'], + 'SECRETS_B4_FIXED_POINT', + ), + diagnosticControl( + 'diagnostic-mask-merge', + overlapMerge, + ' if (open !== undefined && false) {', + headerSchemeLabelCases, + headerSchemeLabelMarker, + ), + diagnosticControl( + 'diagnostic-token-only-mask', + headerTokenMask, + ' mask: WHOLE_MATCH,\n joinedPattern:', + [fixedPointCase], + 'SECRETS_B2_MASKED_FIXED_POINT', + ), + // PR95 round 1: the host-shaped prefix is bounded before the RegExp and anchored at both ends. + diagnosticControl( + 'diagnostic-userinfo-host-length', + hostLengthGuard, + '', + [ + 'masks an admitted-size dotted userinfo prefix without exhausting the RegExp stack', + 'bounds the host-shaped prefix at exactly 260 characters, case-insensitively', + ], + 'SECRETS_B2_USERINFO_HOST_LENGTH', + ), + coreProjectionControl( + 'core-userinfo-host-length', + hostLengthGuard, + '', + [ + 'masks an admitted-size dotted userinfo prefix without exhausting the RegExp stack', + 'bounds the host-shaped prefix at exactly 260 characters, case-insensitively', + ], + 'SECRETS_B2_CORE_USERINFO_HOST_LENGTH', + ), + diagnosticControl( + 'diagnostic-userinfo-host-anchors', + hostShapedLine, + hostShapedUnanchored, + hostLookalikeCases, + userinfoContactMarker, + ), + coreProjectionControl( + 'core-userinfo-host-anchors', + hostShapedLine, + hostShapedUnanchored, + hostLookalikeCases, + coreUserinfoContactMarker, + ), + // PR95: raw ?/# userinfo is public only after a host-shaped prefix AND an assignment. + diagnosticControl( + 'diagnostic-userinfo-contact-assignment', + contactAssignment, + ' return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query));', + contactAssignmentCases('https://host.example#sec@x.example'), + userinfoContactMarker, + ), + control( + 'diagnostic-userinfo-contact-host', + scrubSecretsLeaf, + contactAssignment, + " return userinfo.includes('=', query + 1);", + ['packages/llm/src/scrub-secrets.test.ts'], + LLM_SCRUB_CASES, + contactHostCase, + userinfoContactMarker, + rawPasswordAssignments.map((target) => ({ target, marker: diagnosticMarker })), + ), + coreProjectionControl( + 'core-userinfo-contact-assignment', + contactAssignment, + ' return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query));', + contactAssignmentCases('https://provider.example#section@x.example'), + coreUserinfoContactMarker, + ), + control( + 'core-userinfo-contact-host', + coreProjection, + contactAssignment, + " return userinfo.includes('=', query + 1);", + coreProjectionCases, + CORE_PROJECTION_CASES, + contactHostCase, + coreUserinfoContactMarker, + rawPasswordAssignments.map((target) => ({ target, marker: coreProjectionMarker })), + ), + diagnosticControl( + 'diagnostic-userinfo-authority-rest', + ' return { start, end: start + at + (hostFollows(tail) ? 0 : 1 + tail.length) };', + ' return { start, end: start + at };', + ['masks an ambiguous authority through its end unless a host follows its last @'], + 'SECRETS_B4_AUTHORITY_REST', + ), + diagnosticControl( + 'diagnostic-userinfo-rest-length', + hostFollowsLine, + ' return HOST_SHAPED_AUTHORITY.test(host);', + ['bounds the host test after an ambiguous authority at admitted size'], + 'SECRETS_B4_REST_LENGTH', + ), + // Round 6: core's rules report ranges on one text and merge them, as the scrubber's do. An + // ambiguous userinfo's query is read by the later rules, a key or scheme label inside a value + // starts its own credential, a value runs through earlier masks, and a split header label, a + // scheme label or a credential may start right after another credential. + coreProjectionControl( + 'core-merged-ranges', + ' if (run !== undefined && range.start < run.end) {', + ' if (run !== undefined && false) {', + [ + 'keeps every credential and its context when core masks overlap', + ...['\n', '\t', 'word\t', 'DATABASE_URL:\n'].map( + (prefix) => + `masks an ambiguous userinfo with its query values behind ${JSON.stringify(prefix)}`, + ), + 'masks an ambiguous userinfo whose scheme is split by an LF', + ], + 'SECRETS_B6_CORE_MERGE', + ), + coreProjectionControl( + 'core-userinfo-query-zone', + coreUserinfoZone, + ' const ownsUntil = at;', + [ + 'masks an ambiguous userinfo with the query values its later rules would mask whole', + 'masks an admitted-size query value inside an ambiguous userinfo', + ], + 'SECRETS_B5_CORE_QUERY_EXTENT', + ), + coreProjectionControl( + 'core-key-value-nested', + ' if (delimiterAt - valueStart >= 6) mask(match.index, delimiterAt);', + ' if (delimiterAt - valueStart >= 6) {\n mask(match.index, delimiterAt);\n key.lastIndex = delimiterAt;\n }', + ["reads a key inside another key's value"], + 'SECRETS_B6_CORE_NESTED_KEY', + ), + coreProjectionControl( + 'core-key-value-through-masks', + ' const values = fillRanges(text, masked);', + ' const values = names;', + ['reads a value through earlier masks of every kind, in text order'], + 'SECRETS_B6_CORE_VALUE_THROUGH_MASK', + ), + coreProjectionControl( + 'core-scheme-nested-label', + ' scheme.lastIndex = labelEnd;', + '', + [ + "reads a scheme label inside another label's token", + 'keeps nested values, labels and open PEM blocks linear at admitted size', + ], + 'SECRETS_B6_CORE_NESTED_LABEL', + ), + coreProjectionControl( + 'core-mask-boundaries', + ' return fillRanges(text, ranges);', + ' return text;', + [ + 'joins a split header label right after a credential or a PEM block', + 'reads a scheme label or a credential glued to the end of another credential', + ], + 'SECRETS_B6_CORE_MASK_BOUNDARY', + ), + coreProjectionControl( + 'core-userinfo-split-scheme', + ' schemeStart: match.index,', + ' schemeStart: start,', + ['\n', '\t'].map( + (separator) => + `masks URL userinfo whose scheme is split by an LF or TAB (${JSON.stringify(separator)})`, + ), + 'SECRETS_B5_CORE_SPLIT_SCHEME', + ), + coreProjectionControl( + 'core-userinfo-split-edges', + ' visitSplitSpans(text, visitUrlUserinfo, report, undefined, true);', + ' visitSplitSpans(text, visitUrlUserinfo, report);', + ['\n', '\t'].map( + (separator) => + `masks URL userinfo split at either of its edges (${JSON.stringify(separator)})`, + ), + 'SECRETS_B4_CORE_SPLIT_EDGE', + ), + // Round 7: only a username owns its userinfo's text, and none past an LF or TAB; quoted values are + // scanned without a RegExp repeat; the summary uses the projection's fixed point; and an effect whose + // journal projection never settles is refused before it is prepared. + coreProjectionControl( + 'core-userinfo-username-zone', + coreUserinfoZone, + ' const ownsUntil = query < 0 ? at : colon >= 0 && colon < query ? query : 0;', + ["reads a key in a userinfo's password or parameter position"], + 'SECRETS_B7_CORE_USERNAME_ZONE', + ), + coreProjectionControl( + 'core-split-zone-cap', + ' : { schemeStart, ownsUntil: split < 0 ? ownsUntil : originalStart + split },', + ' : { schemeStart, ownsUntil },', + ["owns no text past a split userinfo's first LF or TAB"], + 'SECRETS_B7_CORE_SPLIT_ZONE', + ), + coreProjectionControl( + 'core-quoted-value-scan', + ' const readings = kept === undefined ? [quotedValueEnd(values, valueStart), ...plain] : plain;', + String.raw` const quoted = opening === '"' ? /"(?:[^"\\\r\n]|\\[^\r\n])*"/y : /'(?:[^'\\\r\n]|\\[^\r\n])*'/y; + quoted.lastIndex = valueStart; + const value = quoted.test(values) + ? { end: quoted.lastIndex, closed: true, lineBreak: false } + : { end: valueStart, closed: false, lineBreak: true }; + const readings = kept === undefined ? [value, ...plain] : plain;`, + ['projects exactly admitted-size quoted values without exhausting the RegExp stack'], + 'SECRETS_B7_CORE_QUOTED_SCAN', + ), + coreProjectionControl( + 'core-summary-fixed-point', + String.raw` const recognized = /[\n\t]/.test(text) ? redactSecretShapedSummaryText(text) : text;`, + String.raw` const recognized = /[\n\t]/.test(text) ? redactSecretShapedOnce(text, true) : text;`, + ["summarizes with the projection's fixed point before one-line normalization"], + 'SECRETS_B7_CORE_SUMMARY', + ), + coreProjectionControl( + 'core-journal-settled', + " settled = false;\n return '[redacted]';", + " return '[redacted]';", + ['reports whether a journal projection settled'], + 'SECRETS_B7_CORE_JOURNAL_SETTLED', + ), + control( + 'core-unsettled-refusal', + 'packages/core/src/tools/registry.ts', + ' if (journal.unsettled.length > 0) {', + ' if (false) {', + ['packages/core/src/tools/registry.test.ts'], + REGISTRY_CASES, + 'refuses a tiered effect whose journal projection does not settle, before preparing it', + 'SECRETS_B7_CORE_UNSETTLED_REFUSAL', + [ + 'refuses a real tiered effect whose arguments never settle', + 'names every argument when only the whole projection does not settle', + ].map((target) => ({ target, marker: 'SECRETS_B7_CORE_UNSETTLED_REFUSAL' })), + ), + // Round 8: a structured result is summarized from its projection, quoted values are read one JSON + // string down, one that does not close on its line is read as an unquoted run unless the pass's other + // masks join its lines, and a label may start at a line start inside a split continuation. + coreProjectionControl( + 'core-summary-walk', + ' const summary = makeSummary(summaryText);', + ' const summary = makeSummary(text);', + ['summarizes a structured result from its JSON once credential members are masked'], + 'SECRETS_B8_CORE_SUMMARY_WALK', + ), + coreProjectionControl( + 'core-escaped-quote', + ' const opening = escaped ? opener[valueStart + 1] : opener[valueStart];', + ' const opening = opener[valueStart];', + ['reads a quoted value one JSON string down'], + 'SECRETS_B8_CORE_ESCAPED_QUOTE', + ), + coreProjectionControl( + 'core-unclosed-run', + " if (end - from >= 6) report(value.start, end, '[redacted]');", + '', + ['reads a quoted value that does not close on its line as an unquoted run'], + 'SECRETS_B8_CORE_UNCLOSED_QUOTE', + ), + coreProjectionControl( + 'core-unclosed-closing-view', + ' views.push(fillRanges(text, list));', + ' views.push(values);', + ['reads a quoted value that does not close on its line as an unquoted run'], + 'SECRETS_B8_CORE_UNCLOSED_QUOTE', + ), + // Pre-review: the summary masks a value that a later quote could close through that quote and reads + // a structured result's JSON as one text, a journal projection whose member names collide does not + // settle, one JSON level down a value opens behind escaped tabs and line breaks, and a value that + // does not close is decided once the pass's other masks are known. + coreProjectionControl( + 'core-summary-through-closer', + " return toFixedPoint((value) => redactSecretShapedOnce(value, true), text, '[redacted]');", + " return toFixedPoint((value) => redactSecretShapedOnce(value, false), text, '[redacted]');", + ['summarizes a value that closes on a later line whole'], + 'SECRETS_B8_CORE_SUMMARY_CLOSER', + ), + coreProjectionControl( + 'core-summary-whole-json', + ' (text) => text,', + ' redactSecretShapedSummaryText,', + ["reads a structured result's values across its line breaks and members"], + 'SECRETS_B8_CORE_SUMMARY_JSON', + ), + coreProjectionControl( + 'core-journal-key-collision', + ' if (Object.hasOwn(out, name)) collide?.();', + '', + ['reports a journal projection whose member names collide as unsettled'], + 'SECRETS_B8_CORE_KEY_COLLISION', + ), + coreProjectionControl( + 'core-escaped-value-start', + ' const valueStart = escapedValueStart(values, key.lastIndex);', + ' const valueStart = key.lastIndex;', + ['reads a quoted value one JSON string down'], + 'SECRETS_B8_CORE_ESCAPED_QUOTE', + ), + coreProjectionControl( + 'core-continuation-label', + ' for (const boundaries of new Set([afterMasks(text, masks), afterMasks(text, firstLines)])) {', + ' for (const boundaries of new Set([afterMasks(text, masks)])) {', + ['reads a label at a line start inside a split continuation'], + 'SECRETS_B8_CORE_CONTINUATION_LABEL', + ), + // Sixth pre-review and round 9: a summary reads the line breaks and tabs that JSON escaping hides, and + // reads a structured result's JSON before the walk's whole masks, which that pass applies. + coreProjectionControl( + 'core-summary-escape-view', + ' if (throughCloser) visitEscapedSeparators(text, record);', + '', + ["reads a structured summary's JSON-escaped line breaks and tabs"], + 'SECRETS_B8_CORE_SUMMARY_ESCAPES', + ), + coreProjectionControl( + 'core-summary-pem-markers', + ' return replaceRanges(text, [...secretRanges(text, true), ...masks]);', + ' return replaceRanges(text, masks);', + ["reads a structured summary's members before the walk masks them whole"], + 'SECRETS_B8_CORE_SUMMARY_PEM', + ), + coreProjectionControl( + 'core-fixed-point', + coreFixedPoint, + ' return redactSecretShapedOnce(text);', + ['reaches a fixed point'], + 'SECRETS_B4_CORE_FIXED_POINT', + ), + diagnosticControl( + 'diagnostic-split-probe-raw', + " if (rawJoined !== undefined && hasSplitCredential(text, rawJoined, rule)) return '[REDACTED]';", + '', + ['blanks a split credential whose first letter a bare terminal introducer would take'], + 'SECRETS_B8_SPLIT_RAW', + ), + diagnosticControl( + 'diagnostic-url-split-retry', + String.raw` /[\n\t]/.test(text[originalOffset + 1] ?? '') &&`, + ' false &&', + ['blanks an LF/TAB-split ftp credential after a line that ends in s'], + 'SECRETS_B5_SPLIT_SFTP', + ), + // PR95: connection-string schemes join the finite URL list. + diagnosticControl( + 'diagnostic-connection-scheme-userinfo', + String.raw` /(?:${urlSchemes}):\/\/[^/\s]*@/gi;`, + String.raw` /(?:https?|wss?|ftps?):\/\/[^/\s]*@/gi;`, + connectionSchemeCases, + connectionSchemeMarker, + ), + // PR95: only the joined scanner sees a split credential around an inner masked span. + diagnosticControl( + 'diagnostic-joined-url-userinfo', + ' joinedPattern: URL_USERINFO_PATTERN,', + ' joinedPattern: /a^/g,', + urlJoinedCases([...colonlessSchemes, ...connectionSchemes]), + urlJoinedMarker, + ), + // PR95: the URL rule blocks no start, so a glued scheme is scanned like any other. + diagnosticControl( + 'diagnostic-url-glued-scheme', + ' boundary: NO_BOUNDARY,', + String.raw` boundary: /\w/,`, + urlGluedCases, + urlGluedMarker, + ), + // PR95: the `://` prefilter window must cover every listed scheme (`ws` alone fits in two). + diagnosticControl( + 'diagnostic-url-scheme-window', + " anchor: { text: '://', window: URL_SCHEME_WINDOW },", + " anchor: { text: '://', window: 2 },", + urlJoinedCases([...colonlessSchemes, ...connectionSchemes].filter((scheme) => scheme !== 'ws')), + urlJoinedMarker, + ), // R1-LN-1: recover a valid original start hidden inside a rejected greedy joined candidate. coreProjectionControl( 'core-joined-original-boundary-recovery',